From 49e6ad16fa507a6b436fb31c57fa9dddd1aeb75a Mon Sep 17 00:00:00 2001 From: erickson Date: Thu, 15 May 2008 02:32:16 +0000 Subject: [PATCH] Merged revisions 9593-9595,9599-9601,9606 via svnmerge from svn://svn.open-ils.org/ILS/trunk ........ r9593 | miker | 2008-05-13 17:38:53 -0400 (Tue, 13 May 2008) | 1 line mod_perl stub for uploading vandelay files ........ r9594 | miker | 2008-05-13 17:42:43 -0400 (Tue, 13 May 2008) | 1 line permacrud, field_safe and sequence updates ........ r9595 | miker | 2008-05-13 22:37:52 -0400 (Tue, 13 May 2008) | 1 line spool processing ........ r9599 | miker | 2008-05-14 12:23:42 -0400 (Wed, 14 May 2008) | 1 line use a RECORD for LOOP for postgres 8.1 compat ........ r9600 | miker | 2008-05-14 12:57:04 -0400 (Wed, 14 May 2008) | 1 line moving purpose to record instead of queue ........ r9601 | miker | 2008-05-14 12:58:12 -0400 (Wed, 14 May 2008) | 1 line moving purpose to record instead of queue ........ r9606 | miker | 2008-05-14 16:31:48 -0400 (Wed, 14 May 2008) | 1 line moving purpose to record instead of queue ........ git-svn-id: svn://svn.open-ils.org/ILS/branches/acq-experiment@9612 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/examples/fm_IDL.xml | 99 ++++++++-------- .../src/perlmods/OpenILS/Application/Vandelay.pm | 131 +++++++++++++++++++-- Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm | 97 +++++++++++++++ Open-ILS/src/sql/Pg/012.schema.vandelay.sql | 16 +-- Open-ILS/src/sql/Pg/100.circ_matrix.sql | 8 +- 5 files changed, 277 insertions(+), 74 deletions(-) create mode 100644 Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index c23bc4c439..e21e47b44a 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -137,7 +137,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -146,7 +146,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - @@ -162,7 +161,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -173,6 +172,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + @@ -190,7 +190,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -213,7 +213,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -237,7 +237,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -263,7 +263,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -288,7 +288,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -298,6 +298,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + @@ -314,7 +315,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -337,7 +338,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -361,7 +362,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -477,7 +478,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -488,10 +489,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + @@ -872,7 +873,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -883,10 +884,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + @@ -963,7 +964,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -975,14 +976,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + - + @@ -993,10 +994,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + @@ -1454,7 +1455,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -1821,7 +1822,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + @@ -1832,14 +1833,14 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + - + @@ -1850,10 +1851,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - - - - + + + + @@ -2439,22 +2440,22 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + - - + + - - - - + + + + diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm index 8b71b50e83..f5d0a25a49 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Vandelay.pm @@ -7,10 +7,12 @@ use OpenSRF::EX qw/:try/; use OpenSRF::AppSession; use OpenSRF::Utils::SettingsClient; +use OpenSRF::Utils::Cache; use OpenILS::Utils::Fieldmapper; use OpenILS::Utils::CStoreEditor qw/:funcs/; +use MARC::Batch; use MARC::Record; use MARC::File::XML; @@ -49,7 +51,6 @@ sub create_bib_queue { my $name = shift; my $owner = shift; my $type = shift; - my $purpose = shift; my $e = new_editor(authtoken => $auth, xact => 1); @@ -64,6 +65,7 @@ sub create_bib_queue { my $new_id = $e->create_vandelay_bib_queue( $queue ); $e->die_event unless ($new_id); + $e->commit; $queue->id($new_id); return $queue; @@ -83,7 +85,6 @@ sub create_auth_queue { my $name = shift; my $owner = shift; my $type = shift; - my $purpose = shift; my $e = new_editor(authtoken => $auth, xact => 1); @@ -98,6 +99,7 @@ sub create_auth_queue { my $new_id = $e->create_vandelay_authority_queue( $queue ); $e->die_event unless ($new_id); + $e->commit; $queue->id($new_id); return $queue; @@ -115,22 +117,21 @@ sub add_record_to_bib_queue { my $auth = shift; my $queue = shift; my $marc = shift; + my $purpose = shift; my $e = new_editor(authtoken => $auth, xact => 1); - $queue = $e->retrieve_vandelay_bib_queue($queue) + $queue = $e->retrieve_vandelay_bib_queue($queue); return $e->die_event unless $e->checkauth; return $e->die_event unless ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) || $e->allowed('CREATE_BIB_IMPORT_QUEUE', $queue->owner)); - my $rec = new Fieldmapper::vandelay::queued_bib_record(); - $rec->marc( $marc ); - $rec->queue( $queue->id ); + my $new_id = _add_auth_rec($e, $marc, $queue->id, $purpose); - my $new_id = $e->create_vandelay_queued_bib_record( $rec ); $e->die_event unless ($new_id); + $e->commit; $rec->id($new_id); return $rec; @@ -142,39 +143,143 @@ __PACKAGE__->register_method( argc => 3, ); +sub _add_bib_rec { + my $e = shift; + my $marc = shift; + my $queue = shift; + my $purpose = shift; + + my $rec = new Fieldmapper::vandelay::queued_bib_record(); + $rec->marc( $marc ); + $rec->queue( $queue ); + $rec->purpose( $purpose ) if ($purpose); + + return $e->create_vandelay_queued_bib_record( $rec ); +} + sub add_record_to_authority_queue { my $self = shift; my $client = shift; my $auth = shift; my $queue = shift; my $marc = shift; + my $purpose = shift; my $e = new_editor(authtoken => $auth, xact => 1); - $queue = $e->retrieve_vandelay_authority_queue($queue) + $queue = $e->retrieve_vandelay_authority_queue($queue); return $e->die_event unless $e->checkauth; return $e->die_event unless ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) || $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', $queue->owner)); - my $rec = new Fieldmapper::vandelay::queued_authority_record(); - $rec->marc( $marc ); - $rec->queue( $queue->id ); + my $new_id = _add_auth_rec($e, $marc, $queue->id, $purpose); - my $new_id = $e->create_vandelay_queued_authority_record( $rec ); $e->die_event unless ($new_id); + $e->commit; $rec->id($new_id); return $rec; } -__PACKAGE__->register_method( +__PACKAGE__->register_method( api_name => "open-ils.vandelay.queued_authority_record.create", method => "add_record_to_authority_queue", api_level => 1, argc => 3, +); + +sub _add_auth_rec { + my $e = shift; + my $marc = shift; + my $queue = shift; + + my $rec = new Fieldmapper::vandelay::queued_authority_record(); + $rec->marc( $marc ); + $rec->queue( $queue ); + $rec->purpose( $purpose ) if ($purpose); + + return $e->create_vandelay_queued_authority_record( $rec ); +} + +sub process_spool { + my $self = shift; + my $client = shift; + my $auth = shift; + my $fingerprint = shift; + my $queue = shift; + + my $e = new_editor(authtoken => $auth, xact => 1); + + if ($self->{record_type} eq 'bib') { + return $e->die_event unless $e->checkauth; + return $e->die_event unless + ($e->allowed('CREATE_BIB_IMPORT_QUEUE', undef, $queue) || + $e->allowed('CREATE_BIB_IMPORT_QUEUE', $queue->owner)); + } else { + return $e->die_event unless $e->checkauth; + return $e->die_event unless + ($e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', undef, $queue) || + $e->allowed('CREATE_AUTHORITY_IMPORT_QUEUE', $queue->owner)); + } + + my $method = 'open-ils.vandelay.queued_'.$self->{record_type}.'_record.create'; + $method = $self->method_lookup( $method ); + + my $cache = new OpenSRF::Utils::Cache(); + + my $data = $cache->get_cache('vandelay_import_spool_' . $fingerprint); + my $purpose = $data->{purpose}; + $data = decode_base64($data->{marc}); + + my $fh = new IO::Scalar \$data; + + my $batch = new MARC::Batch ( $type, $fh ); + $batch->strict_off; + + my $count = 0; + while (my $r = $batch->next) { + try { + (my $xml = $rec->as_xml_record()) =~ s/\n//sog; + $xml =~ s/^<\?xml.+\?\s*>//go; + $xml =~ s/>\s+entityize($xml); + $xml =~ s/[\x00-\x1f]//go; + + if ($self->{record_type} eq 'bib') { + _add_bib_rec( $e, $xml, $queue, $purpose ); + } else { + _add_auth_rec( $e, $xml, $queue, $purpose ); + } + $count++; + + $client->respond( $count ); + } catch Error with { + my $error = shift; + $log->warn("Encountered a bad record at Vandelay ingest: ".$error); + } + } + + $e->commit; + return undef; +} +__PACKAGE__->register_method( + api_name => "open-ils.vandelay.bib.process_spool", + method => "process_spool", + api_level => 1, + argc => 3, + record_type => 'bib' +); +__PACKAGE__->register_method( + api_name => "open-ils.vandelay.auth.process_spool", + method => "process_spool", + api_level => 1, + argc => 3, + record_type => 'auth' ); + 1; diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm b/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm new file mode 100644 index 0000000000..9258681ca4 --- /dev/null +++ b/Open-ILS/src/perlmods/OpenILS/WWW/Vandelay.pm @@ -0,0 +1,97 @@ +package OpenILS::WWW::Vandelay; +use strict; +use warnings; +use bytes; + +use Apache2::Log; +use Apache2::Const -compile => qw(OK REDIRECT DECLINED NOT_FOUND :log); +use APR::Const -compile => qw(:error SUCCESS); +use APR::Table; + +use Apache2::RequestRec (); +use Apache2::RequestIO (); +use Apache2::RequestUtil; +use CGI; +use Data::Dumper; +use Text::CSV; + +use OpenSRF::EX qw(:try); +use OpenSRF::Utils::Cache; +use OpenSRF::System; +use OpenSRF::AppSession; +use XML::LibXML; + +use OpenILS::Utils::Fieldmapper; +use OpenSRF::Utils::Logger qw/$logger/; + +use MARC::Record; +use MARC::File::XML; + +use MIME::Base64; +use Digest::MD5 qw/md5_hex/; + +use UNIVERSAL::require; + +our @formats = qw/USMARC UNIMARC XML BRE/; + +# set the bootstrap config and template include directory when +# this module is loaded +my $bootstrap; + +sub import { + my $self = shift; + $bootstrap = shift; +} + + +sub child_init { + OpenSRF::System->bootstrap_client( config_file => $bootstrap ); +} + +sub spool_marc { + my $r = shift; + my $cgi = new CGI; + + my $auth = $cgi->param('ses') || $cgi->cookie('ses'); + + return Apache2::Const::FORBIDDEN unless verify_login($auth); + + + my $purpose = $cgi->param('purpose'); + my $file = $cgi->param('marc_upload'); + my $filename = "$file"; + + my $data = join '', (<$file>); + $data = encode_base64($data); + + my $data_fingerprint = md5_hex($data); + + OpenSRF::Utils::Cache()->new->put_cache( + 'vandelay_import_spool_' . $data_fingerprint, + { purpose => $purpose, marc => $data } + ); + + print "Content-type: text/plain; charset=utf-8\n\n$data_fingerprint"; + + return Apache2::Const::OK; + +} + +sub verify_login { + my $auth_token = shift; + return undef unless $auth_token; + + my $user = OpenSRF::AppSession + ->create("open-ils.auth") + ->request( "open-ils.auth.session.retrieve", $auth_token ) + ->gather(1); + + if (ref($user) eq 'HASH' && $user->{ilsevent} == 1001) { + return undef; + } + + return $user if ref($user); + return undef; +} + +1; diff --git a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql index cd0599393a..3475e199f0 100644 --- a/Open-ILS/src/sql/Pg/012.schema.vandelay.sql +++ b/Open-ILS/src/sql/Pg/012.schema.vandelay.sql @@ -10,15 +10,15 @@ CREATE TABLE vandelay.queue ( name TEXT NOT NULL, complete BOOL NOT NULL DEFAULT FALSE, queue_type TEXT NOT NULL DEFAULT 'bib' CHECK (queue_type IN ('bib','authority')), - queue_purpose TEXT NOT NULL DEFAULT 'import' CHECK (queue_purpose IN ('import','overlay')), CONSTRAINT vand_queue_name_once_per_owner_const UNIQUE (owner,name,queue_type) ); CREATE TABLE vandelay.queued_record ( - id BIGSERIAL PRIMARY KEY, - create_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), - import_time TIMESTAMP WITH TIME ZONE, - marc TEXT NOT NULL + id BIGSERIAL PRIMARY KEY, + create_time TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(), + import_time TIMESTAMP WITH TIME ZONE, + purpose TEXT NOT NULL DEFAULT 'import' CHECK (purpose IN ('import','overlay')), + marc TEXT NOT NULL ); @@ -57,9 +57,9 @@ CREATE TABLE vandelay.bib_queue ( ALTER TABLE vandelay.bib_queue ADD PRIMARY KEY (id); CREATE TABLE vandelay.queued_bib_record ( - queue INT NOT NULL REFERENCES vandelay.bib_queue (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, - bib_source INT REFERENCES config.bib_source (id) DEFERRABLE INITIALLY DEFERRED, - imported_as INT REFERENCES biblio.record_entry (id) DEFERRABLE INITIALLY DEFERRED + queue INT NOT NULL REFERENCES vandelay.bib_queue (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED, + bib_source INT REFERENCES config.bib_source (id) DEFERRABLE INITIALLY DEFERRED, + imported_as INT REFERENCES biblio.record_entry (id) DEFERRABLE INITIALLY DEFERRED ) INHERITS (vandelay.queued_record); ALTER TABLE vandelay.queued_bib_record ADD PRIMARY KEY (id); diff --git a/Open-ILS/src/sql/Pg/100.circ_matrix.sql b/Open-ILS/src/sql/Pg/100.circ_matrix.sql index bb21ae420d..3a7b122db0 100644 --- a/Open-ILS/src/sql/Pg/100.circ_matrix.sql +++ b/Open-ILS/src/sql/Pg/100.circ_matrix.sql @@ -228,7 +228,7 @@ DECLARE items_overdue INT; current_fines NUMERIC(8,2) := 0.0; tmp_fines NUMERIC(8,2); - tmp_xact BIGINT; + tmp_xact RECORD; done BOOL := FALSE; BEGIN result.success := TRUE; @@ -352,10 +352,10 @@ BEGIN -- Fail if the user has a high fine balance IF circ_test.max_fines IS NOT NULL THEN - FOR tmp_xact IN SELECT id FROM money.billable_xact WHERE usr = match_usr AND xact_finish IS NULL LOOP - SELECT INTO tmp_fines SUM( amount ) FROM money.billing WHERE xact = tmp_xact AND NOT voided; + FOR tmp_xact IN SELECT * FROM money.billable_xact WHERE usr = match_usr AND xact_finish IS NULL LOOP + SELECT INTO tmp_fines SUM( amount ) FROM money.billing WHERE xact = tmp_xact.id AND NOT voided; current_fines = current_fines + COALESCE(tmp_fines, 0.0); - SELECT INTO tmp_fines SUM( amount ) FROM money.payment WHERE xact = tmp_xact AND NOT voided; + SELECT INTO tmp_fines SUM( amount ) FROM money.payment WHERE xact = tmp_xact.id AND NOT voided; current_fines = current_fines - COALESCE(tmp_fines, 0.0); END LOOP; -- 2.11.0