From: erickson Date: Tue, 7 Apr 2009 21:07:34 +0000 (+0000) Subject: moved marc upload into Order.pm X-Git-Tag: sprint4-merge-nov22~10304 X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=40eecf21e1b2022863b474146594c3094421cd4c;p=working%2FEvergreen.git moved marc upload into Order.pm git-svn-id: svn://svn.open-ils.org/ILS/trunk@12815 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Acq/Order.pm b/Open-ILS/src/perlmods/OpenILS/Application/Acq/Order.pm index 9b48cd5176..933b5670bd 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Acq/Order.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Acq/Order.pm @@ -14,6 +14,7 @@ sub new { picklist => undef, complete => 0 }; + $self->{cache} = {}; return $self; } @@ -24,6 +25,9 @@ sub conn { } sub respond { my($self, %other_args) = @_; + if($self->throttle and not %other_args) { + return unless ($self->progress % $self->throttle) == 0; + } $self->conn->respond({ %{$self->{args}}, %other_args }); } sub respond_complete { @@ -82,6 +86,13 @@ sub complete { return $self; } +sub cache { + my($self, $org, $key, $val) = @_; + $self->{cache}->{$org} = {} unless $self->{cache}->{org}; + $self->{cache}->{$org}->{$key} = $val if defined $val; + return $self->{cache}->{$org}->{$key}; +} + package OpenILS::Application::Acq::Order; use base qw/OpenILS::Application/; @@ -95,9 +106,13 @@ use OpenSRF::Utils::Logger qw(:logger); use OpenILS::Utils::Fieldmapper; use OpenILS::Utils::CStoreEditor q/:funcs/; use OpenILS::Const qw/:const/; +use OpenSRF::EX q/:try/; use OpenILS::Application::AppUtils; use OpenILS::Application::Cat::BibCommon; use OpenILS::Application::Cat::AssetCommon; +use MARC::Record; +use MARC::Batch; +use MARC::File::XML; my $U = 'OpenILS::Application::AppUtils'; @@ -259,7 +274,7 @@ sub create_purchase_order { $po->create_time('now'); $po->ordering_agency($mgr->editor->requestor->ws_ou); $po->$_($args{$_}) for keys %args; - return $mgr->editor->create_acq_purchase_order($po); + return $mgr->purchase_order($mgr->editor->create_acq_purchase_order($po)); } @@ -291,17 +306,16 @@ sub create_lineitem_assets { # for each lineitem_detail, create the volume if necessary, create # a copy, and link them all together. # ----------------------------------------------------------------- - my %cache; for my $lid_id (@{$li_details}) { my $lid = $mgr->editor->retrieve_acq_lineitem_detail($lid_id) or return 0; my $org = $lid->owning_lib; my $label = $lid->cn_label; - $cache{$org} = {} unless $cache{$org}; - my $volume = $cache{$org}{$label}; + my $volume = $mgr->cache($org, "cn.$label"); unless($volume) { - $volume = $cache{$org}{$label} = create_volume($li, $lid) or return 0; + $volume = create_volume($li, $lid) or return 0; + $mgr->cache($org, "cn.$label", $volume); } create_copy($mgr, $volume, $lid) or return 0; } @@ -478,7 +492,8 @@ sub upload_records { my $e = new_editor(authtoken => $auth, xact => 1); return $e->die_event unless $e->checkauth; - my $mgr = OpenILS::Application::Acq::BatchManager->new(editor => $e, conn => $conn); + my $mgr = OpenILS::Application::Acq::BatchManager->new( + editor => $e, conn => $conn, throttle => 5); my $cache = OpenSRF::Utils::Cache->new; my $evt; @@ -509,11 +524,10 @@ sub upload_records { } if($create_po) { - $purchase_order = Fieldmapper::acq::purchase_order->new; - $purchase_order->provider($provider->id); - $purchase_order->ordering_agency($ordering_agency); - $evt = OpenILS::Application::Acq::Financials::create_purchase_order_impl($e, $purchase_order); - return $evt if $evt; + my $po = create_purchase_order($mgr, + ordering_agency => $ordering_agency, + provider => $provider->id + ) or return $mgr->editor->die_event; } $logger->info("acq processing MARC file=$filename"); @@ -527,6 +541,7 @@ sub upload_records { while(1) { my $r; + $count++; $logger->info("processing record $count"); try { @@ -534,15 +549,19 @@ sub upload_records { } catch Error with { $r = -1; }; last unless $r; + + $logger->info("found record $count"); if($r == -1) { $logger->warn("Proccessing of record $count in set $key failed. Skipping this record"); - $count++; next; } + $logger->info("HERE 1 $count"); try { + $logger->info("HERE 2 $count"); + (my $xml = $r->as_xml_record()) =~ s/\n//sog; $xml =~ s/^<\?xml.+\?\s*>//go; $xml =~ s/>\s+entityize($xml); $xml =~ s/[\x00-\x1f]//go; - my $li = Fieldmapper::acq::lineitem->new; - $li->picklist($picklist->id) if $picklist; - $li->purchase_order($purchase_order->id) if $purchase_order; - $li->source_label($provider->code); # XXX ?? - $li->provider($provider->id); - $li->selector($e->requestor->id); - $li->creator($e->requestor->id); - $li->editor($e->requestor->id); - $li->edit_time('now'); - $li->create_time('now'); - $li->marc($xml); - $li->state('on-order') if $purchase_order; - $e->create_acq_lineitem($li) or die $e->die_event; - - $conn->respond({count => $count}) if (++$count % 5) == 0; - - $evt = create_lineitem_details($conn, \$count, $e, $ordering_agency, $li, $purchase_order); - die $evt if $evt; # caught below + $logger->info("extracted xml for record $count : $xml"); + + my %args = ( + source_label => $provider->code, + provider => $provider->id, + marc => $xml, + ); + + $args{picklist} = $picklist->id if $picklist; + if($purchase_order) { + $args{purchase_order} = $purchase_order->id; + $args{state} = 'on-order'; + } + + my $li = create_lineitem($mgr, %args); + $mgr->respond; + $logger->info("created lineitem"); + + # XXX XXX + #$evt = create_lineitem_details($conn, \$count, $e, $ordering_agency, $li, $purchase_order); + #die $evt if $evt; # caught below } catch Error with { my $error = shift; $logger->warn("Encountered a bad record at Vandelay ingest: ".$error); - } + }; + + return $e->event if $e->died; } $e->commit; unlink($filename); $cache->delete_cache('vandelay_import_spool_' . $key); - # clear the cached funds - delete $fund_code_map{$_} for keys %fund_code_map; - return { complete => 1, purchase_order => $purchase_order, @@ -589,6 +610,7 @@ sub upload_records { }; } +=head WUT WUT? sub create_lineitem_details { my($conn, $countref, $e, $ordering_agency, $li, $purchase_order) = @_; @@ -663,5 +685,6 @@ sub extract_lineitem_detail_data { return \%compiled; } +=cut 1; diff --git a/Open-ILS/src/perlmods/OpenILS/Application/Acq/Picklist.pm b/Open-ILS/src/perlmods/OpenILS/Application/Acq/Picklist.pm index ff9ef1b9d5..36e22dd166 100644 --- a/Open-ILS/src/perlmods/OpenILS/Application/Acq/Picklist.pm +++ b/Open-ILS/src/perlmods/OpenILS/Application/Acq/Picklist.pm @@ -17,6 +17,7 @@ use MARC::File::XML; use MIME::Base64; use Digest::MD5 qw/md5_hex/; use OpenILS::Application::Acq::Financials; +use DateTime; my $U = 'OpenILS::Application::AppUtils'; @@ -409,109 +410,6 @@ sub retrieve_pl_lineitem { request open-ils.cstore open-ils.cstore.json_query.atomic {"select":{"jub":[{"transform":"count", "attregate":1, "column":"id","alias":"count"}]}, "from":"jub","where":{"picklist":1}} =cut -__PACKAGE__->register_method( - method => 'zsearch', - api_name => 'open-ils.acq.picklist.search.z3950', - stream => 1, - signature => { - desc => 'Performs a z3950 federated search and creates a picklist and associated lineitems', - params => [ - {desc => 'Authentication token', type => 'string'}, - {desc => 'Search definition', type => 'object'}, - {desc => 'Picklist name, optional', type => 'string'}, - ] - } -); - -sub zsearch { - my($self, $conn, $auth, $search, $name, $options) = @_; - my $e = new_editor(authtoken=>$auth); - return $e->event unless $e->checkauth; - return $e->event unless $e->allowed('CREATE_PICKLIST'); - - $search->{limit} ||= 10; - $options ||= {}; - - my $ses = OpenSRF::AppSession->create('open-ils.search'); - my $req = $ses->request('open-ils.search.z3950.search_class', $auth, $search); - - my $first = 1; - my $picklist; - while(my $resp = $req->recv(timeout=>60)) { - - if($first) { - $e = new_editor(requestor=>$e->requestor, xact=>1); - $picklist = zsearch_build_pl($self, $conn, $auth, $e, $name); - $first = 0; - } - - my $result = $resp->content; - my $count = $result->{count}; - my $total = (($count < $search->{limit}) ? $count : $search->{limit})+1; - my $ctr = 0; - $conn->respond({total=>$total, progress=>++$ctr}); - - for my $rec (@{$result->{records}}) { - my $li = Fieldmapper::acq::lineitem->new; - $li->picklist($picklist->id); - $li->source_label($result->{service}); - $li->selector($e->requestor->id); - $li->creator($e->requestor->id); - $li->editor($e->requestor->id); - $li->edit_time('now'); - $li->create_time('now'); - $li->marc($rec->{marcxml}); - $li->state('new'); - $li->eg_bib_id($rec->{bibid}) if $rec->{bibid}; - $e->create_acq_lineitem($li) or return $e->die_event; - - my $response = {total => $total, progress => ++$ctr}; - - if($$options{respond_li}) { - $response->{lineitem} = $li; - $li->attributes($e->search_acq_lineitem_attr({lineitem => $li->id})) - if $$options{flesh_attrs}; - $li->clear_marc if $$options{clear_marc}; - } - - $conn->respond($response); - } - } - - $e->commit; - return {complete=>1, picklist_id=>$picklist->id}; -} - -sub zsearch_build_pl { - my($self, $conn, $auth, $e, $name) = @_; - - $name ||= ''; - my $picklist = $e->search_acq_picklist({owner=>$e->requestor->id, name=>$name})->[0]; - if($name eq '' and $picklist) { - my $evt = delete_picklist($self, $conn, $auth, $picklist->id); - return $evt unless $evt == 1; - $picklist = undef; - } - - unless($picklist) { - $picklist = Fieldmapper::acq::picklist->new; - $picklist->owner($e->requestor->id); - $picklist->creator($e->requestor->id); - $picklist->editor($e->requestor->id); - $picklist->edit_time('now'); - $picklist->name($name); - $picklist->org_unit($e->requestor->ws_ou); - $e->create_acq_picklist($picklist) or return $e->die_event; - - } else { - $picklist->editor($e->requestor->id); - $picklist->edit_time('now'); - $e->update_acq_picklist($picklist) or return $e->die_event; - } - - return $picklist; -} - __PACKAGE__->register_method( @@ -544,119 +442,3 @@ sub ranged_distrib_formulas { return undef; } -__PACKAGE__->register_method( - method => 'upload_records', - api_name => 'open-ils.acq.process_upload_records', - stream => 1, -); - -sub upload_records { - my($self, $conn, $auth, $key) = @_; - my $e = new_editor(authtoken => $auth, xact => 1); - return $e->die_event unless $e->checkauth; - my $cache = OpenSRF::Utils::Cache->new; - - my $data = $cache->get_cache("vandelay_import_spool_$key"); - my $purpose = $data->{purpose}; - my $filename = $data->{path}; - my $provider = $data->{provider}; - my $picklist = $data->{picklist}; - my $create_po = $data->{create_po}; - my $ordering_agency = $data->{ordering_agency}; - my $purchase_order; - - unless(-r $filename) { - $logger->error("unable to read MARC file $filename"); - $e->rollback; - return OpenILS::Event->new('FILE_UPLOAD_ERROR', payload => {filename => $filename}); - } - - $provider = $e->retrieve_acq_provider($provider) or return $e->die_event; - - if($picklist) { - $picklist = $e->retrieve_acq_picklist($picklist) or return $e->die_event; - if($picklist->owner != $e->requestor->id) { - return $e->die_event unless - $e->allowed('CREATE_PICKLIST', $picklist->org_unit, $picklist); - } - } - - if($create_po) { - $purchase_order = Fieldmapper::acq::purchase_order->new; - $purchase_order->provider($provider->id); - $purchase_order->ordering_agency($ordering_agency); - my $evt = OpenILS::Application::Acq::Financials::create_purchase_order_impl($e, $purchase_order); - return $evt if $evt; - } - - $logger->info("acq processing MARC file=$filename"); - - my $marctype = 'USMARC'; # ? - my $batch = new MARC::Batch ($marctype, $filename); - $batch->strict_off; - - my $count = 0; - - while(1) { - - my $r; - $logger->info("processing record $count"); - - try { - $r = $batch->next - } catch Error with { $r = -1; }; - - last unless $r; - - if($r == -1) { - $logger->warn("Proccessing of record $count in set $key failed. Skipping this record"); - $count++; - next; - } - - try { - - (my $xml = $r->as_xml_record()) =~ s/\n//sog; - $xml =~ s/^<\?xml.+\?\s*>//go; - $xml =~ s/>\s+entityize($xml); - $xml =~ s/[\x00-\x1f]//go; - - my $li = Fieldmapper::acq::lineitem->new; - $li->picklist($picklist->id) if $picklist; - $li->purchase_order($purchase_order->id) if $purchase_order; - $li->source_label($provider->code); # XXX ?? - $li->provider($provider->id); - $li->selector($e->requestor->id); - $li->creator($e->requestor->id); - $li->editor($e->requestor->id); - $li->edit_time('now'); - $li->create_time('now'); - $li->marc($xml); - $li->state('on-order') if $purchase_order; - $e->create_acq_lineitem($li) or die $e->die_event; - - $conn->respond({count => $count}) if (++$count % 5) == 0; - - } catch Error with { - my $error = shift; - $logger->warn("Encountered a bad record at Vandelay ingest: ".$error); - } - } - - $e->commit; - unlink($filename); - $cache->delete_cache('vandelay_import_spool_' . $key); - - return { - complete => 1, - purchase_order => $purchase_order, - picklist => $picklist - }; -} - - - - -1;