--- /dev/null
+#-------------- file marc_stream_importer.conf --------------
+
+### user and group to become
+user opensrf
+group opensrf
+
+### logging ?
+log_file LOCALSTATEDIR/log/marc_stream_importer.log
+log_level 3
+pid_file LOCALSTATEDIR/run/marc_stream_importer.pid
+
+### access control
+# allow .+\.(net|com)
+# allow domain\.com
+# deny a.+
+
+### background the process?
+# background 1
+
+### ports to bind
+# host 127.0.0.1
+# port localhost:20204
+port 5544
+
+### reverse lookups ?
+# reverse_lookups on
+
+#-------------- file marc_stream_importer.conf --------------
$(examples)/lib_ips.txt.example \
$(examples)/oils_yaz.xml.example \
$(examples)/kpac.xml.example \
- $(examples)/oils_z3950.xml.example
+ $(examples)/oils_z3950.xml.example \
+ $(examples)/marc_stream_importer.conf.example
#----------------------------
# Build ILS CORE
$(supportscr)/juv_to_adult.srfsh \
$(supportscr)/thaw_expired_frozen_holds.srfsh \
$(supportscr)/long-overdue-status-update.pl \
+ $(supportscr)/oils_header.pl \
$(supportscr)/purge_at_events.srfsh \
$(supportscr)/purge_holds.srfsh \
$(supportscr)/purge_circulations.srfsh \
$(supportscr)/authority_authority_linker.pl \
$(supportscr)/eg_db_config \
$(supportscr)/marc_export \
+ $(supportscr)/marc_stream_importer.pl \
$(supportscr)/offline-blocked-list.pl
# config files that are generated from *.in files and can
$(do_subst) $(supportscr)/marc_export.in > "$@"
chmod 755 "$@"
+$(supportscr)/marc_stream_importer.pl: Makefile $(supportscr)/marc_stream_importer.pl.in
+ $(do_subst) $(supportscr)/marc_stream_importer.pl.in > "$@"
+ chmod 755 "$@"
+
$(supportscr)/offline-blocked-list.pl: Makefile $(supportscr)/offline-blocked-list.pl.in
$(do_subst) $(supportscr)/offline-blocked-list.pl.in > "$@"
chmod 755 "$@"
sed -i 's|LOCALSTATEDIR|@localstatedir@|g' '$(DESTDIR)@sysconfdir@/opensrf.xml.example'
sed -i 's|SYSCONFDIR|@sysconfdir@|g' '$(DESTDIR)@sysconfdir@/opensrf.xml.example'
sed -i 's|LIBDIR|@libdir@|g' '$(DESTDIR)@sysconfdir@/opensrf.xml.example'
+ sed -i 's|LOCALSTATEDIR|@localstatedir@|g' '$(DESTDIR)@sysconfdir@/marc_stream_importer.conf.example'
+ sed -i 's|SYSCONFDIR|@sysconfdir@|g' '$(DESTDIR)@sysconfdir@/marc_stream_importer.conf.example'
sed -i 's|BINDIR|@bindir@|g' '$(DESTDIR)@bindir@/autogen.sh'
sed -i 's|LOCALSTATEDIR|@localstatedir@|g' '$(DESTDIR)@bindir@/autogen.sh'
sed -i 's|SYSCONFDIR|@sysconfdir@|g' '$(DESTDIR)@bindir@/autogen.sh'
+++ /dev/null
-#-------------- file marc_stream_importer.conf --------------
-
-### user and group to become
-user opensrf
-group opensrf
-
-### logging ?
-log_file /openils/var/log/marc_stream_importer.log
-log_level 3
-pid_file /openils/var/run/marc_stream_importer.pid
-
-### access control
-# allow .+\.(net|com)
-# allow domain\.com
-# deny a.+
-
-### background the process?
-# background 1
-
-### ports to bind
-# host 127.0.0.1
-# port localhost:20204
-port 5544
-
-### reverse lookups ?
-# reverse_lookups on
-
-#-------------- file marc_stream_importer.conf --------------
+++ /dev/null
-#!/usr/bin/perl
-# Copyright (C) 2008-2014 Equinox Software, Inc.
-# Copyright (C) 2014 King County Library System
-# Author: Bill Erickson <berickxx@gmail.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# ---------------------------------------------------------------
-# Sends MARC records, either from a file or from data delivered
-# via the network, to open-ils.vandelay to be imported.
-# ---------------------------------------------------------------
-use strict;
-use warnings;
-use Net::Server::PreFork;
-use base qw/Net::Server::PreFork/;
-
-require 'oils_header.pl';
-use vars qw/$apputils $authtoken/;
-
-use Getopt::Long;
-use MARC::Record;
-use MARC::Batch;
-use MARC::File::XML (BinaryEncoding => 'UTF-8');
-use MARC::File::USMARC;
-use File::Basename qw/fileparse/;
-use File::Temp qw/tempfile/;
-use OpenSRF::AppSession;
-use OpenSRF::Utils::Logger qw/$logger/;
-use OpenSRF::Transport::PeerHandle;
-use OpenSRF::Utils::SettingsClient;
-
-use Data::Dumper;
-$Data::Dumper::Indent=0; # for logging
-
-# This script will always be an entry point for opensrf,
-# so go ahead and force log client.
-$ENV{OSRF_LOG_CLIENT} = 1;
-
-# these are updated with each new batch of records
-my $cur_rec_type;
-my $cur_rec_source;
-my $cur_queue;
-
-# cache these
-my $cur_merge_profile; # this is an object
-my $bib_merge_profile_obj;
-my $auth_merge_profile_obj;
-
-# options
-my $help = 0;
-my $osrf_config = '/openils/conf/opensrf_core.xml';
-my $username = '';
-my $password = '';
-my $workstation = '';
-my $tempdir = '';
-my $spoolfile = '';
-my $wait_time = 5;
-my $verbose = 0;
-my $bib_merge_profile;
-my $auth_merge_profile;
-my $bib_queue;
-my $auth_queue;
-my $bib_source;
-my $port;
-my $bib_import_no_match;
-my $bib_auto_overlay_exact;
-my $bib_auto_overlay_1match;
-my $bib_auto_overlay_best_match;
-my $auth_import_no_match;
-my $auth_auto_overlay_exact;
-my $auth_auto_overlay_1match;
-my $auth_auto_overlay_best_match;
-
-# deprecated options; these map to their bib_* equivalents
-my $import_no_match;
-my $auto_overlay_exact;
-my $auto_overlay_1match;
-my $auto_overlay_best_match;
-my $deprecated_queue;
-
-
-
-my $net_server_conf = fileparse($0, '.pl').'.conf';
-
-GetOptions(
- 'osrf-config=s' => \$osrf_config,
- 'verbose' => \$verbose,
- 'username=s' => \$username,
- 'password=s' => \$password,
- 'workstation=s' => \$workstation,
- 'tempdir=s' => \$tempdir,
- 'spoolfile=s' => \$spoolfile,
- 'wait=i' => \$wait_time,
- 'merge-profile=i' => \$bib_merge_profile,
- 'queue=i' => \$deprecated_queue,
- 'bib-queue=i' => \$bib_queue,
- 'source=i' => \$bib_source,
- 'auth-merge-profile=i' => \$auth_merge_profile,
- 'auth-queue=i' => \$auth_queue,
-
- # -- deprecated
- 'import-no-match' => \$import_no_match,
- 'auto-overlay-exact' => \$auto_overlay_exact,
- 'auto-overlay-1match' => \$auto_overlay_1match,
- 'auto-overlay-best-match' => \$auto_overlay_best_match,
- # --
-
- 'bib-import-no-match' => \$bib_import_no_match,
- 'bib-auto-overlay-exact' => \$bib_auto_overlay_exact,
- 'bib-auto-overlay-1match' => \$bib_auto_overlay_1match,
- 'bib-auto-overlay-best-match' => \$bib_auto_overlay_best_match,
- 'auth-import-no-match' => \$auth_import_no_match,
- 'auth-auto-overlay-exact' => \$auth_auto_overlay_exact,
- 'auth-auto-overlay-1match' => \$auth_auto_overlay_1match,
- 'auth-auto-overlay-best-match' => \$auth_auto_overlay_best_match,
- 'help' => \$help,
- 'net-server-config=s' => \$net_server_conf,
- 'port=i' => \$port
-);
-
-sub usage {
- print <<USAGE;
- --osrf-config
- Path to OpenSRF configuration file.
-
- --net-server-conf
- Path to Net::Server configuration file. Defaults to $net_server_conf.
- Only required if --spoolfile is not set.
-
- --verbose
- Log additional details
-
- --username
- Evergreen user account which performs the import actions.
-
- --password
- Evergreen user account password
-
- --workstation
- Evergreen workstation
-
- --tempdir
- MARC data received via the network is stored in a temporary
- file so Vandelay can access it. This must be a directory
- the open-ils.vandelay service can access. If you want the
- file deleted after completion, be sure open-ils.vandelay
- has write access to the directory and the file.
- This value defaults to the Vandelay data directory, however
- this configuratoin value is only accessible when run from
- the private opensrf domain, which you may not want to do.
-
- --spoolfile
- Path to a MARC file to load. When a --spoolfile is specified,
- this script will send the file to vandelay for processing,
- then exit when complete. In other words, it does not stay
- alive to accept requests from the network.
-
- --wait
- Amount of time in seconds this script will wait after receiving
- a connection on the socket and before recieving a complete
- MARC record. This prevents unintentional denial of service by
- clients connecting and never sending anything.
-
- --merge-profile
- ID of the vandelay bib record merge profile
-
- --queue
- ID of the vandelay bib record queue
-
- --source
- ID of the bib source
-
- --auth-merge-profile
- ID of the vandelay authority record merge profile
-
- --auth-queue
- ID of the vandelay authority record queue
-
- --bib-import-no-match
- --bib-auto-overlay-exact
- --bib-auto-overlay-1match
- --bib-auto-overlay-best-match
- --auth-import-no-match
- --auth-auto-overlay-exact
- --auth-auto-overlay-1match
- --auth-auto-overlay-best-match
-
- Bib and auth import options which map directly to Vandelay import
- options.
-
- For example:
- Apply import-no-match to bibs and auto-overlay-exact to auths.
-
- $0 --bib-import-no-match --auth-auto-overlay-exact
-
- --help
- Show this help message
-USAGE
- exit;
-}
-
-usage() if $help;
-
-if ($import_no_match) {
- warn "\nimport-no-match is deprecated; use bib-import-no-match\n";
- $bib_import_no_match = $import_no_match;
-}
-if ($auto_overlay_exact) {
- warn "\nauto-overlay-exact is deprecated; use bib-auto-overlay-exact\n";
- $bib_auto_overlay_exact = $auto_overlay_exact;
-}
-if ($auto_overlay_1match) {
- warn "\nauto-overlay-1match is deprecated; use bib-auto-overlay-1match\n";
- $bib_auto_overlay_1match = $auto_overlay_1match;
-}
-if ($auto_overlay_best_match) {
- warn "\nauto-overlay-best-match is deprecated; use bib-auto-overlay-best-match\n";
- $bib_auto_overlay_best_match = $auto_overlay_best_match;
-}
-if ($deprecated_queue) {
- warn "\n--queue is deprecated; use --bib-queue\n";
- $bib_queue = $deprecated_queue;
-}
-
-
-die "--username, --password, AND --workstation required. --help for more info.\n"
- unless $username and $password and $workstation;
-die "--bib-queue OR --auth-queue required. --help for more info.\n"
- unless $bib_queue or $auth_queue;
-
-sub set_tempdir {
- return if $tempdir; # already read or user provided
- $tempdir = OpenSRF::Utils::SettingsClient->new->config_value(
- qw/apps open-ils.vandelay app_settings databases importer/
- ) || '/tmp';
-}
-
-# Sets cur_rec_type to 'auth' if leader/06 of the first
-# parseable record is 'z', otherwise 'bib'.
-sub set_record_type {
- my $file_name = shift;
-
- my $marctype = 'USMARC';
- open(F, $file_name) or
- die "Unable to open MARC file $file_name : $!\n";
- $marctype = 'XML' if (getc(F) =~ /^\D/o);
- close F;
-
- my $batch = new MARC::Batch ($marctype, $file_name);
- $batch->strict_off;
-
- my $rec;
- my $ldr_06 = '';
- while (1) {
- eval {$rec = $batch->next};
- next if $@; # record parse failure
- last unless $rec;
- $ldr_06 = substr($rec->leader(), 6, 1) || '';
- last;
- }
-
- $cur_rec_type = $ldr_06 eq 'z' ? 'auth' : 'bib';
-
- $cur_queue = $cur_rec_type eq 'auth' ? $auth_queue : $bib_queue;
- $cur_rec_source = $cur_rec_type eq 'auth' ? '' : $bib_source;
- set_merge_profile();
-}
-
-# set vandelay options based on command line ops and the type of record
-# currently in process.
-sub compile_vandelay_ops {
-
- my $vl_ops = {
- report_all => 1,
- merge_profile => $cur_merge_profile ? $cur_merge_profile->id : undef
- };
-
- if ($cur_rec_type eq 'auth') {
- $vl_ops->{import_no_match} = $auth_import_no_match;
- $vl_ops->{auto_overlay_exact} = $auth_auto_overlay_exact;
- $vl_ops->{auto_overlay_1match} = $auth_auto_overlay_1match;
- $vl_ops->{auto_overlay_best_match} = $auth_auto_overlay_best_match;
- } else {
- $vl_ops->{import_no_match} = $bib_import_no_match;
- $vl_ops->{auto_overlay_exact} = $bib_auto_overlay_exact;
- $vl_ops->{auto_overlay_1match} = $bib_auto_overlay_1match;
- $vl_ops->{auto_overlay_best_match} = $bib_auto_overlay_best_match;
- }
-
- # Default to exact match only if not other strategy is selected.
- $vl_ops->{auto_overlay_exact} = 1
- if not (
- $vl_ops->{auto_overlay_1match} or
- $vl_ops->{auto_overlay_best_match}
- );
-
- $logger->info("VL options: ".Dumper($vl_ops)) if $verbose;
- return $vl_ops;
-}
-
-sub process_spool {
- my $file_name = shift; # filename
-
- set_record_type($file_name);
-
- my $ses = OpenSRF::AppSession->create('open-ils.vandelay');
- my $req = $ses->request(
- "open-ils.vandelay.$cur_rec_type.process_spool.stream_results",
- $authtoken, undef, # cache key not needed
- $cur_queue, 'import', $file_name, $cur_rec_source
- );
-
- my @rec_ids;
- while(my $resp = $req->recv) {
-
- if($req->failed) {
- $logger->error("Error spooling MARC data: $resp");
-
- } elsif($resp->content) {
- push(@rec_ids, $resp->content);
- }
- }
-
- return \@rec_ids;
-}
-
-sub import_queued_records {
- my $rec_ids = shift;
- my $vl_ops = compile_vandelay_ops();
-
- my $ses = OpenSRF::AppSession->create('open-ils.vandelay');
- my $req = $ses->request(
- "open-ils.vandelay.${cur_rec_type}_record.list.import",
- $authtoken, $rec_ids, $vl_ops
- );
-
- # collect the successfully imported vandelay records
- my $failed = 0;
- my @cleanup_recs;
- while(my $resp = $req->recv) {
- if($req->failed) {
- $logger->error("Error importing MARC data: $resp");
-
- } elsif(my $data = $resp->content) {
-
- if($data->{err_event}) {
-
- $logger->error(Dumper($data->{err_event}));
- $failed++;
-
- } elsif ($data->{no_import}) {
- # no errors, just didn't import, because of rules.
-
- $failed++;
- $logger->info(
- "record failed to satisfy Vandelay merge/quality/etc. ".
- "requirements: " . ($data->{imported} || ''));
-
- } else {
- push(@cleanup_recs, $data->{imported}) if $data->{imported};
- }
- }
- }
-
- # clean up the successfully imported vandelay records to prevent queue bloat
- my $pcrud = OpenSRF::AppSession->create('open-ils.pcrud');
- $pcrud->connect;
- $pcrud->request('open-ils.pcrud.transaction.begin', $authtoken)->recv;
- my $err;
-
- my $api = 'open-ils.pcrud.delete.';
- $api .= $cur_rec_type eq 'auth' ? 'vqar' : 'vqbr';
-
- foreach (@cleanup_recs) {
- eval {
- $pcrud->request($api, $authtoken, $_)->recv;
- };
-
- if ($@) {
- $logger->error("Error deleting queued $cur_rec_type record $_: $@");
- last;
- }
- }
-
- $pcrud->request('open-ils.pcrud.transaction.commit', $authtoken)->recv unless $err;
- $pcrud->disconnect;
-
- $logger->info("imported queued vandelay records: @cleanup_recs");
- return (scalar(@cleanup_recs), $failed);
-}
-
-
-
-# Each child needs its own opensrf connection.
-sub child_init_hook {
- OpenSRF::System->bootstrap_client(config_file => $osrf_config);
- Fieldmapper->import(IDL =>
- OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
-}
-
-
-# The core Net::Server method
-# Reads streams of MARC data from the network, saves the data as a file,
-# then processes the file via vandelay.
-sub process_request {
- my $self = shift;
- my $client = $self->{server}->{peeraddr}.':'.$self->{server}->{peerport};
-
- $logger->info("$client opened a new connection");
-
- my $ph = OpenSRF::Transport::PeerHandle->retrieve;
- if(!$ph->flush_socket()) {
- $logger->error("We received a request, but we are no longer connected".
- " to opensrf. Exiting and dropping request from $client");
- exit;
- }
-
- my $data = '';
- eval {
- local $SIG{ALRM} = sub { die "alarm\n" };
- alarm $wait_time; # prevent accidental tie ups of backend processes
- local $/ = "\x1D"; # MARC record separator
- $data = <STDIN>;
- alarm 0;
- };
-
- if($@) {
- $logger->error("reading from STDIN failed or timed out: $@");
- return;
- }
-
- $logger->info("stream parser read " . length($data) . " bytes");
-
- set_tempdir();
-
- # copy data to a temporary file so vandelay can scoop it up
- my ($handle, $tempfile) = tempfile("$0_XXXX", DIR => $tempdir)
- or die "Cannot create tempfile in $tempdir : $!";
-
- print $handle $data or die "Error writing to tempfile $tempfile : $!\n";
- close $handle;
-
- process_file($tempfile);
-}
-
-sub set_merge_profile {
-
- # serve from cache
-
- return $cur_merge_profile = $bib_merge_profile_obj
- if $bib_merge_profile_obj and $cur_rec_type eq 'bib';
-
- return $cur_merge_profile = $auth_merge_profile_obj
- if $auth_merge_profile_obj and $cur_rec_type eq 'auth';
-
- # fetch un-cached profile
-
- my $profile_id = $cur_rec_type eq 'bib' ?
- $bib_merge_profile : $auth_merge_profile;
-
- return $cur_merge_profile = undef unless $profile_id;
-
- $cur_merge_profile = $apputils->simplereq(
- 'open-ils.pcrud',
- 'open-ils.pcrud.retrieve.vmp',
- $authtoken, $profile_id);
-
- # cache profile for later
-
- $auth_merge_profile_obj = $cur_merge_profile if $cur_rec_type eq 'auth';
- $bib_merge_profile_obj = $cur_merge_profile if $cur_rec_type eq 'bib';
-}
-
-sub process_file {
- my $file = shift;
-
- new_auth_token(); # login
- my $rec_ids = process_spool($file);
- my ($imported, $failed) = import_queued_records($rec_ids);
-
- if (oils_event_equals($imported, 'NO_SESSION')) {
- # did the session expire while spooling?
- new_auth_token(); # retry with new authtoken
- ($imported, $failed) = import_queued_records($rec_ids);
- }
-
- oils_event_die($imported);
-
- my $profile = $cur_merge_profile ? $cur_merge_profile->name : '';
- my $msg = '';
- $msg .= "Successfully imported $imported $cur_rec_type records ".
- "using merge profile '$profile'\n" if $imported;
- $msg .= "Failed to import $failed $cur_rec_type records\n" if $failed;
- $msg .= "\x00" unless $spoolfile;
- print $msg;
-
- clear_auth_token(); # logout
-}
-
-# the authtoken will timeout after the configured inactivity period.
-# When that happens, get a new one.
-sub new_auth_token {
- oils_login($username, $password, 'staff', $workstation)
- or die "Unable to login to Evergreen as user $username";
-}
-
-sub clear_auth_token {
- $apputils->simplereq(
- 'open-ils.auth',
- 'open-ils.auth.session.delete',
- $authtoken
- );
- $authtoken = undef;
-}
-
-# -- execution starts here
-
-if ($spoolfile) {
- # individual files are processed in standalone mode.
- # No Net::Server innards are necessary.
-
- child_init_hook(); # force an opensrf connection
- process_file($spoolfile);
- exit;
-}
-
-# No spoolfile, run in Net::Server mode
-
-warn <<WARNING;
-
-WARNING: This script provides no security layer. Any client that has
-access to the server+port can inject MARC records into the system.
-
-WARNING
-
-my %args;
-$args{conf_file} = $net_server_conf if -r $net_server_conf;
-$args{port} = $port if $port;
-
-__PACKAGE__->run(%args);
-
-
--- /dev/null
+#!/usr/bin/perl
+# Copyright (C) 2008-2014 Equinox Software, Inc.
+# Copyright (C) 2014 King County Library System
+# Author: Bill Erickson <berickxx@gmail.com>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# ---------------------------------------------------------------
+# Sends MARC records, either from a file or from data delivered
+# via the network, to open-ils.vandelay to be imported.
+# ---------------------------------------------------------------
+use strict;
+use warnings;
+use Net::Server::PreFork;
+use base qw/Net::Server::PreFork/;
+use FindBin;
+require "$FindBin::Bin/oils_header.pl";
+
+use vars qw/$apputils $authtoken/;
+
+use Getopt::Long;
+use MARC::Record;
+use MARC::Batch;
+use MARC::File::XML (BinaryEncoding => 'UTF-8');
+use MARC::File::USMARC;
+use File::Basename qw/fileparse/;
+use File::Temp qw/tempfile/;
+use OpenSRF::AppSession;
+use OpenSRF::Utils::Logger qw/$logger/;
+use OpenSRF::Transport::PeerHandle;
+use OpenSRF::Utils::SettingsClient;
+
+use Data::Dumper;
+$Data::Dumper::Indent=0; # for logging
+
+# This script will always be an entry point for opensrf,
+# so go ahead and force log client.
+$ENV{OSRF_LOG_CLIENT} = 1;
+
+# these are updated with each new batch of records
+my $cur_rec_type;
+my $cur_rec_source;
+my $cur_queue;
+
+# cache these
+my $cur_merge_profile; # this is an object
+my $bib_merge_profile_obj;
+my $auth_merge_profile_obj;
+
+# options
+my $help = 0;
+my $osrf_config = '@sysconfdir@/opensrf_core.xml';
+my $username = '';
+my $password = '';
+my $workstation = '';
+my $tempdir = '';
+my $spoolfile = '';
+my $wait_time = 5;
+my $verbose = 0;
+my $bib_merge_profile;
+my $auth_merge_profile;
+my $bib_queue;
+my $auth_queue;
+my $bib_source;
+my $port;
+my $bib_import_no_match;
+my $bib_auto_overlay_exact;
+my $bib_auto_overlay_1match;
+my $bib_auto_overlay_best_match;
+my $auth_import_no_match;
+my $auth_auto_overlay_exact;
+my $auth_auto_overlay_1match;
+my $auth_auto_overlay_best_match;
+
+# deprecated options; these map to their bib_* equivalents
+my $import_no_match;
+my $auto_overlay_exact;
+my $auto_overlay_1match;
+my $auto_overlay_best_match;
+my $deprecated_queue;
+
+
+
+my $net_server_conf = (-r "@sysconfdir@/marc_stream_importer.conf") ? "@sysconfdir@/marc_stream_importer.conf" : undef;
+
+GetOptions(
+ 'osrf-config=s' => \$osrf_config,
+ 'verbose' => \$verbose,
+ 'username=s' => \$username,
+ 'password=s' => \$password,
+ 'workstation=s' => \$workstation,
+ 'tempdir=s' => \$tempdir,
+ 'spoolfile=s' => \$spoolfile,
+ 'wait=i' => \$wait_time,
+ 'merge-profile=i' => \$bib_merge_profile,
+ 'queue=i' => \$deprecated_queue,
+ 'bib-queue=i' => \$bib_queue,
+ 'source=i' => \$bib_source,
+ 'auth-merge-profile=i' => \$auth_merge_profile,
+ 'auth-queue=i' => \$auth_queue,
+
+ # -- deprecated
+ 'import-no-match' => \$import_no_match,
+ 'auto-overlay-exact' => \$auto_overlay_exact,
+ 'auto-overlay-1match' => \$auto_overlay_1match,
+ 'auto-overlay-best-match' => \$auto_overlay_best_match,
+ # --
+
+ 'bib-import-no-match' => \$bib_import_no_match,
+ 'bib-auto-overlay-exact' => \$bib_auto_overlay_exact,
+ 'bib-auto-overlay-1match' => \$bib_auto_overlay_1match,
+ 'bib-auto-overlay-best-match' => \$bib_auto_overlay_best_match,
+ 'auth-import-no-match' => \$auth_import_no_match,
+ 'auth-auto-overlay-exact' => \$auth_auto_overlay_exact,
+ 'auth-auto-overlay-1match' => \$auth_auto_overlay_1match,
+ 'auth-auto-overlay-best-match' => \$auth_auto_overlay_best_match,
+ 'help' => \$help,
+ 'net-server-config=s' => \$net_server_conf,
+ 'port=i' => \$port
+);
+
+sub usage {
+ print <<USAGE;
+ --osrf-config
+ Path to OpenSRF configuration file.
+
+ --net-server-conf
+ Path to Net::Server configuration file. Defaults to $net_server_conf.
+ Only required if --spoolfile is not set.
+
+ --verbose
+ Log additional details
+
+ --username
+ Evergreen user account which performs the import actions.
+
+ --password
+ Evergreen user account password
+
+ --workstation
+ Evergreen workstation
+
+ --tempdir
+ MARC data received via the network is stored in a temporary
+ file so Vandelay can access it. This must be a directory
+ the open-ils.vandelay service can access. If you want the
+ file deleted after completion, be sure open-ils.vandelay
+ has write access to the directory and the file.
+ This value defaults to the Vandelay data directory, however
+ this configuratoin value is only accessible when run from
+ the private opensrf domain, which you may not want to do.
+
+ --spoolfile
+ Path to a MARC file to load. When a --spoolfile is specified,
+ this script will send the file to vandelay for processing,
+ then exit when complete. In other words, it does not stay
+ alive to accept requests from the network.
+
+ --wait
+ Amount of time in seconds this script will wait after receiving
+ a connection on the socket and before recieving a complete
+ MARC record. This prevents unintentional denial of service by
+ clients connecting and never sending anything.
+
+ --merge-profile
+ ID of the vandelay bib record merge profile
+
+ --queue
+ ID of the vandelay bib record queue
+
+ --source
+ ID of the bib source
+
+ --auth-merge-profile
+ ID of the vandelay authority record merge profile
+
+ --auth-queue
+ ID of the vandelay authority record queue
+
+ --bib-import-no-match
+ --bib-auto-overlay-exact
+ --bib-auto-overlay-1match
+ --bib-auto-overlay-best-match
+ --auth-import-no-match
+ --auth-auto-overlay-exact
+ --auth-auto-overlay-1match
+ --auth-auto-overlay-best-match
+
+ Bib and auth import options which map directly to Vandelay import
+ options.
+
+ For example:
+ Apply import-no-match to bibs and auto-overlay-exact to auths.
+
+ $0 --bib-import-no-match --auth-auto-overlay-exact
+
+ --help
+ Show this help message
+USAGE
+ exit;
+}
+
+usage() if $help;
+
+if ($import_no_match) {
+ warn "\nimport-no-match is deprecated; use bib-import-no-match\n";
+ $bib_import_no_match = $import_no_match;
+}
+if ($auto_overlay_exact) {
+ warn "\nauto-overlay-exact is deprecated; use bib-auto-overlay-exact\n";
+ $bib_auto_overlay_exact = $auto_overlay_exact;
+}
+if ($auto_overlay_1match) {
+ warn "\nauto-overlay-1match is deprecated; use bib-auto-overlay-1match\n";
+ $bib_auto_overlay_1match = $auto_overlay_1match;
+}
+if ($auto_overlay_best_match) {
+ warn "\nauto-overlay-best-match is deprecated; use bib-auto-overlay-best-match\n";
+ $bib_auto_overlay_best_match = $auto_overlay_best_match;
+}
+if ($deprecated_queue) {
+ warn "\n--queue is deprecated; use --bib-queue\n";
+ $bib_queue = $deprecated_queue;
+}
+
+
+die "--username, --password, AND --workstation required. --help for more info.\n"
+ unless $username and $password and $workstation;
+die "--bib-queue OR --auth-queue required. --help for more info.\n"
+ unless $bib_queue or $auth_queue;
+
+sub set_tempdir {
+ return if $tempdir; # already read or user provided
+ $tempdir = OpenSRF::Utils::SettingsClient->new->config_value(
+ qw/apps open-ils.vandelay app_settings databases importer/
+ ) || '/tmp';
+}
+
+# Sets cur_rec_type to 'auth' if leader/06 of the first
+# parseable record is 'z', otherwise 'bib'.
+sub set_record_type {
+ my $file_name = shift;
+
+ my $marctype = 'USMARC';
+ open(F, $file_name) or
+ die "Unable to open MARC file $file_name : $!\n";
+ $marctype = 'XML' if (getc(F) =~ /^\D/o);
+ close F;
+
+ my $batch = new MARC::Batch ($marctype, $file_name);
+ $batch->strict_off;
+
+ my $rec;
+ my $ldr_06 = '';
+ while (1) {
+ eval {$rec = $batch->next};
+ next if $@; # record parse failure
+ last unless $rec;
+ $ldr_06 = substr($rec->leader(), 6, 1) || '';
+ last;
+ }
+
+ $cur_rec_type = $ldr_06 eq 'z' ? 'auth' : 'bib';
+
+ $cur_queue = $cur_rec_type eq 'auth' ? $auth_queue : $bib_queue;
+ $cur_rec_source = $cur_rec_type eq 'auth' ? '' : $bib_source;
+ set_merge_profile();
+}
+
+# set vandelay options based on command line ops and the type of record
+# currently in process.
+sub compile_vandelay_ops {
+
+ my $vl_ops = {
+ report_all => 1,
+ merge_profile => $cur_merge_profile ? $cur_merge_profile->id : undef
+ };
+
+ if ($cur_rec_type eq 'auth') {
+ $vl_ops->{import_no_match} = $auth_import_no_match;
+ $vl_ops->{auto_overlay_exact} = $auth_auto_overlay_exact;
+ $vl_ops->{auto_overlay_1match} = $auth_auto_overlay_1match;
+ $vl_ops->{auto_overlay_best_match} = $auth_auto_overlay_best_match;
+ } else {
+ $vl_ops->{import_no_match} = $bib_import_no_match;
+ $vl_ops->{auto_overlay_exact} = $bib_auto_overlay_exact;
+ $vl_ops->{auto_overlay_1match} = $bib_auto_overlay_1match;
+ $vl_ops->{auto_overlay_best_match} = $bib_auto_overlay_best_match;
+ }
+
+ # Default to exact match only if not other strategy is selected.
+ $vl_ops->{auto_overlay_exact} = 1
+ if not (
+ $vl_ops->{auto_overlay_1match} or
+ $vl_ops->{auto_overlay_best_match}
+ );
+
+ $logger->info("VL options: ".Dumper($vl_ops)) if $verbose;
+ return $vl_ops;
+}
+
+sub process_spool {
+ my $file_name = shift; # filename
+
+ set_record_type($file_name);
+
+ my $ses = OpenSRF::AppSession->create('open-ils.vandelay');
+ my $req = $ses->request(
+ "open-ils.vandelay.$cur_rec_type.process_spool.stream_results",
+ $authtoken, undef, # cache key not needed
+ $cur_queue, 'import', $file_name, $cur_rec_source
+ );
+
+ my @rec_ids;
+ while(my $resp = $req->recv) {
+
+ if($req->failed) {
+ $logger->error("Error spooling MARC data: $resp");
+
+ } elsif($resp->content) {
+ push(@rec_ids, $resp->content);
+ }
+ }
+
+ return \@rec_ids;
+}
+
+sub import_queued_records {
+ my $rec_ids = shift;
+ my $vl_ops = compile_vandelay_ops();
+
+ my $ses = OpenSRF::AppSession->create('open-ils.vandelay');
+ my $req = $ses->request(
+ "open-ils.vandelay.${cur_rec_type}_record.list.import",
+ $authtoken, $rec_ids, $vl_ops
+ );
+
+ # collect the successfully imported vandelay records
+ my $failed = 0;
+ my @cleanup_recs;
+ while(my $resp = $req->recv) {
+ if($req->failed) {
+ $logger->error("Error importing MARC data: $resp");
+
+ } elsif(my $data = $resp->content) {
+
+ if($data->{err_event}) {
+
+ $logger->error(Dumper($data->{err_event}));
+ $failed++;
+
+ } elsif ($data->{no_import}) {
+ # no errors, just didn't import, because of rules.
+
+ $failed++;
+ $logger->info(
+ "record failed to satisfy Vandelay merge/quality/etc. ".
+ "requirements: " . ($data->{imported} || ''));
+
+ } else {
+ push(@cleanup_recs, $data->{imported}) if $data->{imported};
+ }
+ }
+ }
+
+ # clean up the successfully imported vandelay records to prevent queue bloat
+ my $pcrud = OpenSRF::AppSession->create('open-ils.pcrud');
+ $pcrud->connect;
+ $pcrud->request('open-ils.pcrud.transaction.begin', $authtoken)->recv;
+ my $err;
+
+ my $api = 'open-ils.pcrud.delete.';
+ $api .= $cur_rec_type eq 'auth' ? 'vqar' : 'vqbr';
+
+ foreach (@cleanup_recs) {
+ eval {
+ $pcrud->request($api, $authtoken, $_)->recv;
+ };
+
+ if ($@) {
+ $logger->error("Error deleting queued $cur_rec_type record $_: $@");
+ last;
+ }
+ }
+
+ $pcrud->request('open-ils.pcrud.transaction.commit', $authtoken)->recv unless $err;
+ $pcrud->disconnect;
+
+ $logger->info("imported queued vandelay records: @cleanup_recs");
+ return (scalar(@cleanup_recs), $failed);
+}
+
+
+
+# Each child needs its own opensrf connection.
+sub child_init_hook {
+ OpenSRF::System->bootstrap_client(config_file => $osrf_config);
+ Fieldmapper->import(IDL =>
+ OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
+}
+
+
+# The core Net::Server method
+# Reads streams of MARC data from the network, saves the data as a file,
+# then processes the file via vandelay.
+sub process_request {
+ my $self = shift;
+ my $client = $self->{server}->{peeraddr}.':'.$self->{server}->{peerport};
+
+ $logger->info("$client opened a new connection");
+
+ my $ph = OpenSRF::Transport::PeerHandle->retrieve;
+ if(!$ph->flush_socket()) {
+ $logger->error("We received a request, but we are no longer connected".
+ " to opensrf. Exiting and dropping request from $client");
+ exit;
+ }
+
+ my $data = '';
+ eval {
+ local $SIG{ALRM} = sub { die "alarm\n" };
+ alarm $wait_time; # prevent accidental tie ups of backend processes
+ local $/ = "\x1D"; # MARC record separator
+ $data = <STDIN>;
+ alarm 0;
+ };
+
+ if($@) {
+ $logger->error("reading from STDIN failed or timed out: $@");
+ return;
+ }
+
+ $logger->info("stream parser read " . length($data) . " bytes");
+
+ set_tempdir();
+
+ # copy data to a temporary file so vandelay can scoop it up
+ my ($handle, $tempfile) = tempfile("$0_XXXX", DIR => $tempdir)
+ or die "Cannot create tempfile in $tempdir : $!";
+
+ print $handle $data or die "Error writing to tempfile $tempfile : $!\n";
+ close $handle;
+
+ process_file($tempfile);
+}
+
+sub set_merge_profile {
+
+ # serve from cache
+
+ return $cur_merge_profile = $bib_merge_profile_obj
+ if $bib_merge_profile_obj and $cur_rec_type eq 'bib';
+
+ return $cur_merge_profile = $auth_merge_profile_obj
+ if $auth_merge_profile_obj and $cur_rec_type eq 'auth';
+
+ # fetch un-cached profile
+
+ my $profile_id = $cur_rec_type eq 'bib' ?
+ $bib_merge_profile : $auth_merge_profile;
+
+ return $cur_merge_profile = undef unless $profile_id;
+
+ $cur_merge_profile = $apputils->simplereq(
+ 'open-ils.pcrud',
+ 'open-ils.pcrud.retrieve.vmp',
+ $authtoken, $profile_id);
+
+ # cache profile for later
+
+ $auth_merge_profile_obj = $cur_merge_profile if $cur_rec_type eq 'auth';
+ $bib_merge_profile_obj = $cur_merge_profile if $cur_rec_type eq 'bib';
+}
+
+sub process_file {
+ my $file = shift;
+
+ new_auth_token(); # login
+ my $rec_ids = process_spool($file);
+ my ($imported, $failed) = import_queued_records($rec_ids);
+
+ if (oils_event_equals($imported, 'NO_SESSION')) {
+ # did the session expire while spooling?
+ new_auth_token(); # retry with new authtoken
+ ($imported, $failed) = import_queued_records($rec_ids);
+ }
+
+ oils_event_die($imported);
+
+ my $profile = $cur_merge_profile ? $cur_merge_profile->name : '';
+ my $msg = '';
+ $msg .= "Successfully imported $imported $cur_rec_type records ".
+ "using merge profile '$profile'\n" if $imported;
+ $msg .= "Failed to import $failed $cur_rec_type records\n" if $failed;
+ $msg .= "\x00" unless $spoolfile;
+ print $msg;
+
+ clear_auth_token(); # logout
+}
+
+# the authtoken will timeout after the configured inactivity period.
+# When that happens, get a new one.
+sub new_auth_token {
+ oils_login($username, $password, 'staff', $workstation)
+ or die "Unable to login to Evergreen as user $username";
+}
+
+sub clear_auth_token {
+ $apputils->simplereq(
+ 'open-ils.auth',
+ 'open-ils.auth.session.delete',
+ $authtoken
+ );
+ $authtoken = undef;
+}
+
+# -- execution starts here
+
+if ($spoolfile) {
+ # individual files are processed in standalone mode.
+ # No Net::Server innards are necessary.
+
+ child_init_hook(); # force an opensrf connection
+ process_file($spoolfile);
+ exit;
+}
+
+# No spoolfile, run in Net::Server mode
+
+warn <<WARNING;
+
+WARNING: This script provides no security layer. Any client that has
+access to the server+port can inject MARC records into the system.
+
+WARNING
+
+my %args;
+$args{conf_file} = $net_server_conf if -r $net_server_conf;
+$args{port} = $port if $port;
+
+__PACKAGE__->run(%args);
+
+
--- /dev/null
+Install marc_stream_importer.pl By Default
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+The script for the MARC stream importer, `marc_stream_importer.pl`,
+is now installed in the Evergreen `bin` directory (typically
+`/openils/bin`) by default. It now also expects that its configuration
+file will be in the usual config directory (typically `/openils/conf`)
+and the example configuration file is installed their by default.
+
+Upgrade Note
+++++++++++++
+Because `marc_stream_importer.pl` now expects its configuration file to
+be in the configuration directory, not the binary directory, existing
+users will likely need to manually move the configuration file into
+place.
projects / evergreen / equinox.git / commitdiff