From f6a4af42895365be3d8abf89c76bdc09b1288c03 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Thu, 11 May 2017 16:37:52 -0400 Subject: [PATCH] JBAS-1581 Backstage cron updates and docs Signed-off-by: Bill Erickson --- KCLS/authority-control/backstage/README.adoc | 124 +++------------------ .../authority-control/backstage/backstage-agent.sh | 2 + KCLS/utility-scripts/CRONTAB | 13 +++ 3 files changed, 28 insertions(+), 111 deletions(-) diff --git a/KCLS/authority-control/backstage/README.adoc b/KCLS/authority-control/backstage/README.adoc index 247c59d68b..2c2152e616 100644 --- a/KCLS/authority-control/backstage/README.adoc +++ b/KCLS/authority-control/backstage/README.adoc @@ -1,129 +1,31 @@ -= Backstage Processes = - -Perform steps as 'opensrf' - -TODO: update me when backstage-agent.sh is done. - -== Quarterly Export + Import == - -=== Setup === - -[source,sh] --------------------------------------------------------------------- -export EXPORT_DATE=2016-10-01 -export WORKING_DIR=/openils/var/data/authority-control/backstage/quarterly/$EXPORT_DATE -export PGHOST=foo -export PGPASSWORD=foo -mkdir -p $WORKING_DIR --------------------------------------------------------------------- - -=== Exporting Bib Records === - -Bibs are exported as MARC and uploaded to Backstage - -==== Generate Export MARC File ==== - -[source,sh] --------------------------------------------------------------------- -./export-bibs.pl \ - --start-date 2010-01-01 \ - --end-date 2016-06-01 \ - --out-file $WORKING_DIR/exported-bibs.$EXPORT_DATE.mrc - -# Upload to Backstage -curl --user USER:PASS -T \ - $WORKING_DIR/exported-bibs.$EXPORT_DATE.mrc ftp://ftp.bslw.com/in/ --------------------------------------------------------------------- - -=== Process Results === - -==== Fetch Results ==== - -[source,sh] --------------------------------------------------------------------- -cd $WORKING_DIR -curl --user USER:PASS -O ftp://ftp.bslw.com/out/ --------------------------------------------------------------------- - -==== Process Results Files ==== - -* Import new and modified authority records. -* Import modified bib records. += Backstage Processes Synopsis = [source,sh] -------------------------------------------------------------------- -cd /home/opensrf/Evergreen/KCLS/backstage/ +export BACKSTAGE_USER=user +export BACKSTAGE_PASSWORD=pass +export WORKING_DIR="/openils/var/data/authority-control/backstage/quarterly/$(date +'%Y-%m')" -./process-backstage-files.pl \ - --verbose \ - --export-date $EXPORT_DATE \ - --zip-file $WORKING_DIR/ \ - --working-dir $WORKING_DIR \ - --reports-dir $REPORTS_DIR \ # TODO - > $WORKING_DIR/process.log --------------------------------------------------------------------- - -==== Process Bib Collisions ==== - -Bib records that were locally modified during Backstage processing are -re-imported without clobbering the modifications. +cd /home/opensrf/Evergreen/KCLS/authority-control/backstage/ -1. Create a new queue for this batch (rename to suit). - -[source,sh] --------------------------------------------------------------------- -INSERT INTO vandelay.bib_queue (owner, name) - VALUES (1, 'Backstage Q3 2016'); -SELECT id FROM vandelay.bib_queue WHERE name = 'Backstage Q3 2016'; --------------------------------------------------------------------- +# Export bibs, use db05 for heavy query. +./backstage-agent.sh -d db05 -b -2. Import bib collisions via stream importer. +# Import bibs +./backstage-agent.sh -q -[source,sh] --------------------------------------------------------------------- -# Make a copy of the collisions file for safe keeping, -# since open-ils.vandelay deletes spool files. +# Import bib collisions cp $WORKING_DIR/bib-collisions.mrc $WORKING_DIR/bib-collisions.bak.mrc - cd /openils/bin ./marc_stream_importer.pl \ --spoolfile $WORKING_DIR/bib-collisions.mrc \ --user admin \ --password XXX \ --bib-auto-overlay-exact \ - --queue \ + --queue 5413 \ --merge-profile 104 # "Backstage Field Protection" --------------------------------------------------------------------- - -==== Get auth/bib IDs that require re-linking ==== - -[source,sh] --------------------------------------------------------------------- -cd /home/opensrf/Evergreen/KCLS/linking/ - -./link-new-auth-records.pl --modified-since \ - --progress --print-auth-ids \ - > $WORKING_DIR/auths-to-link.txt - -./link-new-auth-records.pl --modified-since \ - --progress --print-bib-ids \ - > $WORKING_DIR/bibs-to-link.txt --------------------------------------------------------------------- - -==== Re-Link Modified Auths and Bibs ==== - -[source,sh] --------------------------------------------------------------------- -cd /home/opensrf/Evergreen/KCLS/authority-control/linking/ - -./authority_authority_linker.pl --verbose \ - --file $WORKING_DIR/auths-to-link.txt \ - | tee -a $WORKING_DIR/auth2auth-linking.log - -# Bib linking takes many hours, sometimes days. -./authority_control_fields.pl --verbose --refresh \ - --file $WORKING_DIR/bibs-to-link.txt \ - | tee -a $WORKING_DIR/bib-linking.log +# Monthly authority imports +./backstage-agent.sh -a -------------------------------------------------------------------- diff --git a/KCLS/authority-control/backstage/backstage-agent.sh b/KCLS/authority-control/backstage/backstage-agent.sh index 5f18d492eb..91037f336e 100755 --- a/KCLS/authority-control/backstage/backstage-agent.sh +++ b/KCLS/authority-control/backstage/backstage-agent.sh @@ -145,6 +145,8 @@ function bs_import_qtrly_results { function bs_export_qtrly_bibs { bs_set_qtr_dates; bs_make_dirs "quarterly" + # check and exit early if needed. + bs_check_creds; EXPORT_FILE="$WORKING_DIR/bib-export-qtrly.$CUR_YEAR4-$CUR_MONTH.mrc" diff --git a/KCLS/utility-scripts/CRONTAB b/KCLS/utility-scripts/CRONTAB index 216f2f1418..8203af8fbb 100644 --- a/KCLS/utility-scripts/CRONTAB +++ b/KCLS/utility-scripts/CRONTAB @@ -14,6 +14,7 @@ PGUSER = evergreen PGDATABASE = evergreen # change for cluster install PGHOST = localhost +FTP_PASSIVE = 1 BACKSTAGE_USER = BSUSER BACKSTAGE_PASSWORD = BSPASS # Uncomment on production to generate email alerts for certain actions @@ -124,6 +125,18 @@ BACKSTAGE_PASSWORD = BSPASS # Update auth-2-auth links for new or modified authority records # 0 6 * * * . ~/.bashrc && cd $SCRIPT_DIR/../authority-control/linking && ./authority_authority_linker.pl --auth-mod-since $(date -d -1day +'\%F') +# Backstage quarterly bib export +# Exports occur the second Friday of each quarter at noon. +# 0 12 8-14 1 * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b +# 0 12 8-14 4 * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b +# 0 12 8-14 7 * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b +# 0 12 8-14 10 * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b +# +# Backstage authority record monthly updates +# Imports are available the 3rd Thursday of every month, +# but are imported early on the 3rd Friday of every month. +# 0 1 15-21 * * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -a + # Anonymize a range of historical circs each night, with a maximum run # time of 120 minutes. We split them up, because each range takes time # to run (~45 minutes). The first range starts at 2004-08-12, which is the -- 2.11.0