JBAS-1581 Backstage cron updates and docs

Signed-off-by: Bill Erickson <berickxx@gmail.com>

diff --git a/KCLS/authority-control/backstage/README.adoc b/KCLS/authority-control/backstage/README.adoc
index 247c59d..2c2152e 100644 (file)
--- a/KCLS/authority-control/backstage/README.adoc
+++ b/KCLS/authority-control/backstage/README.adoc
@@ -1,129 +1,31 @@
-= Backstage Processes =
-
-Perform steps as 'opensrf'
-
-TODO: update me when backstage-agent.sh is done.
-
-== Quarterly Export + Import ==
-
-=== Setup ===
-
-[source,sh]
---------------------------------------------------------------------
-export EXPORT_DATE=2016-10-01
-export WORKING_DIR=/openils/var/data/authority-control/backstage/quarterly/$EXPORT_DATE
-export PGHOST=foo
-export PGPASSWORD=foo
-mkdir -p $WORKING_DIR
---------------------------------------------------------------------
-
-=== Exporting Bib Records ===
-
-Bibs are exported as MARC and uploaded to Backstage
-
-==== Generate Export MARC File ====
-
-[source,sh]
---------------------------------------------------------------------
-./export-bibs.pl \
-    --start-date 2010-01-01 \
-    --end-date 2016-06-01 \
-    --out-file $WORKING_DIR/exported-bibs.$EXPORT_DATE.mrc
-
-# Upload to Backstage
-curl --user USER:PASS -T \
-    $WORKING_DIR/exported-bibs.$EXPORT_DATE.mrc ftp://ftp.bslw.com/in/
---------------------------------------------------------------------
-
-=== Process Results ===
-
-==== Fetch Results ====
-
-[source,sh]
---------------------------------------------------------------------
-cd $WORKING_DIR
-curl --user USER:PASS -O ftp://ftp.bslw.com/out/<result-file>
---------------------------------------------------------------------
-
-==== Process Results Files ====
-
-* Import new and modified authority records.
-* Import modified bib records.
+= Backstage Processes Synopsis =
 
 [source,sh]
 --------------------------------------------------------------------
-cd /home/opensrf/Evergreen/KCLS/backstage/
+export BACKSTAGE_USER=user
+export BACKSTAGE_PASSWORD=pass
+export WORKING_DIR="/openils/var/data/authority-control/backstage/quarterly/$(date +'%Y-%m')"
 
-./process-backstage-files.pl \
-    --verbose \
-    --export-date $EXPORT_DATE \
-    --zip-file $WORKING_DIR/<result-file> \
-    --working-dir $WORKING_DIR \
-    --reports-dir $REPORTS_DIR \ # TODO
-    > $WORKING_DIR/process.log
---------------------------------------------------------------------
-
-==== Process Bib Collisions ====
-
-Bib records that were locally modified during Backstage processing are
-re-imported without clobbering the modifications.
+cd /home/opensrf/Evergreen/KCLS/authority-control/backstage/
 
-1. Create a new queue for this batch (rename to suit).
-
-[source,sh]
---------------------------------------------------------------------
-INSERT INTO vandelay.bib_queue (owner, name) 
-    VALUES (1, 'Backstage Q3 2016');
-SELECT id FROM vandelay.bib_queue WHERE name = 'Backstage Q3 2016';
---------------------------------------------------------------------
+# Export bibs, use db05 for heavy query.
+./backstage-agent.sh -d db05 -b
 
-2. Import bib collisions via stream importer.
+# Import bibs
+./backstage-agent.sh -q
 
-[source,sh]
---------------------------------------------------------------------
-# Make a copy of the collisions file for safe keeping, 
-# since open-ils.vandelay deletes spool files.
+# Import bib collisions
 cp $WORKING_DIR/bib-collisions.mrc $WORKING_DIR/bib-collisions.bak.mrc
-
 cd /openils/bin
 ./marc_stream_importer.pl \
     --spoolfile $WORKING_DIR/bib-collisions.mrc \
     --user admin \
     --password XXX \
     --bib-auto-overlay-exact \
-    --queue <new-queue-id> \
+    --queue 5413 \
     --merge-profile 104 # "Backstage Field Protection"
---------------------------------------------------------------------
-
-==== Get auth/bib IDs that require re-linking ====
-
-[source,sh]
---------------------------------------------------------------------
-cd /home/opensrf/Evergreen/KCLS/linking/
-
-./link-new-auth-records.pl --modified-since <days-since-reimport> \
-    --progress --print-auth-ids \
-    > $WORKING_DIR/auths-to-link.txt
-
-./link-new-auth-records.pl --modified-since <days-since-reimport> \
-    --progress --print-bib-ids \
-    > $WORKING_DIR/bibs-to-link.txt
---------------------------------------------------------------------
-
-==== Re-Link Modified Auths and Bibs ====
-
-[source,sh]
---------------------------------------------------------------------
-cd /home/opensrf/Evergreen/KCLS/authority-control/linking/
-
-./authority_authority_linker.pl --verbose \
-    --file $WORKING_DIR/auths-to-link.txt \
-    | tee -a $WORKING_DIR/auth2auth-linking.log
-
-# Bib linking takes many hours, sometimes days.
 
-./authority_control_fields.pl --verbose --refresh \
-    --file $WORKING_DIR/bibs-to-link.txt \
-    | tee -a $WORKING_DIR/bib-linking.log
+# Monthly authority imports
+./backstage-agent.sh -a
 --------------------------------------------------------------------
 

diff --git a/KCLS/authority-control/backstage/backstage-agent.sh b/KCLS/authority-control/backstage/backstage-agent.sh
index 5f18d49..91037f3 100755 (executable)
--- a/KCLS/authority-control/backstage/backstage-agent.sh
+++ b/KCLS/authority-control/backstage/backstage-agent.sh
@@ -145,6 +145,8 @@ function bs_import_qtrly_results {
 function bs_export_qtrly_bibs {
     bs_set_qtr_dates;
     bs_make_dirs "quarterly"
+    # check and exit early if needed.
+    bs_check_creds;
 
     EXPORT_FILE="$WORKING_DIR/bib-export-qtrly.$CUR_YEAR4-$CUR_MONTH.mrc"
 

diff --git a/KCLS/utility-scripts/CRONTAB b/KCLS/utility-scripts/CRONTAB
index 216f2f1..8203af8 100644 (file)
--- a/KCLS/utility-scripts/CRONTAB
+++ b/KCLS/utility-scripts/CRONTAB
@@ -14,6 +14,7 @@ PGUSER     = evergreen
 PGDATABASE = evergreen
 # change for cluster install
 PGHOST     = localhost 
+FTP_PASSIVE = 1
 BACKSTAGE_USER = BSUSER
 BACKSTAGE_PASSWORD = BSPASS
 # Uncomment on production to generate email alerts for certain actions
@@ -124,6 +125,18 @@ BACKSTAGE_PASSWORD = BSPASS
 # Update auth-2-auth links for new or modified authority records
 # 0 6 * * * . ~/.bashrc && cd $SCRIPT_DIR/../authority-control/linking && ./authority_authority_linker.pl --auth-mod-since $(date -d -1day +'\%F')
 
+# Backstage quarterly bib export                                               
+# Exports occur the second Friday of each quarter at noon.
+# 0 12 8-14 1  * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b
+# 0 12 8-14 4  * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b
+# 0 12 8-14 7  * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b
+# 0 12 8-14 10 * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -d db05 -b
+#
+# Backstage authority record monthly updates                                   
+# Imports are available the 3rd Thursday of every month,                       
+# but are imported early on the 3rd Friday of every month.                     
+# 0 1 15-21 * * [ `date +\%u` = 5 ] && cd $SCRIPT_DIR/../authority-control/backstage/ && ./backstage-agent.sh -a
+
 # Anonymize a range of historical circs each night, with a maximum run
 # time of 120 minutes.  We split them up, because each range takes time
 # to run (~45 minutes). The first range starts at 2004-08-12, which is the