From: Bill Erickson <berickxx@gmail.com>
Date: Mon, 26 Sep 2016 15:18:06 +0000 (-0400)
Subject: JBAS-1437 Backstage quarterly export docs
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=0e472e06b7c096b232b393c509a4e02ec45c5ae6;p=working%2FEvergreen.git

JBAS-1437 Backstage quarterly export docs

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---

diff --git a/KCLS/backstage/README.adoc b/KCLS/backstage/README.adoc
index 360f086dd3..0f834eaada 100644
--- a/KCLS/backstage/README.adoc
+++ b/KCLS/backstage/README.adoc
@@ -1,23 +1,127 @@
-= Backstage Process =
+= Backstage Processes =
 
-== Setup ==
+Perform steps as 'opensrf'
+
+== Quarterly Export + Import ==
+
+=== Setup ===
 
 [source,sh]
 --------------------------------------------------------------------
-mkdir -p /openils/var/data/backstage
+export EXPORT_DATE=2016-10-01 # for example
+export WORKING_DIR=/openils/var/data/authority-control/backstage/quarterly/$EXPORT_DATE
+export PGHOST=foo
+export PGPASSWORD=foo
+export PGUSER=evergreen
+mkdir -p $WORKING_DIR
 --------------------------------------------------------------------
 
-== Exporting Bib Records ==
+=== Exporting Bib Records ===
 
 Bibs are exported as MARC and uploaded to Backstage
 
-=== Generate Export MARC File ===
+==== Generate Export MARC File ====
 
 [source,sh]
 --------------------------------------------------------------------
 ./export-bibs.pl \
-    --cat-date-start 2010-01-01 \
-    --cat-date-end 2016-06-01 \
-    > /openils/var/data/backstage/bib-export-2016-06-01
+    --start-date 2010-01-01 \
+    --end-date 2016-06-01 \
+    --export-date $EXPORT_DATE \
+    --out-file $WORKING_DIR/exported-bibs.$EXPORT_DATE.mrc
+
+# Send file to BS FTP server in(bound) directory.
+--------------------------------------------------------------------
+
+=== Process Results ===
+
+==== Fetch Results ====
+
+[source,sh]
+--------------------------------------------------------------------
+cd $WORKING_DIR
+wget <backstage-results-file>
+--------------------------------------------------------------------
+
+==== Process Results Files ====
+
+* Import new and modified authority records.
+* Import modified bib records.
+
+[source,sh]
+--------------------------------------------------------------------
+cd /home/opensrf/Evergreen/KCLS/backstage/
+
+./process-backstage-files.pl \
+    --verbose \
+    --export-date $EXPORT_DATE \
+    --file $WORKING_DIR/<results-file> \
+    --working-dir $WORKING_DIR \
+    --bib-collision-file bib-collisions.mrc \
+    > $WORKING_DIR/process.log
+--------------------------------------------------------------------
+
+==== Process Bib Collisions ====
+
+Bib records that were locally modified during Backstage processing are
+re-imported without clobbering the modifications.
+
+1. Create a new queue for this batch (rename to suit).
+
+[source,sh]
+--------------------------------------------------------------------
+INSERT INTO vandelay.bib_queue (owner, name) 
+    VALUES (1, 'Backstage Q3 2016');
+SELECT id FROM vandelay.bib_queue WHERE name = 'Backstage Q3 2016';
+--------------------------------------------------------------------
+
+2. Import bib collisions via stream importer.
+
+[source,sh]
+--------------------------------------------------------------------
+# Make a copy of the collisions file for safe keeping, 
+# since open-ils.vandelay deletes spool files.
+cp $WORKING_DIR/bib-collisions.mrc $WORKING_DIR/bib-collisions.bak.mrc
+
+cd /openils/bin
+./marc_stream_importer.pl \
+    --spoolfile $WORKING_DIR/bib-collisions.mrc \
+    --user admin \
+    --password XXX \
+    --bib-auto-overlay-exact \
+    --queue <new-queue-id> \
+    --merge-profile 104 # "Backstage Field Protection"
+--------------------------------------------------------------------
+
+==== Get auth/bib IDs that require re-linking ====
+
+[source,sh]
+--------------------------------------------------------------------
+cd /home/opensrf/Evergreen/KCLS/linking/
+
+./link-new-auth-records.pl --modified-since <days-since-reimport> \
+    --progress --print-auth-ids \
+    > $WORKING_DIR/auths-to-link.txt
+
+./link-new-auth-records.pl --modified-since <days-since-reimport> \
+    --progress --print-bib-ids \
+    > $WORKING_DIR/bibs-to-link.txt
+--------------------------------------------------------------------
+
+==== Re-Link Modified Auths and Bibs ====
+
+[source,sh]
+--------------------------------------------------------------------
+cd /home/opensrf/Evergreen/KCLS/authority-control/linking/
+
+./authority_authority_linker.pl --verbose \
+    --file $WORKING_DIR/auths-to-link.txt \
+    | tee -a $WORKING_DIR/auth2auth-linking.log
+
+# Bib linking takes many hours, sometimes days.
+
+./authority_control_fields.pl --verbose --refresh \
+    --file $WORKING_DIR/bibs-to-link.txt \
+    | tee -a $WORKING_DIR/bib-linking.log
 --------------------------------------------------------------------