From aabc213518beb6316fe117417647b968bbb7bb5a Mon Sep 17 00:00:00 2001
From: Bill Erickson <berickxx@gmail.com>
Date: Tue, 12 Apr 2016 10:18:36 -0400
Subject: [PATCH] Min/Max ID clarification; --sort-id-desc

Replace start-id/end-id arguments with min-id/max-id to avoid any
confusion when using descending sort options.

Add support for --sort-id-desc to process records in reverse order of
record ID.  This works better across multiple batches when you don't
know what both the min/max ID values are goign to be ahead of time.

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---
 pingest.pl | 52 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/pingest.pl b/pingest.pl
index d1bb30f7d..47f630a3b 100755
--- a/pingest.pl
+++ b/pingest.pl
@@ -33,10 +33,11 @@ my $skip_browse;  # Skip the browse reingest.
 my $skip_attrs;   # Skip the record attributes reingest.
 my $skip_search;  # Skip the search reingest.
 my $skip_facets;  # Skip the facets reingest.
-my $start_id;     # start processing at this bib ID.
-my $end_id;       # stop processing when this bib ID is reached.
+my $min_id;       # start processing at this bib ID.
+my $max_id;       # stop processing when this bib ID is reached.
 my $max_duration; # max processing duration in seconds
 my $newest_first; # Process records in descending order of edit_date
+my $sort_id_desc; # Sort by record ID descending
 my $help;         # show help text
 
 GetOptions(
@@ -46,9 +47,10 @@ GetOptions(
     'skip-attrs'     => \$skip_attrs,
     'skip-search'    => \$skip_search,
     'skip-facets'    => \$skip_facets,
-    'start-id=i'     => \$start_id,
-    'end-id=i'       => \$end_id,
+    'min-id=i'       => \$min_id,
+    'max-id=i'       => \$max_id,
     'newest-first'   => \$newest_first,
+    'sort-id-desc'   => \$sort_id_desc,
     'max-duration=i' => \$max_duration,
     'help'           => \$help
 );
@@ -57,7 +59,7 @@ sub help {
     print <<HELP;
 
     $0 --batch-size $batch_size --max-child $max_child \
-        --start-id 1 --end-id 500000 --duration 14400
+        --min-id 1 --max-id 500000 --duration 14400
 
     --batch-size
         Number of records to process per batch
@@ -71,20 +73,33 @@ sub help {
     --skip-facets
         Skip the selected reingest component
 
-    --start-id
+    --min-id
         Start processing at this record ID.
 
-    --end-id
+    --max-id
         Stop processing when this record ID is reached
 
+    --sort-id-desc
+        Process records in descending order of record ID.  This is a
+        rough approximation of sorting by most recently created record
+        first, but is better suited for multiple runs of the script,
+        where batches of records are processed in ID groups.
+
     --newest-first
         During the initial bib record query, sort records by edit
         date in descending order, so that records more recently
         modified/created are processed before older records.
+        This takes precedence over --sort-id-desc.
 
     --max-duration
         Stop processing after this many total seconds have passed.
 
+    --min-id
+        Only process records whose ID is equal to or greater than min-id.
+
+    --max-id
+        Only process records whose ID is equal to or less than max-id.
+
     --help
         Show this help text.
 
@@ -95,17 +110,22 @@ HELP
 help() if $help;
 
 my $where = "WHERE deleted = 'f'";
-if ($start_id && $end_id) {
-    $where .= " AND id BETWEEN $start_id AND $end_id";
-} elsif ($start_id) {
-    $where .= " AND id >= $start_id";
-} elsif ($end_id) {
-    $where .= " AND id <= $end_id";
+if ($min_id && $max_id) {
+    $where .= " AND id BETWEEN $min_id AND $max_id";
+} elsif ($min_id) {
+    $where .= " AND id >= $min_id";
+} elsif ($max_id) {
+    $where .= " AND id <= $max_id";
 }
 
-my $order_by = $newest_first ?
-    'ORDER BY edit_date DESC, id DESC' :
-    'ORDER BY id ASC';
+my $order_by = 'ORDER BY ';
+if ($newest_first) {
+    $order_by .= 'edit_date DESC, id DESC';
+} elsif ($sort_id_desc) {
+    $order_by .= 'id DESC';
+} else {
+    $order_by .= 'id ASC';
+}
 
 # "Gimme the keys!  I'll drive!"
 my $q = <<END_OF_Q;
-- 
2.11.0