From: Bill Erickson <berickxx@gmail.com>
Date: Thu, 3 Oct 2019 15:05:29 +0000 (-0400)
Subject: more in-db field migration
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=5863450dd9c7653ab3f2cd921535f41162e82910;p=working%2FEvergreen.git

more in-db field migration

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
index c5a87df68b..6ad35962ca 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
@@ -255,12 +255,7 @@ sub search {
 # at 1/4 the limit to accomodate all UTF-8 chars.
 sub truncate_value {
     my ($self, $value) = @_;
-
-    if (length(Encode::encode('UTF-8', $value)) > 32760) {
-        $value = substr($value, 0, 8190);
-    }
-
-    return $value;
+    return substr($value, 0, 8190);
 }
 
 sub get_index_def {
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
index d45d62a318..e53cacb6ba 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
@@ -375,13 +375,6 @@ sub populate_bib_index_batch {
 
     my $holdings = $self->load_holdings($bib_ids);
     my $marc = $self->load_marc($bib_ids);
-    my $def = $self->get_index_def;
-
-    # Ask ES what the index properties are so we can avoid passing data
-    # that will not be indexed, since ES will store the data on the source
-    # object even if it's not indexed.  This reduces bulk.
-    my $properties =
-        $def->{$self->index_name}->{mappings}->{record}->{properties};
 
     for my $bib_id (@$bib_ids) {
 
@@ -411,6 +404,9 @@ sub populate_bib_index_batch {
             my $fclass = $field->{search_group};
             my $fname = $field->{name};
             my $value = $field->{value};
+
+            next unless defined $value && $value ne '';
+
             $fname = "$fclass|$fname" if $fclass;
             $value = $self->truncate_value($value);
 
@@ -419,7 +415,7 @@ sub populate_bib_index_batch {
             } elsif ($fname eq 'identifier|issn') {
                 index_issns($body, $value);
             } else {
-                append_field_value($body, $fname, $value, $properties);
+                append_field_value($body, $fname, $value);
             }
         }
 
@@ -446,15 +442,17 @@ sub index_isbns {
     # Chop up the collected raw values into parts and let
     # Business::* tell us which parts looks like ISBNs.
     for my $token (split(/ /, $value)) {
-        my $isbn = Business::ISBN->new($token);
-        if ($isbn && $isbn->is_valid) {
-            if ($isbn->as_isbn10) {
-                $seen{$isbn->as_isbn10->isbn} = 1;
-                $seen{$isbn->as_isbn10->as_string} = 1;
-            }
-            if ($isbn->as_isbn13) {
-                $seen{$isbn->as_isbn13->isbn} = 1;
-                $seen{$isbn->as_isbn13->as_string} = 1;
+        if (length($token) > 8) {
+            my $isbn = Business::ISBN->new($token);
+            if ($isbn && $isbn->is_valid) {
+                if ($isbn->as_isbn10) {
+                    $seen{$isbn->as_isbn10->isbn} = 1;
+                    $seen{$isbn->as_isbn10->as_string} = 1;
+                }
+                if ($isbn->as_isbn13) {
+                    $seen{$isbn->as_isbn13->isbn} = 1;
+                    $seen{$isbn->as_isbn13->as_string} = 1;
+                }
             }
         }
     }
@@ -485,11 +483,7 @@ sub index_issns {
 }
 
 sub append_field_value {
-    my ($body, $fname, $value, $properties) = @_;
-
-    # Confirm the data is wanted in the index before passing to ES to
-    # reduce the overall data footprint.
-    return unless $properties->{$fname};
+    my ($body, $fname, $value) = @_;
 
     if ($body->{$fname}) {
         if (ref $body->{$fname}) {
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
index a8890f1839..201b0003bb 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
@@ -223,3 +223,4 @@ WHERE name NOT IN (
     'vr_format'
 );
 
+*/
diff --git a/Open-ILS/src/support-scripts/test-scripts/elastic-search-samples.pl b/Open-ILS/src/support-scripts/test-scripts/elastic-search-samples.pl
index dfe38b2008..c45e7c4d07 100755
--- a/Open-ILS/src/support-scripts/test-scripts/elastic-search-samples.pl
+++ b/Open-ILS/src/support-scripts/test-scripts/elastic-search-samples.pl
@@ -5,7 +5,7 @@ use Getopt::Long;
 use Time::HiRes qw/time/;
 use OpenSRF::Utils::JSON;
 use OpenILS::Utils::Fieldmapper;
-use OpenILS::Elastic::Bib::Search;
+use OpenILS::Elastic::BibSearch;
 
 use utf8;
 binmode(STDIN, ':utf8');
@@ -117,7 +117,7 @@ Fieldmapper->import(
     IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
 OpenILS::Utils::CStoreEditor::init();
 
-my $es = OpenILS::Elastic::Bib::Search->new($cluster);
+my $es = OpenILS::Elastic::BibSearch->new($cluster);
 $es->connect;
 
 print "Searching...\n";
diff --git a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl
index ef5999e657..297b92fa34 100755
--- a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl
+++ b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl
@@ -5,7 +5,7 @@ use Getopt::Long;
 use Time::HiRes qw/time/;
 use OpenSRF::Utils::JSON;
 use OpenILS::Utils::Fieldmapper;
-use OpenILS::Elastic::Bib::Search;
+use OpenILS::Elastic::BibSearch;
 
 use utf8;
 binmode(STDIN, ':utf8');
@@ -46,7 +46,7 @@ Fieldmapper->import(
     IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
 OpenILS::Utils::CStoreEditor::init();
 
-my $es = OpenILS::Elastic::Bib::Search->new($cluster);
+my $es = OpenILS::Elastic::BibSearch->new($cluster);
 $es->connect;
 
 print <<MESSAGE;