LP1844418 Direct indexing experiment WIP; field class

Signed-off-by: Bill Erickson <berickxx@gmail.com>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
index e96949c..5bcb5ae 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
@@ -258,19 +258,19 @@ sub create_index_properties {
         
 
         my $field_name = $field->name;
-        my $search_group = $field->search_group;
-        $field_name = "$search_group|$field_name" if $search_group;
+        my $field_class = $field->field_class;
+        $field_name = "$field_class|$field_name" if $field_class;
 
         $logger->info("ES ONE FIELD name=$field_name: " . OpenSRF::Utils::JSON->perl2JSON($field));
 
         my $def;
 
-        if ($search_group) {
+        if ($field_class) {
             if ($field->search_field eq 't') {
 
                 # Use the same fields and analysis as the 'grouped' field.
-                $def = clone($properties->{$search_group});
-                $def->{copy_to} = [$search_group, $SHORT_GROUP_MAP{$search_group}];
+                $def = clone($properties->{$field_class});
+                $def->{copy_to} = [$field_class, $SHORT_GROUP_MAP{$field_class}];
 
                 # Apply ranking boost to each analysis variation.
                 my $flds = $def->{fields};
@@ -398,7 +398,7 @@ sub get_bib_data {
     my $ids_str = join(',', @$record_ids);
 
     my $sql = <<SQL;
-SELECT DISTINCT ON (bre.id, search_group, name, value)
+SELECT DISTINCT ON (bre.id, field_class, name, value)
     bre.id, 
     bre.create_date, 
     bre.edit_date, 
@@ -472,7 +472,7 @@ sub populate_bib_index_batch {
                 ($body->{edit_date} = $field->{edit_date}) =~ s/ /T/g;
             }
 
-            my $fclass = $field->{search_group};
+            my $fclass = $field->{field_class};
             my $fname = $field->{name};
             my $value = $field->{value};
 

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm
index 88a1032..6178936 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm
@@ -24,9 +24,9 @@ sub name {
     my $self = shift;
     return $self->{name};
 }
-sub search_group {
+sub field_class {
     my $self = shift;
-    return $self->{search_group};
+    return $self->{field_class};
 }
 sub search_field {
     my $self = shift;
@@ -57,7 +57,6 @@ use XML::LibXSLT;
 use OpenSRF::Utils::Logger qw/:logger/;
 use OpenILS::Utils::CStoreEditor qw/:funcs/;
 use OpenILS::Elastic::BibSearch;
-use OpenILS::Utils::Normalize;
 use base qw/OpenILS::Elastic::BibSearch/;
 
 
@@ -88,26 +87,24 @@ sub xsl_sheet {
 
 my @seen_fields;
 sub add_dynamic_field {
-    my ($self, $fields, $purpose, $search_group, $name, $weight, $normalizers) = @_;
+    my ($self, $fields, $purpose, $field_class, $name, $weight) = @_;
     return unless $name;
 
     $weight = '' if !$weight || $weight eq '_';
-    $search_group = '' if !$search_group || $search_group eq '_';
-    $normalizers = '' if !$normalizers || $normalizers eq '_';
+    $field_class = '' if !$field_class || $field_class eq '_';
 
-    my $tag = $purpose . $search_group . $name;
+    my $tag = $purpose . $field_class . $name;
     return if grep {$_ eq $tag} @seen_fields;
     push(@seen_fields, $tag);
 
     $logger->info("ES adding dynamic field purpose=$purpose ".
-        "search_group=$search_group name=$name weight=$weight");
+        "field_class=$field_class name=$name weight=$weight");
 
     my $field = OpenILS::Elastic::BibSearch::BibField->new(
         purpose => $purpose, 
-        search_group => $search_group, 
+        field_class => $field_class, 
         name => $name,
-        weight => $weight,
-        normalizers => $normalizers
+        weight => $weight
     );
 
     push(@$fields, $field);
@@ -152,15 +149,15 @@ sub get_bib_data {
 
         my @rows = split(/\n/, $output);
         for my $row (@rows) {
-            my ($purpose, $search_group, $name, @tokens) = split(/ /, $row);
+            my ($purpose, $field_class, $name, @tokens) = split(/ /, $row);
 
-            $search_group = '' if ($search_group || '') eq '_';
+            $field_class = '' if ($field_class || '') eq '_';
 
             my $value = join(' ', @tokens);
 
             my $field = {
                 purpose => $purpose,
-                search_group => $search_group,
+                field_class => $field_class,
                 name => $name,
                 value => $value
             };

diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
index fc5430f..193002a 100644 (file)
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
@@ -67,7 +67,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS
             NULL::INT AS metabib_field,
             crad.name,
             crad.label,
-            NULL AS search_group,
+            NULL AS field_class,
             crad.sorter,
             FALSE AS search_field,
             FALSE AS facet_field,
@@ -79,7 +79,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS
             cmf.id AS metabib_field,
             cmf.name,
             cmf.label,
-            cmf.field_class AS search_group,
+            cmf.field_class,
             FALSE AS sorter,
             -- always treat identifier fields as non-search fields.
             (cmf.field_class <> 'identifier' AND cmf.search_field) AS search_field,
@@ -92,7 +92,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS
 
 CREATE OR REPLACE FUNCTION elastic.bib_record_attrs(bre_id BIGINT)
 RETURNS TABLE (
-    search_group TEXT,
+    field_class TEXT,
     name TEXT,
     source BIGINT,
     value TEXT
@@ -100,7 +100,7 @@ RETURNS TABLE (
 AS $FUNK$
     SELECT DISTINCT record.* FROM (
         SELECT 
-            NULL::TEXT AS search_group, 
+            NULL::TEXT AS field_class, 
             crad.name, 
             mrs.source, 
             mrs.value
@@ -111,7 +111,7 @@ AS $FUNK$
 
         -- record attributes
         SELECT 
-            NULL::TEXT AS search_group, 
+            NULL::TEXT AS field_class, 
             crad.name, 
             mraf.id AS source, 
             mraf.value
@@ -123,7 +123,7 @@ $FUNK$ LANGUAGE SQL STABLE;
 
 CREATE OR REPLACE FUNCTION elastic.bib_record_static_props(bre_id BIGINT)
 RETURNS TABLE (
-    search_group TEXT,
+    field_class TEXT,
     name TEXT,
     source BIGINT,
     value TEXT
@@ -131,7 +131,7 @@ RETURNS TABLE (
 AS $FUNK$
     SELECT DISTINCT record.* FROM (
         SELECT
-            cmf.field_class AS search_group, 
+            cmf.field_class,
             cmf.name, 
             props.source, 
             CASE WHEN cmf.joiner IS NOT NULL THEN
@@ -140,17 +140,26 @@ AS $FUNK$
                 props.value
             END AS value
         FROM (
-            SELECT * FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1
+            SELECT field, source, value 
+                FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1
             UNION 
-            SELECT * FROM metabib.author_field_entry mafe WHERE mafe.source = $1
+            SELECT field, source, value 
+                FROM metabib.author_field_entry mafe WHERE mafe.source = $1
             UNION 
-            SELECT * FROM metabib.subject_field_entry msfe WHERE msfe.source = $1
+            SELECT field, source, value 
+                FROM metabib.subject_field_entry msfe WHERE msfe.source = $1
             UNION 
-            SELECT * FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1
+            SELECT field, source, value 
+                FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1
             UNION 
-            SELECT * FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1
+            SELECT field, source, value 
+                FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1
             UNION 
-            SELECT * FROM metabib.identifier_field_entry mife WHERE mife.source = $1
+            SELECT field, source, value 
+                FROM metabib.identifier_field_entry mife WHERE mife.source = $1
+            UNION 
+            SELECT field, source, value 
+                FROM metabib.facet_entry mfe WHERE mfe.source = $1
         ) props
         JOIN config.metabib_field cmf ON (cmf.id = props.field)
         WHERE cmf.elastic_field
@@ -159,7 +168,7 @@ $FUNK$ LANGUAGE SQL STABLE;
 
 CREATE OR REPLACE FUNCTION elastic.bib_record_dynamic_props(bre_id BIGINT)
 RETURNS TABLE (
-    search_group TEXT,
+    field_class TEXT,
     name TEXT,
     source BIGINT,
     value TEXT
@@ -167,7 +176,7 @@ RETURNS TABLE (
 AS $FUNK$
     SELECT DISTINCT record.* FROM (
         SELECT
-            cmf.field_class AS search_group, 
+            cmf.field_class,
             cmf.name, 
             props.source, 
             CASE WHEN cmf.joiner IS NOT NULL THEN
@@ -186,7 +195,7 @@ $FUNK$ LANGUAGE SQL STABLE;
 
 CREATE OR REPLACE FUNCTION elastic.bib_record_properties(bre_id BIGINT) 
     RETURNS TABLE (
-        search_group TEXT,
+        field_class TEXT,
         name TEXT,
         source BIGINT,
         value TEXT
@@ -290,4 +299,9 @@ WHERE name NOT IN (
     'vr_format'
 );
 
+-- Bill's elastic VM for testing.
+UPDATE elastic.node 
+    SET host = 'elastic.gamma', port = 80, path = '/elastic/node1' 
+    WHERE id = 1;
+
 */

diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl
index 6da51e1..a03ea9b 100755 (executable)
--- a/Open-ILS/src/support-scripts/elastic-index.pl
+++ b/Open-ILS/src/support-scripts/elastic-index.pl
@@ -22,7 +22,7 @@ my $start_record;
 my $stop_record;
 my $modified_since;
 my $max_duration;
-my $use_xslt;
+my $bib_transform;
 my $batch_size = 500;
 
 # Database settings read from ENV by default.
@@ -48,7 +48,7 @@ GetOptions(
     'modified-since=s'  => \$modified_since,
     'max-duration=s'    => \$max_duration,
     'batch-size=s'      => \$batch_size,
-    'use-xslt=s'        => \$use_xslt,
+    'bib-transform=s'   => \$bib_transform,
     'db-name=s'         => \$db_name,
     'db-host=s'         => \$db_host,
     'db-port=s'         => \$db_port,
@@ -145,12 +145,12 @@ my $es;
 
 if ($index_class eq 'bib-search') {
 
-    if ($use_xslt) {
+    if ($bib_transform) {
         $es = OpenILS::Elastic::BibSearch::XSLT->new(
             cluster => $cluster, 
             index_name => $index_name,
             write_mode => 1,
-            xsl_file => $use_xslt
+            xsl_file => $bib_transform
         );
     } else {