From db45d54a633e134860e3493bad65e4dfb683c105 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Thu, 20 Feb 2020 10:41:58 -0500 Subject: [PATCH] LP1844418 Direct indexing experiment WIP; field class Signed-off-by: Bill Erickson --- .../src/perlmods/lib/OpenILS/Elastic/BibSearch.pm | 14 +++---- .../perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm | 25 ++++++------ .../sql/Pg/upgrade/XXXX.schema.elastic-search.sql | 46 ++++++++++++++-------- Open-ILS/src/support-scripts/elastic-index.pl | 8 ++-- 4 files changed, 52 insertions(+), 41 deletions(-) diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm index e96949c327..5bcb5aed4d 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm @@ -258,19 +258,19 @@ sub create_index_properties { my $field_name = $field->name; - my $search_group = $field->search_group; - $field_name = "$search_group|$field_name" if $search_group; + my $field_class = $field->field_class; + $field_name = "$field_class|$field_name" if $field_class; $logger->info("ES ONE FIELD name=$field_name: " . OpenSRF::Utils::JSON->perl2JSON($field)); my $def; - if ($search_group) { + if ($field_class) { if ($field->search_field eq 't') { # Use the same fields and analysis as the 'grouped' field. - $def = clone($properties->{$search_group}); - $def->{copy_to} = [$search_group, $SHORT_GROUP_MAP{$search_group}]; + $def = clone($properties->{$field_class}); + $def->{copy_to} = [$field_class, $SHORT_GROUP_MAP{$field_class}]; # Apply ranking boost to each analysis variation. my $flds = $def->{fields}; @@ -398,7 +398,7 @@ sub get_bib_data { my $ids_str = join(',', @$record_ids); my $sql = <{edit_date} = $field->{edit_date}) =~ s/ /T/g; } - my $fclass = $field->{search_group}; + my $fclass = $field->{field_class}; my $fname = $field->{name}; my $value = $field->{value}; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm index 88a1032ef0..6178936dbc 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm @@ -24,9 +24,9 @@ sub name { my $self = shift; return $self->{name}; } -sub search_group { +sub field_class { my $self = shift; - return $self->{search_group}; + return $self->{field_class}; } sub search_field { my $self = shift; @@ -57,7 +57,6 @@ use XML::LibXSLT; use OpenSRF::Utils::Logger qw/:logger/; use OpenILS::Utils::CStoreEditor qw/:funcs/; use OpenILS::Elastic::BibSearch; -use OpenILS::Utils::Normalize; use base qw/OpenILS::Elastic::BibSearch/; @@ -88,26 +87,24 @@ sub xsl_sheet { my @seen_fields; sub add_dynamic_field { - my ($self, $fields, $purpose, $search_group, $name, $weight, $normalizers) = @_; + my ($self, $fields, $purpose, $field_class, $name, $weight) = @_; return unless $name; $weight = '' if !$weight || $weight eq '_'; - $search_group = '' if !$search_group || $search_group eq '_'; - $normalizers = '' if !$normalizers || $normalizers eq '_'; + $field_class = '' if !$field_class || $field_class eq '_'; - my $tag = $purpose . $search_group . $name; + my $tag = $purpose . $field_class . $name; return if grep {$_ eq $tag} @seen_fields; push(@seen_fields, $tag); $logger->info("ES adding dynamic field purpose=$purpose ". - "search_group=$search_group name=$name weight=$weight"); + "field_class=$field_class name=$name weight=$weight"); my $field = OpenILS::Elastic::BibSearch::BibField->new( purpose => $purpose, - search_group => $search_group, + field_class => $field_class, name => $name, - weight => $weight, - normalizers => $normalizers + weight => $weight ); push(@$fields, $field); @@ -152,15 +149,15 @@ sub get_bib_data { my @rows = split(/\n/, $output); for my $row (@rows) { - my ($purpose, $search_group, $name, @tokens) = split(/ /, $row); + my ($purpose, $field_class, $name, @tokens) = split(/ /, $row); - $search_group = '' if ($search_group || '') eq '_'; + $field_class = '' if ($field_class || '') eq '_'; my $value = join(' ', @tokens); my $field = { purpose => $purpose, - search_group => $search_group, + field_class => $field_class, name => $name, value => $value }; diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql index fc5430f459..193002adff 100644 --- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql @@ -67,7 +67,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS NULL::INT AS metabib_field, crad.name, crad.label, - NULL AS search_group, + NULL AS field_class, crad.sorter, FALSE AS search_field, FALSE AS facet_field, @@ -79,7 +79,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS cmf.id AS metabib_field, cmf.name, cmf.label, - cmf.field_class AS search_group, + cmf.field_class, FALSE AS sorter, -- always treat identifier fields as non-search fields. (cmf.field_class <> 'identifier' AND cmf.search_field) AS search_field, @@ -92,7 +92,7 @@ CREATE OR REPLACE VIEW elastic.bib_field AS CREATE OR REPLACE FUNCTION elastic.bib_record_attrs(bre_id BIGINT) RETURNS TABLE ( - search_group TEXT, + field_class TEXT, name TEXT, source BIGINT, value TEXT @@ -100,7 +100,7 @@ RETURNS TABLE ( AS $FUNK$ SELECT DISTINCT record.* FROM ( SELECT - NULL::TEXT AS search_group, + NULL::TEXT AS field_class, crad.name, mrs.source, mrs.value @@ -111,7 +111,7 @@ AS $FUNK$ -- record attributes SELECT - NULL::TEXT AS search_group, + NULL::TEXT AS field_class, crad.name, mraf.id AS source, mraf.value @@ -123,7 +123,7 @@ $FUNK$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION elastic.bib_record_static_props(bre_id BIGINT) RETURNS TABLE ( - search_group TEXT, + field_class TEXT, name TEXT, source BIGINT, value TEXT @@ -131,7 +131,7 @@ RETURNS TABLE ( AS $FUNK$ SELECT DISTINCT record.* FROM ( SELECT - cmf.field_class AS search_group, + cmf.field_class, cmf.name, props.source, CASE WHEN cmf.joiner IS NOT NULL THEN @@ -140,17 +140,26 @@ AS $FUNK$ props.value END AS value FROM ( - SELECT * FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1 + SELECT field, source, value + FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1 UNION - SELECT * FROM metabib.author_field_entry mafe WHERE mafe.source = $1 + SELECT field, source, value + FROM metabib.author_field_entry mafe WHERE mafe.source = $1 UNION - SELECT * FROM metabib.subject_field_entry msfe WHERE msfe.source = $1 + SELECT field, source, value + FROM metabib.subject_field_entry msfe WHERE msfe.source = $1 UNION - SELECT * FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1 + SELECT field, source, value + FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1 UNION - SELECT * FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1 + SELECT field, source, value + FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1 UNION - SELECT * FROM metabib.identifier_field_entry mife WHERE mife.source = $1 + SELECT field, source, value + FROM metabib.identifier_field_entry mife WHERE mife.source = $1 + UNION + SELECT field, source, value + FROM metabib.facet_entry mfe WHERE mfe.source = $1 ) props JOIN config.metabib_field cmf ON (cmf.id = props.field) WHERE cmf.elastic_field @@ -159,7 +168,7 @@ $FUNK$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION elastic.bib_record_dynamic_props(bre_id BIGINT) RETURNS TABLE ( - search_group TEXT, + field_class TEXT, name TEXT, source BIGINT, value TEXT @@ -167,7 +176,7 @@ RETURNS TABLE ( AS $FUNK$ SELECT DISTINCT record.* FROM ( SELECT - cmf.field_class AS search_group, + cmf.field_class, cmf.name, props.source, CASE WHEN cmf.joiner IS NOT NULL THEN @@ -186,7 +195,7 @@ $FUNK$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION elastic.bib_record_properties(bre_id BIGINT) RETURNS TABLE ( - search_group TEXT, + field_class TEXT, name TEXT, source BIGINT, value TEXT @@ -290,4 +299,9 @@ WHERE name NOT IN ( 'vr_format' ); +-- Bill's elastic VM for testing. +UPDATE elastic.node + SET host = 'elastic.gamma', port = 80, path = '/elastic/node1' + WHERE id = 1; + */ diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl index 6da51e1af7..a03ea9b9b7 100755 --- a/Open-ILS/src/support-scripts/elastic-index.pl +++ b/Open-ILS/src/support-scripts/elastic-index.pl @@ -22,7 +22,7 @@ my $start_record; my $stop_record; my $modified_since; my $max_duration; -my $use_xslt; +my $bib_transform; my $batch_size = 500; # Database settings read from ENV by default. @@ -48,7 +48,7 @@ GetOptions( 'modified-since=s' => \$modified_since, 'max-duration=s' => \$max_duration, 'batch-size=s' => \$batch_size, - 'use-xslt=s' => \$use_xslt, + 'bib-transform=s' => \$bib_transform, 'db-name=s' => \$db_name, 'db-host=s' => \$db_host, 'db-port=s' => \$db_port, @@ -145,12 +145,12 @@ my $es; if ($index_class eq 'bib-search') { - if ($use_xslt) { + if ($bib_transform) { $es = OpenILS::Elastic::BibSearch::XSLT->new( cluster => $cluster, index_name => $index_name, write_mode => 1, - xsl_file => $use_xslt + xsl_file => $bib_transform ); } else { -- 2.11.0