From: Bill Erickson <berickxx@gmail.com>
Date: Tue, 3 Sep 2019 19:24:05 +0000 (-0400)
Subject: tighter catalog integration WIP
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=ac1dfc356a94a6dd6dbf4d4021cac7f64c52e062;p=working%2FEvergreen.git

tighter catalog integration WIP

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---

diff --git a/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts b/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts
index de19ca59d0..f5474bcd5f 100644
--- a/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts
+++ b/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts
@@ -6,7 +6,8 @@ class ElasticSearchParams {
     search_depth: number;
     available: boolean;
     sort: any[] = [];
-    query: any = {bool: {must: [], filter: []}};
+    searches: any[] = [];
+    filters: any[] = [];
 }
 
 export class ElasticSearchContext extends CatalogSearchContext {
@@ -17,63 +18,31 @@ export class ElasticSearchContext extends CatalogSearchContext {
     compileTerms(params: ElasticSearchParams) {
 
         const ts = this.termSearch;
-        const terms: any = {
-            bool: {
-                should: [
-                    {bool: {must: []}} // ANDs
-                    // ORs
-                ]
-            }
-        };
 
         ts.joinOp.forEach((op, idx) => {
             let matchOp = 'match';
 
-            // The 'and' operator here tells EL to treat multi-word search
-            // terms as an ANDed pair (e.g. "harry potter" = "harry and potter")
-            let operator = 'and';
-
             switch (ts.matchOp[idx]) {
                 case 'phrase':
                     matchOp = 'match_phrase';
-                    operator = null;
                     break;
                 case 'nocontains':
                     matchOp = 'must_not';
                     break;
                 case 'exact':
                     matchOp = 'term';
-                    operator = null;
                     break;
                 case 'starts':
                     matchOp = 'match_phrase_prefix';
-                    operator = null;
                     break;
             }
 
-            let node: any = {};
-            node[matchOp] = {};
-
-            if (operator) {
-                node[matchOp][ts.fieldClass[idx]] = 
-                    {query: ts.query[idx], operator: operator};
-            } else {
-                node[matchOp][ts.fieldClass[idx]] = ts.query[idx];
-            }
-
-            if (matchOp === 'must_not') {
-                // adds a boolean sub-node
-                node = {bool: node};
-            }
-
-            if (ts.joinOp[idx] === 'or') {
-                terms.bool.should.push(node);
-            } else {
-                terms.bool.should[0].bool.must.push(node);
-            }
+            params.searches.push({
+                field: ts.fieldClass[idx],
+                match_op: matchOp,
+                value: ts.query[idx]
+            });
         });
-
-        params.query.bool.must.push(terms);
     }
 
     addFilter(params: ElasticSearchParams, name: string, value: any) {
@@ -81,9 +50,28 @@ export class ElasticSearchContext extends CatalogSearchContext {
             value === null || 
             value === undefined) { return; }
 
+        // Multiple filter values for a single filter are OR'ed.
+        for (let idx = 0; idx < params.filters.length; idx++) {
+            const filter = params.filters[idx];
+
+            if (filter.term && name in filter.term) {
+                // Pluralize an existing filter
+                filter.terms = {};
+                filter.terms[name] = [filter.term[name], value];
+                delete filter.term;
+                return;
+
+            } else if (filter.terms && name in filter.terms) {
+                // Append a filter value to an already pluralized filter.
+                filter.terms[name].push(value);
+                return;
+            }
+        }
+
+        // New filter type
         const node: any = {term: {}};
         node.term[name] = value;
-        params.query.bool.filter.push(node);
+        params.filters.push(node);
     }
 
     compileTermSearchQuery(): any {
@@ -101,20 +89,21 @@ export class ElasticSearchContext extends CatalogSearchContext {
         }
 
         if (ts.date1 && ts.dateOp) {
+            const dateFilter: Object = {};
             switch (ts.dateOp) {
                 case 'is':
                     this.addFilter(params, 'date1', ts.date1);
                     break;
                 case 'before':
-                    this.addFilter(params, 'range', {date1: {'lt': ts.date1}});
+                    params.filters.push({range: {date1: {lt: ts.date1}}});
                     break;
                 case 'after':
-                    this.addFilter(params, 'range', {date1: {'gt': ts.date1}});
+                    params.filters.push({range: {date1: {gt: ts.date1}}});
                     break;
                 case 'between':
                     if (ts.date2) {
-                        this.addFilter(params, 'range', 
-                            {date1: {'gt': ts.date1, 'lt': ts.date2}});
+                        params.filters.push(
+                            {range: {date1: {gt: ts.date1, lt: ts.date2}}});
                     }
             }
         }
@@ -147,7 +136,7 @@ export class ElasticSearchContext extends CatalogSearchContext {
 
         ts.facetFilters.forEach(f => {
             this.addFilter(params, 
-                `${f.facetClass}:${f.facetName}`, f.facetValue);
+                `${f.facetClass}|${f.facetName}`, f.facetValue);
         });
 
         return params;
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search.pm
index b1d34d90f1..f025bb7216 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search.pm
@@ -37,6 +37,7 @@ sub initialize {
 sub child_init {
     OpenILS::Application::Search::Z3950->child_init;
     OpenILS::Application::Search::Browse->child_init;
+    OpenILS::Application::Search::Elastic->child_init;
 }
     
 
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
index 47d95d7089..af73b793ca 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Biblio.pm
@@ -10,7 +10,6 @@ use OpenSRF::Utils::SettingsClient;
 use OpenILS::Utils::CStoreEditor q/:funcs/;
 use OpenSRF::Utils::Cache;
 use Encode;
-use OpenILS::Application::Search::Elastic;
 use OpenILS::Application::Search::ElasticMapper;
 
 use OpenSRF::Utils::Logger qw/:logger/;
@@ -1162,7 +1161,7 @@ sub staged_search {
         $search_hash->{query}, # query string
         ($method =~ /staff/ ? 1 : 0),
         $user_offset, $user_limit
-    ) if OpenILS::Application::Search::Elastic->is_enabled('bib-search');
+    ) if OpenILS::Application::Search::ElasticMapper->is_enabled('bib-search');
 
     # we're grabbing results on a per-superpage basis, which means the 
     # limit and offset should coincide with superpage boundaries
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm
index be47a75dde..42e199e3c3 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm
@@ -38,32 +38,8 @@ my $hidden_copy_locations;
 my $avail_copy_statuses;
 our $enabled = {};
 
-# Returns true if the Elasticsearch 'bib-search' index is active.
-sub is_enabled {
-    my ($class, $index) = @_;
-
-    $class->init;
-
-    return $enabled->{$index} if exists $enabled->{$index};
-
-    # Elastic bib search is enabled if a "bib-search" index is enabled.
-    my $config = new_editor()->search_elastic_index(
-        {active => 't', code => $index})->[0];
-
-    if ($config) {
-        $logger->info("ES '$index' index is enabled");
-        $enabled->{$index} = 1;
-    } else {
-        $enabled->{$index} = 0;
-    }
-
-    return $enabled->{$index};
-}
-
-my $init_complete = 0;
-sub init {
+sub child_init {
     my $class = shift;
-    return if $init_complete;
 
     my $e = new_editor();
 
@@ -93,7 +69,6 @@ sub init {
 
     $hidden_copy_locations = [map {$_->{id}} @$locs];
 
-    $init_complete = 1;
     return 1;
 }
 
@@ -101,6 +76,7 @@ __PACKAGE__->register_method(
     method   => 'bib_search',
     api_name => 'open-ils.search.elastic.bib_search'
 );
+
 __PACKAGE__->register_method(
     method   => 'bib_search',
     api_name => 'open-ils.search.elastic.bib_search.staff'
@@ -153,9 +129,16 @@ sub compile_elastic_query {
         size => $options->{limit},
         from => $options->{offset},
         sort => $query->{sort},
-        query => $query->{query}
+        query => {
+            bool => {
+                must => [],
+                filter => $query->{filters} || []
+            }
+        }
     };
 
+    append_search_nodes($elastic, $_) for @{$query->{searches}};
+
     add_elastic_holdings_filter($elastic, $staff, 
         $query->{search_org}, $query->{search_depth}, $query->{available});
 
@@ -166,6 +149,81 @@ sub compile_elastic_query {
     return $elastic;
 }
 
+
+# Translate the simplified boolean search nodes into an Elastic
+# boolean structure with the appropriate index names.
+sub append_search_nodes {
+    my ($elastic, $search) = @_;
+
+    my ($field_class, $field_name) = split(/\|/, $search->{field});
+    my $match_op = $search->{match_op};
+    my $value = $search->{value};
+
+    my @fields;
+    if ($field_name) {
+        @fields = ($field_name);
+
+    } else {
+        # class-level searches are OR ("should") searches across all
+        # fields in the selected class.
+
+        @fields = map {$_->name} 
+            grep {$_->search_group eq $field_class} @$bib_fields;
+    }
+
+    $logger->info("ES adding searches for class=$field_class and fields=@fields");
+
+    my $must_not = $match_op eq 'must_not';
+
+    # Build a must_not query as a collection of must queries, which will 
+    # be combined under a single must_not parent query.
+    $match_op = 'must' if $must_not; 
+
+    # for match queries, treat multi-word search as AND searches
+    # instead of the default ES OR searches.
+    $value = {query => $value, operator => 'and'} if $match_op eq 'match';
+
+    my $field_nodes = [];
+    for my $field (@fields) {
+        my $key = "$field_class|$field";
+
+        if ($match_op eq 'term' || $match_op eq 'match_phrase_prefix') {
+
+            # Use the lowercase normalized keyword index for exact-match searches.
+            push(@$field_nodes, {$match_op => {"$key.lower" => $value}});
+
+        } else {
+
+            # use the full-text indices
+            
+            push(@$field_nodes, 
+                {$match_op => {"$key.text" => $value}});
+
+            push(@$field_nodes, 
+                {$match_op => {"$key.text_folded" => $value}});
+        }
+    }
+
+    my $query_part;
+    if (scalar(@$field_nodes) == 1) {
+        $query_part = {bool => {must => $field_nodes}};
+    } else {
+        # Query multiple fields within a search class via OR query.
+        $query_part = {bool => {should => $field_nodes}};
+    }
+
+    if ($must_not) {
+        # Negation query.  Wrap the whole shebang in a must_not
+        $query_part = {bool => {must_not => $query_part}};
+    }
+
+    $logger->info("ES field search part: ". 
+        OpenSRF::Utils::JSON->perl2JSON($query_part));
+
+    push(@{$elastic->{query}->{bool}->{must}}, $query_part);
+}
+
+
 # Format ES search aggregations to match the API response facet structure
 # {$cmf_id => {"Value" => $count}, $cmf_id2 => {"Value Two" => $count2}, ...}
 sub format_facets {
@@ -211,7 +269,7 @@ sub add_elastic_facet_aggregations {
 sub add_elastic_holdings_filter {
     my ($elastic_query, $staff, $org_id, $depth, $available) = @_;
 
-    # in non-staff mode, ensure at least on copy in scope is visible
+    # in non-staff mode, ensure at least one copy in scope is visible
     my $visible = !$staff;
 
     if ($org_id) {