From c0f2d73ad7af0dbfddd9a2ab43026eb461c5bfcf Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Tue, 3 Sep 2019 17:29:42 -0400 Subject: [PATCH] tighter catalog integration WIP Signed-off-by: Bill Erickson --- .../eg2/src/app/share/catalog/catalog.service.ts | 16 ++- .../app/share/catalog/elastic-search-context.ts | 78 ++++++++++----- .../eg2/src/app/share/catalog/search-context.ts | 13 +-- .../lib/OpenILS/Application/Search/Elastic.pm | 107 +++++---------------- .../OpenILS/Application/Search/ElasticMapper.pm | 2 +- .../src/perlmods/lib/OpenILS/Elastic/Bib/Search.pm | 1 + 6 files changed, 97 insertions(+), 120 deletions(-) diff --git a/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts b/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts index abf6640ed3..1b91d82ebb 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts @@ -95,6 +95,8 @@ export class CatalogService { let method = 'open-ils.search.biblio.marc'; if (ctx.isStaff) { method += '.staff'; } + const method = ctx.getApiName(); + const queryStruct = ctx.compileMarcSearchArgs(); return this.net.request('open-ils.search', method, queryStruct) @@ -125,7 +127,19 @@ export class CatalogService { console.debug('search query', JSON.stringify(fullQuery)); - const method = ctx.getApiName(); + let method = ctx.getApiName(); + if (method === null) { + method = 'open-ils.search.biblio.multiclass.query'; + + if (ctx.termSearch.groupByMetarecord + && !ctx.termSearch.fromMetarecord) { + method = 'open-ils.search.metabib.multiclass.query'; + } + + if (ctx.isStaff) { + method += '.staff'; + } + } return new Promise((resolve, reject) => { this.net.request( diff --git a/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts b/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts index f5474bcd5f..bf4cd6d3ee 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/elastic-search-context.ts @@ -7,12 +7,12 @@ class ElasticSearchParams { available: boolean; sort: any[] = []; searches: any[] = []; + marc_searches: any[] = []; filters: any[] = []; } export class ElasticSearchContext extends CatalogSearchContext { - // The UI is ambiguous re: mixing ANDs and ORs. // Here booleans are grouped ANDs first, then each OR is given its own node. compileTerms(params: ElasticSearchParams) { @@ -45,7 +45,7 @@ export class ElasticSearchContext extends CatalogSearchContext { }); } - addFilter(params: ElasticSearchParams, name: string, value: any) { + addTermFilter(params: ElasticSearchParams, name: string, value: any) { if (value === '' || value === null || value === undefined) { return; } @@ -76,23 +76,15 @@ export class ElasticSearchContext extends CatalogSearchContext { compileTermSearchQuery(): any { const ts = this.termSearch; - const params = new ElasticSearchParams(); + const params = this.newParams(); params.available = ts.available; - if (this.sort) { - // e.g. title, title.descending => [{title => 'desc'}] - const parts = this.sort.split(/\./); - const sort: any = {}; - sort[parts[0]] = parts[1] ? 'desc' : 'asc'; - params.sort = [sort]; - } - if (ts.date1 && ts.dateOp) { const dateFilter: Object = {}; switch (ts.dateOp) { case 'is': - this.addFilter(params, 'date1', ts.date1); + this.addTermFilter(params, 'date1', ts.date1); break; case 'before': params.filters.push({range: {date1: {lt: ts.date1}}}); @@ -109,7 +101,6 @@ export class ElasticSearchContext extends CatalogSearchContext { } this.compileTerms(params); - params.search_org = this.searchOrg.id(); if (this.global) { params.search_depth = this.org.root().ou_type().depth(); @@ -123,38 +114,77 @@ export class ElasticSearchContext extends CatalogSearchContext { */ if (ts.format) { - this.addFilter(params, ts.formatCtype, ts.format); + this.addTermFilter(params, ts.formatCtype, ts.format); } Object.keys(ts.ccvmFilters).forEach(field => { ts.ccvmFilters[field].forEach(value => { if (value !== '') { - this.addFilter(params, field, value); + this.addTermFilter(params, field, value); } }); }); ts.facetFilters.forEach(f => { - this.addFilter(params, + this.addTermFilter(params, `${f.facetClass}|${f.facetName}`, f.facetValue); }); return params; } + newParams(): ElasticSearchParams { + const params = new ElasticSearchParams(); + params.limit = this.pager.limit; + params.offset = this.pager.offset; + params.search_org = this.searchOrg.id() + + if (this.sort) { + // e.g. title, title.descending => [{title => 'desc'}] + const parts = this.sort.split(/\./); + const sort: any = {}; + sort[parts[0]] = parts[1] ? 'desc' : 'asc'; + params.sort = [sort]; + } + + return params; + } + + compileMarcSearchArgs(): any { + const ms = this.marcSearch; + const params = this.newParams(); + + ms.values.forEach((val, idx) => { + if (val !== '') { + params.marc_searches.push({ + tag: ms.tags[idx], + subfield: ms.subfields[idx] ? ms.subfields[idx] : null, + value: ms.values[idx] + }); + } + }); + + return params; + } + getApiName(): string { // Elastic covers only a subset of available search types. - if (!this.termSearch.isSearchable() || - this.termSearch.groupByMetarecord || - this.termSearch.fromMetarecord + if (this.marcSearch.isSearchable() || + ( + this.termSearch.isSearchable() && + !this.termSearch.groupByMetarecord && + !this.termSearch.fromMetarecord + ) ) { - return super.getApiName(); - } - return this.isStaff ? - 'open-ils.search.elastic.bib_search.staff' : - 'open-ils.search.elastic.bib_search'; + return this.isStaff ? + 'open-ils.search.elastic.bib_search.staff' : + 'open-ils.search.elastic.bib_search'; + } + + // Fall back to existing APIs. + return super.getApiName(); } } diff --git a/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts b/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts index eb81f41ffc..223c87768e 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts @@ -682,18 +682,7 @@ export class CatalogSearchContext { } getApiName(): string { - let method = 'open-ils.search.biblio.multiclass.query'; - - if (this.termSearch.groupByMetarecord - && !this.termSearch.fromMetarecord) { - method = 'open-ils.search.metabib.multiclass.query'; - } - - if (this.isStaff) { - method += '.staff'; - } - - return method; + return null; } } diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm index 42e199e3c3..025ba60c77 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm @@ -22,7 +22,6 @@ use OpenILS::Utils::Fieldmapper; use OpenSRF::Utils::SettingsClient; use OpenILS::Utils::CStoreEditor q/:funcs/; use OpenILS::Elastic::BibSearch; -use OpenILS::Elastic::BibMarc; use List::Util qw/min/; use OpenILS::Application::AppUtils; @@ -139,6 +138,8 @@ sub compile_elastic_query { append_search_nodes($elastic, $_) for @{$query->{searches}}; + append_marc_nodes($elastic, $_) for @{$query->{marc_searches}}; + add_elastic_holdings_filter($elastic, $staff, $query->{search_org}, $query->{search_depth}, $query->{available}); @@ -363,98 +364,40 @@ sub add_elastic_holdings_filter { } -sub compile_elastic_marc_query { - my ($args, $staff, $offset, $limit) = @_; - - # args->{searches} = - # [{term => "harry", restrict => [{tag => 245, subfield => "a"}]}] - - my $root_and = []; - for my $search (@{$args->{searches}}) { - - # NOTE Assume only one tag/subfield will be queried per search term. - my $tag = $search->{restrict}->[0]->{tag}; - my $sf = $search->{restrict}->[0]->{subfield}; - my $value = $search->{term}; - - # Use text searching on the value field - my $value_query = { - bool => { - should => [ - {match => {'marc.value.text' => - {query => $value, operator => 'and'}}}, - {match => {'marc.value.text_folded' => - {query => $value, operator => 'and'}}} - ] - } - }; - - my @must = ($value_query); - - # tag (ES-only) and subfield are both optional - push (@must, {term => {'marc.tag' => $tag}}) if $tag; - push (@must, {term => {'marc.subfield' => $sf}}) if $sf && $sf ne '_'; +sub append_marc_nodes { + my ($marc_search) = @_; - my $sub_query = {bool => {must => \@must}}; + my $tag = $marc_search->{tag}; + my $sf = $marc_search->{subfield}; + my $value = $marc_search->{value}; - push (@$root_and, { - nested => { - path => 'marc', - query => {bool => {must => $sub_query}} - } - }); - } - - return { - _source => ['id'], # Fetch bib ID only - size => $limit, - from => $offset, - sort => [], - query => { - bool => { - must => $root_and, - filter => [] - } + # Use text searching on the value field + my $value_query = { + bool => { + should => [ + {match => {'marc.value.text' => + {query => $value, operator => 'and'}}}, + {match => {'marc.value.text_folded' => + {query => $value, operator => 'and'}}} + ] } }; -} + my @must = ($value_query); + # tag (ES-only) and subfield are both optional + push (@must, {term => {'marc.tag' => $tag}}) if $tag; + push (@must, {term => {'marc.subfield' => $sf}}) if $sf; -# Translate a MARC search API call into something consumable by Elasticsearch -# Translate search results into a structure consistent with a bib search -# API response. -# TODO: This version is not currently holdings-aware, meaning it will return -# results for all non-deleted bib records that match the query. -sub marc_search { - my ($class, $args, $staff, $limit, $offset) = @_; - - return {count => 0, ids => []} - unless $args->{searches} && @{$args->{searches}}; - - my $elastic_query = - compile_elastic_marc_query($args, $staff, $offset, $limit); - - my $es = OpenILS::Elastic::BibMarc->new('main'); - - $es->connect; - my $results = $es->search($elastic_query); - - $logger->debug("ES elasticsearch returned: ". - OpenSRF::Utils::JSON->perl2JSON($results)); - - return {count => 0, ids => []} unless $results; - - my @bib_ids = map {$_->{_id}} - grep {defined $_} @{$results->{hits}->{hits}}; + my $sub_query = {bool => {must => \@must}}; return { - ids => \@bib_ids, - count => $results->{hits}->{total} + nested => { + path => 'marc', + query => {bool => {must => $sub_query}} + } }; } - - 1; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/ElasticMapper.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/ElasticMapper.pm index 43d36e1242..291fe4241a 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/ElasticMapper.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/ElasticMapper.pm @@ -21,7 +21,7 @@ use OpenILS::Utils::Fieldmapper; use OpenSRF::Utils::SettingsClient; use OpenILS::Utils::CStoreEditor q/:funcs/; use OpenILS::Elastic::Bib::Search; -use OpenILS::Elastic::Bib::Marc; +#use OpenILS::Elastic::Bib::Marc; use List::Util qw/min/; use Digest::MD5 qw(md5_hex); diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/Bib/Search.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/Bib/Search.pm index 3f26524283..c45beec76c 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/Bib/Search.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/Bib/Search.pm @@ -87,6 +87,7 @@ my $BASE_PROPERTIES = { } } } + }; sub index_name { -- 2.11.0