From 21822cd44e14a68eafae9aa401523fb82140ab4c Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Fri, 6 Sep 2019 16:36:38 -0400 Subject: [PATCH] elastic-builder in action Signed-off-by: Bill Erickson --- .../src/app/share/catalog/catalog-common.module.ts | 4 +- .../eg2/src/app/share/catalog/catalog.service.ts | 18 +- .../eg2/src/app/share/catalog/elastic.service.ts | 184 +++++++++++++++++++++ .../eg2/src/app/share/catalog/search-context.ts | 38 +---- Open-ILS/src/eg2/src/app/staff/nav.component.ts | 6 - .../lib/OpenILS/Application/Search/Elastic.pm | 138 +--------------- 6 files changed, 205 insertions(+), 183 deletions(-) create mode 100644 Open-ILS/src/eg2/src/app/share/catalog/elastic.service.ts diff --git a/Open-ILS/src/eg2/src/app/share/catalog/catalog-common.module.ts b/Open-ILS/src/eg2/src/app/share/catalog/catalog-common.module.ts index ba8c915884..7d229eee33 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/catalog-common.module.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/catalog-common.module.ts @@ -7,6 +7,7 @@ import {BibRecordService} from './bib-record.service'; import {UnapiService} from './unapi.service'; import {MarcHtmlComponent} from './marc-html.component'; import {BibDisplayFieldComponent} from './bib-display-field.component'; +import {ElasticService} from './elastic.service'; @NgModule({ @@ -26,7 +27,8 @@ import {BibDisplayFieldComponent} from './bib-display-field.component'; CatalogUrlService, UnapiService, BibRecordService, - BasketService + BasketService, + ElasticService ] }) diff --git a/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts b/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts index f741fec3c2..2bb25f8989 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/catalog.service.ts @@ -10,6 +10,7 @@ import {CatalogSearchContext, CatalogSearchState} from './search-context'; import {BibRecordService, BibRecordSummary} from './bib-record.service'; import {BasketService} from './basket.service'; import {CATALOG_CCVM_FILTERS} from './search-context'; +import {ElasticService} from './elastic.service'; @Injectable() export class CatalogService { @@ -34,10 +35,10 @@ export class CatalogService { private unapi: UnapiService, private pcrud: PcrudService, private bibService: BibRecordService, - private basket: BasketService + private basket: BasketService, + private elastic: ElasticService ) { this.onSearchComplete = new EventEmitter(); - } search(ctx: CatalogSearchContext): Promise { @@ -110,6 +111,15 @@ export class CatalogService { termSearch(ctx: CatalogSearchContext): Promise { + if (this.elastic.canSearch(ctx)) { + return this.elastic.performSearch(ctx) + .then(result => { + this.applyResultData(ctx, result); + ctx.searchState = CatalogSearchState.COMPLETE; + this.onSearchComplete.emit(ctx); + }); + } + let method = 'open-ils.search.biblio.multiclass.query'; let fullQuery; @@ -129,10 +139,6 @@ export class CatalogService { } } - // TODO XXX TESTING - method = 'open-ils.search.elastic.bib_search'; - fullQuery = ctx.compileElasticSearchQuery(); - console.debug(`search query: ${fullQuery}`); if (ctx.isStaff) { diff --git a/Open-ILS/src/eg2/src/app/share/catalog/elastic.service.ts b/Open-ILS/src/eg2/src/app/share/catalog/elastic.service.ts new file mode 100644 index 0000000000..feb0690988 --- /dev/null +++ b/Open-ILS/src/eg2/src/app/share/catalog/elastic.service.ts @@ -0,0 +1,184 @@ +import {Injectable, EventEmitter} from '@angular/core'; +import {tap} from 'rxjs/operators'; +import {IdlService, IdlObject} from '@eg/core/idl.service'; +import {OrgService} from '@eg/core/org.service'; +import {NetService} from '@eg/core/net.service'; +import {PcrudService} from '@eg/core/pcrud.service'; +import {CatalogSearchContext} from './search-context'; +import {RequestBodySearch, MatchQuery, MultiMatchQuery, + Sort, BoolQuery, TermQuery} from 'elastic-builder'; + +@Injectable() +export class ElasticService { + + bibFields: IdlObject[] = []; + + constructor( + private idl: IdlService, + private net: NetService, + private org: OrgService, + private pcrud: PcrudService + ) {} + + init(): Promise { + if (this.bibFields.length > 0) { + return Promise.resolve(); + } + + return this.pcrud.search('ebf', {search_field: 't'}) + .pipe(tap(field => this.bibFields.push(field))) + .toPromise(); + } + + canSearch(ctx: CatalogSearchContext): boolean { + + if (ctx.marcSearch.isSearchable()) { return true; } + + if ( ctx.termSearch.isSearchable() && + !ctx.termSearch.groupByMetarecord && + !ctx.termSearch.fromMetarecord + ) { return true; } + + return false; + } + + + // For API consistency, returns an array of arrays whose first + // entry within each sub-array is a record ID. + performSearch(ctx: CatalogSearchContext): Promise { + + const requestBody = this.compileRequestBody(ctx); + + const method = ctx.isStaff ? + 'open-ils.search.elastic.bib_search.staff' : + 'open-ils.search.elastic.bib_search'; + + // Extract just the bits that get sent to ES. + const elasticStruct: Object = requestBody.toJSON(); + + console.log(JSON.stringify(elasticStruct)); + + const options: any = {search_org: ctx.searchOrg.id()}; + if (ctx.global) { + options.search_depth = this.org.root().ou_type().depth(); + } + + return this.net.request( + 'open-ils.search', method, elasticStruct, options + ).toPromise(); + } + + compileRequestBody(ctx: CatalogSearchContext): RequestBodySearch { + + const search = new RequestBodySearch() + search.source(['id']); // only retrieve IDs + search.size(ctx.pager.limit) + search.from(ctx.pager.offset); + + const rootAnd = new BoolQuery(); + + this.compileTermSearch(ctx, rootAnd); + this.addFilters(ctx, rootAnd); + this.addSort(ctx, search); + + search.query(rootAnd); + + return search; + } + + addSort(ctx: CatalogSearchContext, search: RequestBodySearch) { + + if (!ctx.sort) { return; } + + // e.g. title, title.descending => [{title => 'desc'}] + const parts = ctx.sort.split(/\./); + search.sort(new Sort(parts[0], parts[1] ? 'desc' : 'asc')); + } + + addFilters(ctx: CatalogSearchContext, rootAnd: BoolQuery) { + const ts = ctx.termSearch; + + if (ts.format) { + rootAnd.filter(new TermQuery(ts.formatCtype, ts.format)); + } + + Object.keys(ts.ccvmFilters).forEach(field => { + ts.ccvmFilters[field].forEach(value => { + if (value !== '') { + rootAnd.filter(new TermQuery(field, value)); + } + }); + }); + + ts.facetFilters.forEach(f => { + if (f.facetValue !== '') { + rootAnd.filter(new TermQuery( + `${f.facetClass}|${f.facetName}`, f.facetValue)); + } + }); + } + + compileTermSearch(ctx: CatalogSearchContext, rootAnd: BoolQuery) { + + // TODO: boolean OR support. + const ts = ctx.termSearch; + ts.joinOp.forEach((op, idx) => { + + const fieldClass = ts.fieldClass[idx]; + const textIndex = `${fieldClass}|*text*`; + const value = ts.query[idx]; + let query; + + switch (ts.matchOp[idx]) { + + case 'contains': + query = new MultiMatchQuery([textIndex], value); + query.operator('and'); + query.type('most_fields'); + rootAnd.must(query); + break; + + case 'phrase': + query = new MultiMatchQuery([textIndex], value); + query.type('phrase'); + rootAnd.must(query); + break; + + case 'nocontains': + query = new MultiMatchQuery([textIndex], value); + query.operator('and'); + query.type('most_fields'); + rootAnd.mustNot(query); + break; + + case 'exact': + + // TODO: these need to be grouped first by field + // so we can search multiple values on a singel term + // via 'terms' search. + + /* + const shoulds = []; + this.bibFields.filter(f => ( + f.search_field() === 't' && + f.search_group() === fieldClass + )).forEach(field => { + shoulds.push( + }); + + const should = new BoolQuery(); + */ + break; + + case 'starts': + query = new MultiMatchQuery([textIndex], value); + query.type('phrase_prefix'); + rootAnd.must(query); + break; + } + }); + } + + +} + diff --git a/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts b/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts index a733d7643c..790010d69b 100644 --- a/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts +++ b/Open-ILS/src/eg2/src/app/share/catalog/search-context.ts @@ -3,8 +3,6 @@ import {IdlObject} from '@eg/core/idl.service'; import {Pager} from '@eg/share/util/pager'; import {ArrayUtil} from '@eg/share/util/array'; -import {RequestBodySearch, MatchQuery} from 'elastic-builder'; - // CCVM's we care about in a catalog context // Don't fetch them all because there are a lot. export const CATALOG_CCVM_FILTERS = [ @@ -194,6 +192,7 @@ export class CatalogTermContext { matchOp: string[]; format: string; available = false; + // TODO: configurable // format limiter default to using the search_format filter formatCtype = 'search_format'; @@ -644,41 +643,6 @@ export class CatalogSearchContext { return str; } - compileElasticSearchQuery(): any { - const search = new RequestBodySearch(); - search.query(new MatchQuery('body', 'hello, ma!')); - - const ts = this.termSearch; - - ts.joinOp.forEach((op, idx) => { - let matchOp = 'match'; - - switch (ts.matchOp[idx]) { - case 'phrase': - matchOp = 'match_phrase'; - break; - case 'nocontains': - matchOp = 'must_not'; - break; - case 'exact': - matchOp = 'term'; - break; - case 'starts': - matchOp = 'match_phrase_prefix'; - break; - } - - params.searches.push({ - field: ts.fieldClass[idx], - match_op: matchOp, - value: ts.query[idx] - }); - - }); - - console.log(JSON.stringify(search)); - } - // A search context can collect enough data for multiple search // types to be searchable (e.g. users navigate through parts of a // search form). Calling this method and providing a search type diff --git a/Open-ILS/src/eg2/src/app/staff/nav.component.ts b/Open-ILS/src/eg2/src/app/staff/nav.component.ts index 3c695ae40a..f143727a33 100644 --- a/Open-ILS/src/eg2/src/app/staff/nav.component.ts +++ b/Open-ILS/src/eg2/src/app/staff/nav.component.ts @@ -49,17 +49,11 @@ export class StaffNavComponent implements OnInit { // NOTE: this can eventually go away. // Avoid attempts to fetch org settings if the user has not yet // logged in (e.g. this is the login page). - - // Force-show the angular catalog for Elastic dev, since that's - // the only site that will support it for now. - this.showAngularCatalog = true; - /* if (this.user()) { this.org.settings('ui.staff.angular_catalog.enabled') .then(settings => this.showAngularCatalog = Boolean(settings['ui.staff.angular_catalog.enabled'])); } - */ } user() { diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm index 025ba60c77..26e5892c8e 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm @@ -85,7 +85,7 @@ __PACKAGE__->register_method( # Translate search results into a structure consistent with a bib search # API response. sub bib_search { - my ($self, $client, $options, $query) = @_; + my ($self, $client, $query, $options) = @_; $options ||= {}; my $staff = ($self->api_name =~ /staff/); @@ -93,7 +93,7 @@ sub bib_search { $logger->info("ES parsing API query $query staff=$staff"); my ($elastic_query, $cache_key) = - compile_elastic_query($query, $options, $options); + compile_elastic_query($query, $options, $staff); my $es = OpenILS::Elastic::BibSearch->new('main'); @@ -121,110 +121,18 @@ sub bib_search { } sub compile_elastic_query { - my ($query, $options, $staff) = @_; - - my $elastic = { - _source => ['id'], # Fetch bib ID only - size => $options->{limit}, - from => $options->{offset}, - sort => $query->{sort}, - query => { - bool => { - must => [], - filter => $query->{filters} || [] - } - } - }; - - append_search_nodes($elastic, $_) for @{$query->{searches}}; - - append_marc_nodes($elastic, $_) for @{$query->{marc_searches}}; + my ($elastic, $options, $staff) = @_; add_elastic_holdings_filter($elastic, $staff, - $query->{search_org}, $query->{search_depth}, $query->{available}); + $options->{search_org}, $options->{search_depth}, $options->{available}); add_elastic_facet_aggregations($elastic); - $elastic->{sort} = ['_score'] unless @{$elastic->{sort}}; + $elastic->{sort} = ['_score'] unless @{$elastic->{sort} || []}; return $elastic; } - -# Translate the simplified boolean search nodes into an Elastic -# boolean structure with the appropriate index names. -sub append_search_nodes { - my ($elastic, $search) = @_; - - my ($field_class, $field_name) = split(/\|/, $search->{field}); - my $match_op = $search->{match_op}; - my $value = $search->{value}; - - my @fields; - if ($field_name) { - @fields = ($field_name); - - } else { - # class-level searches are OR ("should") searches across all - # fields in the selected class. - - @fields = map {$_->name} - grep {$_->search_group eq $field_class} @$bib_fields; - } - - $logger->info("ES adding searches for class=$field_class and fields=@fields"); - - my $must_not = $match_op eq 'must_not'; - - # Build a must_not query as a collection of must queries, which will - # be combined under a single must_not parent query. - $match_op = 'must' if $must_not; - - # for match queries, treat multi-word search as AND searches - # instead of the default ES OR searches. - $value = {query => $value, operator => 'and'} if $match_op eq 'match'; - - my $field_nodes = []; - for my $field (@fields) { - my $key = "$field_class|$field"; - - if ($match_op eq 'term' || $match_op eq 'match_phrase_prefix') { - - # Use the lowercase normalized keyword index for exact-match searches. - push(@$field_nodes, {$match_op => {"$key.lower" => $value}}); - - } else { - - # use the full-text indices - - push(@$field_nodes, - {$match_op => {"$key.text" => $value}}); - - push(@$field_nodes, - {$match_op => {"$key.text_folded" => $value}}); - } - } - - my $query_part; - if (scalar(@$field_nodes) == 1) { - $query_part = {bool => {must => $field_nodes}}; - } else { - # Query multiple fields within a search class via OR query. - $query_part = {bool => {should => $field_nodes}}; - } - - if ($must_not) { - # Negation query. Wrap the whole shebang in a must_not - $query_part = {bool => {must_not => $query_part}}; - } - - $logger->info("ES field search part: ". - OpenSRF::Utils::JSON->perl2JSON($query_part)); - - push(@{$elastic->{query}->{bool}->{must}}, $query_part); -} - - # Format ES search aggregations to match the API response facet structure # {$cmf_id => {"Value" => $count}, $cmf_id2 => {"Value Two" => $count2}, ...} sub format_facets { @@ -363,41 +271,5 @@ sub add_elastic_holdings_filter { push(@{$elastic_query->{query}->{bool}->{filter}}, $filter); } - -sub append_marc_nodes { - my ($marc_search) = @_; - - my $tag = $marc_search->{tag}; - my $sf = $marc_search->{subfield}; - my $value = $marc_search->{value}; - - # Use text searching on the value field - my $value_query = { - bool => { - should => [ - {match => {'marc.value.text' => - {query => $value, operator => 'and'}}}, - {match => {'marc.value.text_folded' => - {query => $value, operator => 'and'}}} - ] - } - }; - - my @must = ($value_query); - - # tag (ES-only) and subfield are both optional - push (@must, {term => {'marc.tag' => $tag}}) if $tag; - push (@must, {term => {'marc.subfield' => $sf}}) if $sf; - - my $sub_query = {bool => {must => \@must}}; - - return { - nested => { - path => 'marc', - query => {bool => {must => $sub_query}} - } - }; -} - 1; -- 2.11.0