From 676a73232abd69f45774b285b3ddf9ea9a8d4d47 Mon Sep 17 00:00:00 2001 From: Bill Erickson Date: Wed, 24 Oct 2018 10:36:56 -0400 Subject: [PATCH] bib-search indexing wip Signed-off-by: Bill Erickson --- Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm | 43 +++++++++++++++++----- .../src/perlmods/lib/OpenILS/Elastic/BibSearch.pm | 14 +++++-- Open-ILS/src/support-scripts/elastic-index.pl | 20 +++++----- .../support-scripts/test-scripts/elastic-search.pl | 12 ++---- 4 files changed, 59 insertions(+), 30 deletions(-) diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm index d6222bf63b..64312fc157 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm @@ -20,6 +20,7 @@ use OpenSRF::Utils::Logger qw/:logger/; use OpenSRF::Utils::SettingsClient; use OpenILS::Utils::CStoreEditor qw/:funcs/; use Search::Elasticsearch; +use Data::Dumper; sub new { my ($class, $cluster) = @_; @@ -157,25 +158,49 @@ sub delete_index { sub index_document { my ($self, $id, $body) = @_; - my $result = $self->es->index( - index => $self->index_name, - id => $id, - body => $body - ); + my $result; + + eval { + $result = $self->es->index( + index => $self->index_name, + type => 'record', # deprecated in v6 + id => $id, + body => $body + ); + }; + + if ($@) { + $logger->error("ES index_document failed with $@"); + return undef; + } $logger->debug("ES index command returned $result"); + return $result; } sub search { my ($self, $query) = @_; - return $self->es->search( - index => $self->index_name, - body => $query - ); + my $result; + + eval { + $result = $self->es->search( + index => $self->index_name, + body => $query + ); + }; + + if ($@) { + $logger->error("ES search failed with $@"); + $logger->error("ES failed query: " . Dumper($query)); + return undef; + } + + return $result; } + 1; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm index 51e63d220d..47cecb96d8 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm @@ -22,6 +22,8 @@ use XML::LibXSLT; use OpenSRF::Utils::Logger qw/:logger/; use OpenILS::Elastic; use base qw/OpenILS::Elastic/; +use Data::Dumper; +$Data::Dumper::Indent = 2; my $INDEX_NAME = 'bib-search'; @@ -119,7 +121,7 @@ sub index_name { sub index { my $self = shift; return $self->{index} if $self->{index}; - ($self->{index}) = grep {$_->{name} eq $INDEX_NAME} @{$self->{indices}}; + ($self->{index}) = grep {$_->{purpose} eq $INDEX_NAME} @{$self->{indices}}; return $self->{index}; } @@ -197,14 +199,16 @@ sub create_index { } my $settings = $BASE_INDEX_SETTINGS; - $settings->{number_of_replicas} = scalar($self->{servers}); + $settings->{number_of_replicas} = scalar(@{$self->{servers}}); $settings->{number_of_shards} = $self->index->{num_shards}; my $conf = { index => $INDEX_NAME, body => { settings => $settings, - mappings => {properties => $mappings} + mappings => {record => {properties => $mappings}} + # document type (i.e. 'record') deprecated in v6 + #mappings => {properties => $mappings} } }; @@ -214,6 +218,7 @@ sub create_index { if ($@) { $logger->error("ES failed to create index cluster=". $self->cluster. "index=$INDEX_NAME error=$@"); + print "$@\n\n"; return 0; } @@ -294,7 +299,8 @@ sub populate_bib_search_index_page { $body->{$field} = $val; } - $self->index_document($INDEX_NAME, $bib_id, $body); + return 0 unless + $self->index_document($bib_id, $body); $state->{last_bib_id} = $bib_id; $index_count++; diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl index 65adf7d6bf..27bbc81361 100755 --- a/Open-ILS/src/support-scripts/elastic-index.pl +++ b/Open-ILS/src/support-scripts/elastic-index.pl @@ -6,25 +6,21 @@ use OpenILS::Utils::Fieldmapper; use OpenILS::Elastic::BibSearch; my $help; -my $elastic_config; my $osrf_config = '/openils/conf/opensrf_core.xml'; my $cluster = 'main'; my $create_index; my $delete_index; -my $index_name; # use "all" to affect all configured indexes +my $index_name; my $populate; -my $partial; GetOptions( 'help' => \$help, - 'elastic-config=s' => \$elastic_config, 'osrf-config=s' => \$osrf_config, 'cluster=s' => \$cluster, 'create-index' => \$create_index, 'delete-index' => \$delete_index, 'index=s' => \$index_name, - 'populate' => \$populate, - 'partial' => \$partial + 'populate' => \$populate ) || die "\nSee --help for more\n"; # connect to osrf... @@ -36,10 +32,16 @@ my $es = OpenILS::Elastic::BibSearch->new($cluster); $es->connect; -$es->delete_index if $delete_index; +if ($delete_index) { + $es->delete_index or die "Index delete failed.\n"; +} -$es->create_index if $create_index; +if ($create_index) { + $es->create_index or die "Index create failed.\n"; +} -$es->populate_index if $populate; +if ($populate) { + $es->populate_index or die "Index populate failed.\n"; +} diff --git a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl index ffa5684a21..3c85937c3e 100755 --- a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl +++ b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl @@ -5,10 +5,9 @@ use Getopt::Long; use Time::HiRes qw/time/; use OpenSRF::Utils::JSON; use OpenILS::Utils::Fieldmapper; -use OpenILS::Elastic; +use OpenILS::Elastic::BibSearch; my $help; -my $elastic_config; my $osrf_config = '/openils/conf/opensrf_core.xml'; my $cluster = 'main'; my $index = 'bib-search'; @@ -16,7 +15,6 @@ my $query_string; GetOptions( 'help' => \$help, - 'elastic-config=s' => \$elastic_config, 'osrf-config=s' => \$osrf_config, 'cluster=s' => \$cluster, 'index=s' => \$index, @@ -98,14 +96,12 @@ my $query = { } }; -my $es = OpenILS::Utils::ElasticSearch->new( - config_file => $elastic_config -); +my $es = OpenILS::Elastic::BibSearch->new($cluster); -$es->connect($cluster); +$es->connect; my $start = time(); -my $results = $es->search($index, $query); +my $results = $es->search($query); my $duration = substr(time() - $start, 0, 6); print OpenSRF::Utils::JSON->perl2JSON($results) . "\n\n"; -- 2.11.0