From: Bill Erickson Date: Mon, 3 Feb 2020 22:12:44 +0000 (-0500) Subject: LP1844418 Decouple index class from index name X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=01c150ffec6c3fcfc66bc5a56876fa5995504bee;p=working%2FEvergreen.git LP1844418 Decouple index class from index name Allow for the creation of multiple indices for a given index class so the user can toggle between them, e.g. to switch to newly built index. Signed-off-by: Bill Erickson --- diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index 5dd07627bb..b5e19c315d 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -12971,7 +12971,8 @@ SELECT usr, reporter:label="Elastic Index"> - + + diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm index 3fde569d09..25799443c1 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/Search/Elastic.pm @@ -208,7 +208,7 @@ sub bib_search { $elastic_query->{size} = 1000; } - my $es = OpenILS::Elastic::BibSearch->new('main'); + my $es = OpenILS::Elastic::BibSearch->new; $es->connect; my $results = $es->search($elastic_query); diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm index 6ad35962ca..dfd501a7e3 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm @@ -17,22 +17,24 @@ use strict; use warnings; use DBI; use Time::HiRes qw/time/; -use OpenSRF::Utils::Logger qw/:logger/; -use OpenILS::Utils::CStoreEditor qw/:funcs/; use Search::Elasticsearch; use OpenSRF::Utils::JSON; +use OpenSRF::Utils::Logger qw/:logger/; +use OpenILS::Utils::CStoreEditor qw/:funcs/; +use OpenILS::Utils::Fieldmapper; use Data::Dumper; $Data::Dumper::Indent = 0; sub new { - my ($class, $cluster) = @_; + my ($class, %args) = @_; my $self = { - cluster => $cluster, - indices => [], - marc_fields => [] + %args, + indices => [] }; + $self->{cluster} = 'main' unless $args{cluster}; + return bless($self, $class); } @@ -57,7 +59,19 @@ sub es { } sub index_name { - die "Index name must be provided by sub-class\n"; + my ($self) = @_; + return $self->{index_name}; +} + +sub index_class { + die "index_class() should be implemented by sub-classes\n"; +} + +# When write_mode is enable, it means we're editing indexes instead +# of just searching them. +sub write_mode { + my $self = shift; + return $self->{write_mode}; } sub language_analyzers { @@ -106,27 +120,67 @@ sub get_db_rows { # load the config via cstore. sub load_config { - my $self = shift; + my $self = @_; + my $e = new_editor(); my $cluster = $self->cluster; - $self->{nodes} = $e->search_elastic_node({cluster => $cluster, active => 't'}); + my %active = $self->write_mode ? () : (active => 't'); + + $self->{nodes} = $e->search_elastic_node({cluster => $cluster, %active}); unless (@{$self->nodes}) { $logger->error("ES no nodes defined for cluster $cluster"); return; } - $self->{indices} = $e->search_elastic_index({cluster => $cluster}); + $self->{indices} = $e->search_elastic_index({cluster => $cluster, %active}); - unless (@{$self->indices}) { - $logger->error("ES no indices defined for cluster $cluster"); + unless ($self->write_mode || @{$self->indices}) { + $logger->warn("ES no active indices defined for cluster $cluster"); return; } } +sub find_or_create_index_config { + my $self = shift; + + my ($conf) = grep { + $_->name eq $self->index_name && + $_->type eq $self->index_class + } @{$self->indices}; + + return $conf if $conf; + + $logger->info("ES creating new index configuration for ". + sprintf("cluster=%s type=%s name=%s", + $self->cluster, $self->index_class, $self->index_name)); + + my $e = new_editor(xact => 1); + $conf = Fieldmapper::elastic::index->new; + + $conf->cluster($self->cluster); + $conf->index_class($self->index_class); + $conf->index_name($self->index_name); + + # Created by default with active=false and num_shards=1 + + unless ($e->create_elastic_index($conf)) { + $logger->error("ES failed creating index " . + $self->index_name . ": " . $e->die_event); + return undef; + } + + $e->commit; + + push(@{$self->indices}, $conf); + + return $conf; +} + sub connect { my ($self) = @_; + $self->load_config; my @nodes; diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm index 1d5340d343..8384baf086 100644 --- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm +++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm @@ -30,8 +30,7 @@ use base qw/OpenILS::Elastic/; # default number of bibs to index per batch. my $DEFAULT_BIB_BATCH_SIZE = 500; - -my $INDEX_NAME = 'bib-search'; +my $INDEX_CLASS = 'bib-search'; my $BASE_INDEX_SETTINGS = { analysis => { @@ -206,8 +205,8 @@ my %SHORT_GROUP_MAP = ( identifier => 'id' ); -sub index_name { - return $INDEX_NAME; +sub index_class { + return $INDEX_CLASS; } # TODO: add index-specific language analyzers to DB config @@ -293,34 +292,38 @@ sub create_index_properties { sub create_index { my ($self) = @_; + my $index_name = $self->index_name; - if ($self->es->indices->exists(index => $INDEX_NAME)) { - $logger->warn("ES index '$INDEX_NAME' already exists"); + if ($self->es->indices->exists(index => $index_name)) { + $logger->warn("ES index '$index_name' already exists in ES"); return; } + # Add a record of our new index to EG's DB if necessary. + my $eg_conf = $self->find_or_create_index_config; + $logger->info( - "ES creating index '$INDEX_NAME' on cluster '".$self->cluster."'"); + "ES creating index '$index_name' on cluster '".$self->cluster."'"); my $properties = $self->create_index_properties; my $settings = $BASE_INDEX_SETTINGS; $settings->{number_of_replicas} = scalar(@{$self->nodes}); - $settings->{number_of_shards} = $self->index->num_shards; + $settings->{number_of_shards} = $eg_conf->num_shards; my $conf = { - index => $INDEX_NAME, + index => $index_name, body => {settings => $settings} }; - $logger->info("ES creating index '$INDEX_NAME'"); + $logger->info("ES creating index '$index_name'"); # Create the base index with settings eval { $self->es->indices->create($conf) }; if ($@) { $logger->error("ES failed to create index cluster=". - $self->cluster. "index=$INDEX_NAME error=$@"); + $self->cluster. "index=$index_name error=$@"); warn "$@\n\n"; return 0; } @@ -333,7 +336,7 @@ sub create_index { eval { $self->es->indices->put_mapping({ - index => $INDEX_NAME, + index => $index_name, type => 'record', body => {dynamic => 'strict', properties => {$field => $properties->{$field}}} }); @@ -343,7 +346,7 @@ sub create_index { my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties->{$field}); $logger->error("ES failed to create index mapping: " . - "index=$INDEX_NAME field=$field error=$@ mapping=$mapjson"); + "index=$index_name field=$field error=$@ mapping=$mapjson"); warn "$@\n\n"; return 0; @@ -642,19 +645,6 @@ SQL return $marc; } - - -sub index { - my $self = shift; - return $self->{index} if $self->{index}; - ($self->{index}) = grep {$_->code eq $self->index_name} @{$self->indices}; - - $logger->error("No ndex configured named ".$self->index_name) unless $self->{index}; - - return $self->{index}; -} - - # Add data to the bib-search index sub populate_index { my ($self, $settings) = @_; diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql index b492659c1c..356ecd9d1a 100644 --- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql @@ -51,12 +51,14 @@ CREATE TABLE elastic.node ( CREATE TABLE elastic.index ( id SERIAL PRIMARY KEY, - code TEXT NOT NULL, -- e.g. 'bib-search' + name TEXT NOT NULL, + index_class TEXT NOT NULL, cluster TEXT NOT NULL REFERENCES elastic.cluster (code) ON DELETE CASCADE, active BOOLEAN NOT NULL DEFAULT FALSE, num_shards INTEGER NOT NULL DEFAULT 1, - CONSTRAINT index_type_once_per_cluster UNIQUE (code, cluster) + CONSTRAINT active_index_once_per_cluster UNIQUE (active, index_class, cluster), + CONSTRAINT valid_index_class CHECK (index_class IN ('bib-search')) ); CREATE OR REPLACE VIEW elastic.bib_field AS @@ -245,9 +247,6 @@ INSERT INTO elastic.cluster (code, label) INSERT INTO elastic.node (label, host, proto, port, active, cluster) VALUES ('Localhost', 'localhost', 'http', 9200, TRUE, 'main'); -INSERT INTO elastic.index (code, active, cluster) - VALUES ('bib-search', TRUE, 'main'); - COMMIT; /* UNDO diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl index 05ebc6b9dc..e2c9753dff 100755 --- a/Open-ILS/src/support-scripts/elastic-index.pl +++ b/Open-ILS/src/support-scripts/elastic-index.pl @@ -9,10 +9,11 @@ use OpenILS::Elastic::BibSearch; my $help; my $osrf_config = '/openils/conf/opensrf_core.xml'; -my $cluster = 'main'; +my $cluster; my $create_index; my $delete_index; -my $index_name = 'bib-search'; # only supported index at time of writing +my $index_class = 'bib-search'; +my $index_name; my $populate; my $index_record; my $start_record; @@ -35,7 +36,8 @@ GetOptions( 'cluster=s' => \$cluster, 'create-index' => \$create_index, 'delete-index' => \$delete_index, - 'index=s' => \$index_name, + 'index-name=s' => \$index_name, + 'index-class=s' => \$index_class, 'index-record=s' => \$index_record, 'start-record=s' => \$start_record, 'stop-record=s' => \$stop_record, @@ -55,7 +57,8 @@ sub help { print < Specify a cluster name. Defaults to 'main'. - --index - Specify an index name. Defaults to 'bib-search'. + --index-class + Specifies which data set the current index manages (e.g. bib-search) + + --index-name + Specify an index name. Defaults to --index-class. --delete-index Delete the specified index and all of its data. @@ -126,12 +132,16 @@ OpenILS::Utils::CStoreEditor::init(); my $es; -if ($index_name eq 'bib-search') { - $es = OpenILS::Elastic::BibSearch->new($cluster); +if ($index_class eq 'bib-search') { + $es = OpenILS::Elastic::BibSearch->new( + cluster => $cluster, + index_name => $index_name, + write_mode => 1 + ); } if (!$es) { - die "Unknown index type: $index_name\n"; + die "Unknown index class: $index_class\n"; } $es->connect; diff --git a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl index 9ff4ef9559..6882434375 100755 --- a/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl +++ b/Open-ILS/src/support-scripts/test-scripts/elastic-search.pl @@ -46,7 +46,7 @@ Fieldmapper->import( IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL")); OpenILS::Utils::CStoreEditor::init(); -my $es = OpenILS::Elastic::BibSearch->new($cluster); +my $es = OpenILS::Elastic::BibSearch->new; $es->connect; print <