reporter:label="Elastic Index">
<fields oils_persist:primary="id" oils_persist:sequence="elastic.index_id_seq">
<field reporter:label="Cluster" name="cluster" reporter:datatype="link"/>
- <field reporter:label="Code" name="code" reporter:datatype="text"/>
+ <field reporter:label="Index Class" name="index_class" reporter:datatype="text"/>
+ <field reporter:label="Index Name" name="name" reporter:datatype="text"/>
<field reporter:label="Number of Shards" name="num_shards" reporter:datatype="int"/>
<field reporter:label="Active" name="active" reporter:datatype="bool"/>
</fields>
$elastic_query->{size} = 1000;
}
- my $es = OpenILS::Elastic::BibSearch->new('main');
+ my $es = OpenILS::Elastic::BibSearch->new;
$es->connect;
my $results = $es->search($elastic_query);
use warnings;
use DBI;
use Time::HiRes qw/time/;
-use OpenSRF::Utils::Logger qw/:logger/;
-use OpenILS::Utils::CStoreEditor qw/:funcs/;
use Search::Elasticsearch;
use OpenSRF::Utils::JSON;
+use OpenSRF::Utils::Logger qw/:logger/;
+use OpenILS::Utils::CStoreEditor qw/:funcs/;
+use OpenILS::Utils::Fieldmapper;
use Data::Dumper;
$Data::Dumper::Indent = 0;
sub new {
- my ($class, $cluster) = @_;
+ my ($class, %args) = @_;
my $self = {
- cluster => $cluster,
- indices => [],
- marc_fields => []
+ %args,
+ indices => []
};
+ $self->{cluster} = 'main' unless $args{cluster};
+
return bless($self, $class);
}
}
sub index_name {
- die "Index name must be provided by sub-class\n";
+ my ($self) = @_;
+ return $self->{index_name};
+}
+
+sub index_class {
+ die "index_class() should be implemented by sub-classes\n";
+}
+
+# When write_mode is enable, it means we're editing indexes instead
+# of just searching them.
+sub write_mode {
+ my $self = shift;
+ return $self->{write_mode};
}
sub language_analyzers {
# load the config via cstore.
sub load_config {
- my $self = shift;
+ my $self = @_;
+
my $e = new_editor();
my $cluster = $self->cluster;
- $self->{nodes} = $e->search_elastic_node({cluster => $cluster, active => 't'});
+ my %active = $self->write_mode ? () : (active => 't');
+
+ $self->{nodes} = $e->search_elastic_node({cluster => $cluster, %active});
unless (@{$self->nodes}) {
$logger->error("ES no nodes defined for cluster $cluster");
return;
}
- $self->{indices} = $e->search_elastic_index({cluster => $cluster});
+ $self->{indices} = $e->search_elastic_index({cluster => $cluster, %active});
- unless (@{$self->indices}) {
- $logger->error("ES no indices defined for cluster $cluster");
+ unless ($self->write_mode || @{$self->indices}) {
+ $logger->warn("ES no active indices defined for cluster $cluster");
return;
}
}
+sub find_or_create_index_config {
+ my $self = shift;
+
+ my ($conf) = grep {
+ $_->name eq $self->index_name &&
+ $_->type eq $self->index_class
+ } @{$self->indices};
+
+ return $conf if $conf;
+
+ $logger->info("ES creating new index configuration for ".
+ sprintf("cluster=%s type=%s name=%s",
+ $self->cluster, $self->index_class, $self->index_name));
+
+ my $e = new_editor(xact => 1);
+ $conf = Fieldmapper::elastic::index->new;
+
+ $conf->cluster($self->cluster);
+ $conf->index_class($self->index_class);
+ $conf->index_name($self->index_name);
+
+ # Created by default with active=false and num_shards=1
+
+ unless ($e->create_elastic_index($conf)) {
+ $logger->error("ES failed creating index " .
+ $self->index_name . ": " . $e->die_event);
+ return undef;
+ }
+
+ $e->commit;
+
+ push(@{$self->indices}, $conf);
+
+ return $conf;
+}
+
sub connect {
my ($self) = @_;
+
$self->load_config;
my @nodes;
# default number of bibs to index per batch.
my $DEFAULT_BIB_BATCH_SIZE = 500;
-
-my $INDEX_NAME = 'bib-search';
+my $INDEX_CLASS = 'bib-search';
my $BASE_INDEX_SETTINGS = {
analysis => {
identifier => 'id'
);
-sub index_name {
- return $INDEX_NAME;
+sub index_class {
+ return $INDEX_CLASS;
}
# TODO: add index-specific language analyzers to DB config
sub create_index {
my ($self) = @_;
+ my $index_name = $self->index_name;
- if ($self->es->indices->exists(index => $INDEX_NAME)) {
- $logger->warn("ES index '$INDEX_NAME' already exists");
+ if ($self->es->indices->exists(index => $index_name)) {
+ $logger->warn("ES index '$index_name' already exists in ES");
return;
}
+ # Add a record of our new index to EG's DB if necessary.
+ my $eg_conf = $self->find_or_create_index_config;
+
$logger->info(
- "ES creating index '$INDEX_NAME' on cluster '".$self->cluster."'");
+ "ES creating index '$index_name' on cluster '".$self->cluster."'");
my $properties = $self->create_index_properties;
my $settings = $BASE_INDEX_SETTINGS;
$settings->{number_of_replicas} = scalar(@{$self->nodes});
- $settings->{number_of_shards} = $self->index->num_shards;
+ $settings->{number_of_shards} = $eg_conf->num_shards;
my $conf = {
- index => $INDEX_NAME,
+ index => $index_name,
body => {settings => $settings}
};
- $logger->info("ES creating index '$INDEX_NAME'");
+ $logger->info("ES creating index '$index_name'");
# Create the base index with settings
eval { $self->es->indices->create($conf) };
if ($@) {
$logger->error("ES failed to create index cluster=".
- $self->cluster. "index=$INDEX_NAME error=$@");
+ $self->cluster. "index=$index_name error=$@");
warn "$@\n\n";
return 0;
}
eval {
$self->es->indices->put_mapping({
- index => $INDEX_NAME,
+ index => $index_name,
type => 'record',
body => {dynamic => 'strict', properties => {$field => $properties->{$field}}}
});
my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties->{$field});
$logger->error("ES failed to create index mapping: " .
- "index=$INDEX_NAME field=$field error=$@ mapping=$mapjson");
+ "index=$index_name field=$field error=$@ mapping=$mapjson");
warn "$@\n\n";
return 0;
return $marc;
}
-
-
-sub index {
- my $self = shift;
- return $self->{index} if $self->{index};
- ($self->{index}) = grep {$_->code eq $self->index_name} @{$self->indices};
-
- $logger->error("No ndex configured named ".$self->index_name) unless $self->{index};
-
- return $self->{index};
-}
-
-
# Add data to the bib-search index
sub populate_index {
my ($self, $settings) = @_;
CREATE TABLE elastic.index (
id SERIAL PRIMARY KEY,
- code TEXT NOT NULL, -- e.g. 'bib-search'
+ name TEXT NOT NULL,
+ index_class TEXT NOT NULL,
cluster TEXT NOT NULL
REFERENCES elastic.cluster (code) ON DELETE CASCADE,
active BOOLEAN NOT NULL DEFAULT FALSE,
num_shards INTEGER NOT NULL DEFAULT 1,
- CONSTRAINT index_type_once_per_cluster UNIQUE (code, cluster)
+ CONSTRAINT active_index_once_per_cluster UNIQUE (active, index_class, cluster),
+ CONSTRAINT valid_index_class CHECK (index_class IN ('bib-search'))
);
CREATE OR REPLACE VIEW elastic.bib_field AS
INSERT INTO elastic.node (label, host, proto, port, active, cluster)
VALUES ('Localhost', 'localhost', 'http', 9200, TRUE, 'main');
-INSERT INTO elastic.index (code, active, cluster)
- VALUES ('bib-search', TRUE, 'main');
-
COMMIT;
/* UNDO
my $help;
my $osrf_config = '/openils/conf/opensrf_core.xml';
-my $cluster = 'main';
+my $cluster;
my $create_index;
my $delete_index;
-my $index_name = 'bib-search'; # only supported index at time of writing
+my $index_class = 'bib-search';
+my $index_name;
my $populate;
my $index_record;
my $start_record;
'cluster=s' => \$cluster,
'create-index' => \$create_index,
'delete-index' => \$delete_index,
- 'index=s' => \$index_name,
+ 'index-name=s' => \$index_name,
+ 'index-class=s' => \$index_class,
'index-record=s' => \$index_record,
'start-record=s' => \$start_record,
'stop-record=s' => \$stop_record,
print <<HELP;
Synopsis:
- $0 --delete-index --create-index --index bib-search --populate
+ $0 --delete-index --create-index --index-class bib-search \
+ --index-name bib-search-take-2 --populate
Options:
--cluster <name>
Specify a cluster name. Defaults to 'main'.
- --index <name>
- Specify an index name. Defaults to 'bib-search'.
+ --index-class <class>
+ Specifies which data set the current index manages (e.g. bib-search)
+
+ --index-name <name>
+ Specify an index name. Defaults to --index-class.
--delete-index
Delete the specified index and all of its data.
my $es;
-if ($index_name eq 'bib-search') {
- $es = OpenILS::Elastic::BibSearch->new($cluster);
+if ($index_class eq 'bib-search') {
+ $es = OpenILS::Elastic::BibSearch->new(
+ cluster => $cluster,
+ index_name => $index_name,
+ write_mode => 1
+ );
}
if (!$es) {
- die "Unknown index type: $index_name\n";
+ die "Unknown index class: $index_class\n";
}
$es->connect;
IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
OpenILS::Utils::CStoreEditor::init();
-my $es = OpenILS::Elastic::BibSearch->new($cluster);
+my $es = OpenILS::Elastic::BibSearch->new;
$es->connect;
print <<MESSAGE;
projects / working / Evergreen.git / commitdiff