From: Bill Erickson <berickxx@gmail.com>
Date: Fri, 26 Oct 2018 18:00:25 +0000 (-0400)
Subject: el conf idl fields, etc.
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=7f95f09d908f2a8b5ed7d24f8a948ead9a6de5f7;p=working%2FEvergreen.git

el conf idl fields, etc.

Signed-off-by: Bill Erickson <berickxx@gmail.com>
---

diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml
index 0a14cb5f14..1c75b2a185 100644
--- a/Open-ILS/examples/fm_IDL.xml
+++ b/Open-ILS/examples/fm_IDL.xml
@@ -12814,6 +12814,74 @@ SELECT  usr,
 		</links>
 	</class>
 
+	<!-- Elasticsearch Classes -->
+	<!-- TODO pcrud -->
+	<class id="ec" 
+		controller="open-ils.cstore"
+		oils_obj:fieldmapper="elastic::cluster"
+		oils_persist:tablename="elastic.cluster"
+		reporter:label="Elastic Cluster">
+		<fields oils_persist:primary="code">
+			<field reporter:label="Code" name="code" reporter:datatype="text" />
+			<field reporter:label="Label" name="label" reporter:datatype="text"/>
+			<field reporter:label="Nodes" name="nodes" reporter:datatype="link" oils_persist:virtual="true" />
+		</fields>
+		<links>
+			<link field="nodes" reltype="has_many" key="cluster" class="en"/>
+		</links>
+	</class>
+	<class id="en" 
+		controller="open-ils.cstore"
+		oils_obj:fieldmapper="elastic::node"
+		oils_persist:tablename="elastic.node"
+		reporter:label="Elastic Node">
+		<fields oils_persist:primary="id" oils_persist:sequence="elastic.node_id_seq">
+			<field reporter:label="Cluster" name="cluster" reporter:datatype="link"/>
+			<field reporter:label="Label" name="label" reporter:datatype="text"/>
+			<field reporter:label="Host" name="host" reporter:datatype="text"/>
+			<field reporter:label="Protocol" name="proto" reporter:datatype="text"/>
+			<field reporter:label="Port" name="port" reporter:datatype="int"/>
+			<field reporter:label="Active" name="active" reporter:datatype="bool"/>
+		</fields>
+		<links>
+			<link field="cluster" reltype="has_a" key="code" map="" class="ec"/>
+		</links>
+	</class>
+	<class id="ei" 
+		controller="open-ils.cstore"
+		oils_obj:fieldmapper="elastic::index"
+		oils_persist:tablename="elastic.index"
+		reporter:label="Elastic Index">
+		<fields oils_persist:primary="id" oils_persist:sequence="elastic.index_id_seq">
+			<field reporter:label="Cluster" name="cluster" reporter:datatype="link"/>
+			<field reporter:label="Code" name="code" reporter:datatype="text"/>
+			<field reporter:label="Number of Shards" name="num_shards" reporter:datatype="int"/>
+			<field reporter:label="Active" name="active" reporter:datatype="bool"/>
+		</fields>
+		<links>
+			<link field="cluster" reltype="has_a" key="code" map="" class="ec"/>
+		</links>
+	</class>
+	<class id="ebip" 
+		controller="open-ils.cstore"
+		oils_obj:fieldmapper="elastic::bib_field"
+		oils_persist:tablename="elastic.bib_field"
+		reporter:label="Elastic Bib Index field"
+		oils_persist:readonly="true">
+		<fields oils_persist:primary="id" oils_persist:sequence="elastic.index_id_seq">
+			<field reporter:label="Metabib Field" name="metabib_field" reporter:datatype="link"/>
+			<field reporter:label="Name" name="name" reporter:datatype="text"/>
+			<field reporter:label="Search Group" name="search_group" reporter:datatype="text"/>
+			<field reporter:label="Is Sort Field" name="sorter" reporter:datatype="bool"/>
+			<field reporter:label="Is Search Field" name="search_field" reporter:datatype="bool"/>
+			<field reporter:label="Is Facet Field" name="facet_field" reporter:datatype="bool"/>
+			<field reporter:label="Field Weight (Boost)" name="weight" reporter:datatype="int"/>
+		</fields>
+		<links>
+			<link field="metabib_field" reltype="has_a" key="id" map="" class="cmf"/>
+		</links>
+	</class>
+
 	<class id="cht" controller="open-ils.cstore open-ils.pcrud" oils_obj:fieldmapper="config::hold_type" oils_persist:tablename="config.hold_type" reporter:label="Hold Type" oils_persist:readonly="true">
     	<fields oils_persist:primary="id" oils_persist:sequence="config.hold_type_id_seq">
         	<field name="id" reporter:selector="name" reporter:datatype="id"/>
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
index 7e008823c3..dc4711ad18 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
@@ -40,6 +40,16 @@ sub cluster {
     return $self->{cluster};
 }
 
+sub nodes {
+    my $self = shift;
+    return $self->{nodes};
+}
+
+sub indices {
+    my $self = shift;
+    return $self->{indices};
+}
+
 sub es {
     my ($self) = @_;
     return $self->{es};
@@ -49,8 +59,9 @@ sub index_name {
     die "Index name must be provided by sub-class\n";
 }
 
-# Returns database connection object.
-# Connects if necessary.
+# Provide a direct DB connection so some high-volume activities,
+# like indexing bib records, can take advantage of a direct connection.
+# Returns database connection object -- connects if necessary.
 sub db {
 	my ($self) = @_;
 
@@ -89,24 +100,22 @@ sub get_db_rows {
     return $self->db->selectall_arrayref($sql, {Slice => {}});
 }
 
-# TODO: Add IDL entries for config.elastic* so we can
 # load the config via cstore.
 sub load_config {
     my $self = shift;
+    my $e = new_editor();
     my $cluster = $self->cluster;
 
-    $self->{nodes} = $self->get_db_rows(
-        "SELECT * FROM elastic.node WHERE cluster = '$cluster' AND active");
+    $self->{nodes} = $e->search_elastic_node({cluster => $cluster});
 
-    unless (@{$self->{nodes}}) {
+    unless (@{$self->nodes}) {
         $logger->error("ES no nodes defined for cluster $cluster");
         return;
     }
 
-    $self->{indices} = $self->get_db_rows(
-        "SELECT * FROM elastic.index WHERE cluster = '$cluster' AND active");
+    $self->{indices} = $e->search_elastic_index({cluster => $cluster});
 
-    unless (@{$self->{indices}}) {
+    unless (@{$self->indices}) {
         $logger->error("ES no indices defined for cluster $cluster");
         return;
     }
@@ -117,9 +126,9 @@ sub connect {
     $self->load_config;
 
     my @nodes;
-    for my $server (@{$self->{nodes}}) {
+    for my $server (@{$self->nodes}) {
         push(@nodes, sprintf("%s://%s:%d", 
-            $server->{proto}, $server->{host}, $server->{port}));
+            $server->proto, $server->host, $server->port));
     }
 
     $logger->info("ES connecting to nodes @nodes");
diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
index 6728233439..f910b6c2c6 100644
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
@@ -17,6 +17,7 @@ use strict;
 use warnings;
 use OpenSRF::Utils::Logger qw/:logger/;
 use OpenSRF::Utils::JSON;
+use OpenILS::Utils::CStoreEditor qw/:funcs/;
 use OpenILS::Elastic;
 use base qw/OpenILS::Elastic/;
 
@@ -109,7 +110,7 @@ sub index_name {
 sub index {
     my $self = shift;
     return $self->{index} if $self->{index};
-    ($self->{index}) = grep {$_->{code} eq $INDEX_NAME} @{$self->{indices}};
+    ($self->{index}) = grep {$_->code eq $INDEX_NAME} @{$self->indices};
     return $self->{index};
 }
 
@@ -126,17 +127,16 @@ sub create_index {
 
     my $mappings = $BASE_PROPERTIES;
 
-    my $fields = $self->get_db_rows(
-        'SELECT * FROM elastic.bib_index_properties');
+    my $fields = new_editor()->retrieve_all_elastic_bib_field();
 
     for my $field (@$fields) {
 
-        my $field_name = $field->{name};
-        my $search_group = $field->{search_group};
+        my $field_name = $field->name;
+        my $search_group = $field->search_group;
         $field_name = "$search_group|$field_name" if $search_group;
 
         my $def;
-        if ($field->{search_field}) {
+        if ($field->search_field eq 't') {
             # Search fields get full text indexing and analysis
 
             $def = {
@@ -147,7 +147,7 @@ sub create_index {
                 }
             };
 
-            if ($field->{facet_field} || $field->{sorter}) {
+            if ($field->facet_field eq 't' || $field->sorter eq 't') {
                 # If it's also a sort/facet field, add a keyword version
                 # of the field to use for sorting and aggregation
                 $def->{fields}{raw} = {type => 'keyword'};
@@ -167,7 +167,7 @@ sub create_index {
         }
 
         # Apply field boost.
-        $def->{boost} = $field->{weight} if ($field->{weight} || 1) > 1;
+        $def->{boost} = $field->weight if ($field->weight || 1) > 1;
 
         $logger->debug("ES adding field $field_name: ". 
             OpenSRF::Utils::JSON->perl2JSON($def));
@@ -176,8 +176,8 @@ sub create_index {
     }
 
     my $settings = $BASE_INDEX_SETTINGS;
-    $settings->{number_of_replicas} = scalar(@{$self->{nodes}});
-    $settings->{number_of_shards} = $self->index->{num_shards};
+    $settings->{number_of_replicas} = scalar(@{$self->nodes});
+    $settings->{number_of_shards} = $self->index->num_shards;
 
     my $conf = {
         index => $INDEX_NAME,
@@ -364,8 +364,6 @@ SQL
             status => $copy->{status},
             circ_lib => $copy->{circ_lib},
             location => $copy->{location},
-            #circulate => $copy->{circulate} eq 't' ? 'true' : 'false',
-            #opac_visbile => $copy->{opac_visible} eq 't' ? 'true' : 'false'
             circulate => $copy->{circulate} ? 'true' : 'false',
             opac_visbile => $copy->{opac_visible} ? 'true' : 'false'
         });
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
index 5125c5ae7b..97adf226c2 100644
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
@@ -33,14 +33,13 @@ CREATE TABLE elastic.index (
     CONSTRAINT    index_type_once_per_cluster UNIQUE (code, cluster)
 );
 
-CREATE OR REPLACE VIEW elastic.bib_index_properties AS
+CREATE OR REPLACE VIEW elastic.bib_field AS
     SELECT fields.* FROM (
         SELECT 
             NULL::INT AS metabib_field,
             crad.name,
             NULL AS search_group,
             crad.sorter,
-            crad.multi,
             FALSE AS search_field,
             FALSE AS facet_field,
             1 AS weight
@@ -52,7 +51,6 @@ CREATE OR REPLACE VIEW elastic.bib_index_properties AS
             cmf.name,
             cmf.field_class AS search_group,
             FALSE AS sorter,
-            TRUE AS multi,
             -- always treat identifier fields as non-search fields.
             (cmf.field_class <> 'identifier' AND cmf.search_field) AS search_field,
             cmf.facet_field,
@@ -62,7 +60,7 @@ CREATE OR REPLACE VIEW elastic.bib_index_properties AS
     ) fields;
 
 -- Note this could be done with a view, but pushing the bib ID
--- filter down to the base filter makes it a lot faster.
+-- filter down to the bottom of the query makes it a lot faster.
 CREATE OR REPLACE FUNCTION elastic.bib_record_properties(bre_id BIGINT) 
     RETURNS TABLE (
         search_group TEXT,
@@ -105,29 +103,38 @@ BEGIN
                 mfe.source, 
                 -- Index individual values instead of string-joined values
                 -- so they may be treated individually.  This is useful,
-                -- for example, when aggregating on individual subjects.
+                -- for example, when aggregating on subjects.
                 CASE WHEN cmf.joiner IS NOT NULL THEN
                     REGEXP_SPLIT_TO_TABLE(mfe.value, cmf.joiner)
                 ELSE
                     mfe.value
                 END AS value
             FROM (
-                SELECT * FROM metabib.title_field_entry UNION 
-                SELECT * FROM metabib.author_field_entry UNION
-                SELECT * FROM metabib.subject_field_entry UNION
-                SELECT * FROM metabib.series_field_entry UNION
-                SELECT * FROM metabib.keyword_field_entry UNION
-                SELECT * FROM metabib.identifier_field_entry
+                SELECT * FROM metabib.title_field_entry mtfe
+                    WHERE mtfe.source = $$ || QUOTE_LITERAL(bre_id) || $$ 
+                UNION 
+                SELECT * FROM metabib.author_field_entry mafe
+                    WHERE mafe.source = $$ || QUOTE_LITERAL(bre_id) || $$ 
+                UNION 
+                SELECT * FROM metabib.subject_field_entry msfe
+                    WHERE msfe.source = $$ || QUOTE_LITERAL(bre_id) || $$
+                UNION 
+                SELECT * FROM metabib.series_field_entry msrfe
+                    WHERE msrfe.source = $$ || QUOTE_LITERAL(bre_id) || $$ 
+                UNION 
+                SELECT * FROM metabib.keyword_field_entry mkfe
+                    WHERE mkfe.source = $$ || QUOTE_LITERAL(bre_id) || $$
+                UNION 
+                SELECT * FROM metabib.identifier_field_entry mife
+                    WHERE mife.source = $$ || QUOTE_LITERAL(bre_id) || $$
             ) mfe
             JOIN config.metabib_field cmf ON (cmf.id = mfe.field)
-            WHERE mfe.source = $$ || QUOTE_LITERAL(bre_id) || $$
-                AND (cmf.search_field OR cmf.facet_field)
+            WHERE (cmf.search_field OR cmf.facet_field)
         ) record
     $$;
 END $FUNK$ LANGUAGE PLPGSQL;
 
 
-
 /* SEED DATA ------------------------------------------------------------ */
 
 INSERT INTO elastic.cluster (code, label) VALUES ('main', 'Main Cluster');
diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl
index 27bbc81361..eeefe26048 100755
--- a/Open-ILS/src/support-scripts/elastic-index.pl
+++ b/Open-ILS/src/support-scripts/elastic-index.pl
@@ -3,6 +3,7 @@ use strict;
 use warnings;
 use Getopt::Long;
 use OpenILS::Utils::Fieldmapper;
+use OpenILS::Utils::CStoreEditor;
 use OpenILS::Elastic::BibSearch;
 
 my $help;
@@ -27,6 +28,7 @@ GetOptions(
 OpenSRF::System->bootstrap_client(config_file => $osrf_config);
 Fieldmapper->import(
     IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
+OpenILS::Utils::CStoreEditor::init();
 
 my $es = OpenILS::Elastic::BibSearch->new($cluster);