LP1844418 Direct indexing experiment WIP

Signed-off-by: Bill Erickson <berickxx@gmail.com>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
index 9462500..5f67427 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic.pm
@@ -367,7 +367,8 @@ sub search {
 # Avoid trying to index such data by lazily chopping it off
 # at 1/4 the limit to accomodate all UTF-8 chars.
 sub truncate_value {
-    my ($self, $value) = @_;
+    my ($self, $value, $length) = @_;
+    $length = 8190 unless $length;
     return substr($value, 0, 8190);
 }
 

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
index 5bcb5ae..7143bda 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch.pm
@@ -1,6 +1,5 @@
-package OpenILS::Elastic::BibSearch;
 # ---------------------------------------------------------------
-# Copyright (C) 2019 King County Library System
+# Copyright (C) 2019-2020 King County Library System
 # Author: Bill Erickson <berickxx@gmail.com>
 #
 # This program is free software; you can redistribute it and/or
@@ -13,9 +12,48 @@ package OpenILS::Elastic::BibSearch;
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR code.  See the
 # GNU General Public License for more details.
 # ---------------------------------------------------------------
+package OpenILS::Elastic::BibSearch::BibField;
+# Models a single indexable field.
+use strict;
+use warnings;
+
+sub new {
+    my ($class, %args) = @_;
+    return bless(\%args, $class);
+}
+sub name {
+    my $self = shift;
+    return $self->{name};
+}
+sub field_class {
+    my $self = shift;
+    return $self->{field_class};
+}
+sub search_field {
+    my $self = shift;
+    return $self->{purpose} eq 'search';
+}
+sub facet_field {
+    my $self = shift;
+    return $self->{purpose} eq 'facet';
+}
+sub sorter {
+    my $self = shift;
+    return $self->{purpose} eq 'sorter';
+}
+sub filter {
+    my $self = shift;
+    return $self->{purpose} eq 'filter';
+}
+sub weight {
+    my $self = shift;
+    return $self->{weight} || 1;
+}
+
+# ---------------------------------------------------------------
+package OpenILS::Elastic::BibSearch;
 use strict;
 use warnings;
-use Encode;
 use DateTime;
 use Clone 'clone';
 use Time::HiRes qw/time/;
@@ -32,11 +70,17 @@ my $DEFAULT_BIB_BATCH_SIZE = 500;
 my $INDEX_CLASS = 'bib-search';
 
 # https://www.elastic.co/guide/en/elasticsearch/reference/current/ignore-above.html
-# Useful for ignoring excessively long filters, sorters, and facets.
+# Useful for ignoring excessively long filters and facets.
 # Only applied to the keyword variation of each index.  Does not affect
-# the 'text' varieties.
+# the 'text' varieties. The selected limit is arbitrary.
 my $IGNORE_ABOVE = 256;
 
+# Individual characters of some values like sorters provide less and less
+# value as the length of the text gets longer and longer.  Unlike
+# $IGNORE_ABOVE, this only trims the string, it does not prevent it from
+# getting indexed in the first place.  The selected limit is arbitrary.
+my $TRIM_ABOVE = 512;
+
 my $BASE_INDEX_SETTINGS = {
     analysis => {
         analyzer => {
@@ -188,17 +232,6 @@ my $BASE_PROPERTIES = {
         ignore_above => $IGNORE_ABOVE,
         normalizer => 'custom_lowercase',
     },
-
-    # Create some shortcut indexes for streamlining query_string searches.
-    ti => {type => 'text'},
-    au => {type => 'text'},
-    se => {type => 'text'},
-    su => {type => 'text'},
-    kw => {type => 'text'},
-    id => {
-        type => 'keyword',
-        ignore_above => $IGNORE_ABOVE
-    }
 };
 
 my %SHORT_GROUP_MAP = (
@@ -214,20 +247,144 @@ sub index_class {
     return $INDEX_CLASS;
 }
 
-# TODO: add index-specific language analyzers to DB config
+# TODO: determine when/how to apply language analyzers.
+# e.g. create lang-specific index fields?
 sub language_analyzers {
     return ("english");
 }
 
+sub xsl_file {
+    my ($self, $filename) = @_;
+    $self->{xsl_file} = $filename if $filename;
+    return $self->{xsl_file};
+}
+
+sub xsl_doc {
+    my ($self) = @_;
+
+    $self->{xsl_doc} = XML::LibXML->load_xml(location => $self->xsl_file)
+        unless $self->{xsl_doc};
+
+    return $self->{xsl_doc};
+}
+
+sub xsl_sheet {
+    my $self = shift;
+
+    $self->{xsl_sheet} = XML::LibXSLT->new->parse_stylesheet($self->xsl_doc)
+        unless $self->{xsl_sheet};
+
+    return $self->{xsl_sheet};
+}
+
+my @seen_fields;
+sub add_dynamic_field {
+    my ($self, $fields, $purpose, $field_class, $name, $weight) = @_;
+    return unless $name;
+
+    $weight = '' if !$weight || $weight eq '_';
+    $field_class = '' if !$field_class || $field_class eq '_';
+
+    my $tag = $purpose . $field_class . $name;
+    return if grep {$_ eq $tag} @seen_fields;
+    push(@seen_fields, $tag);
+
+    $logger->info("ES adding dynamic field purpose=$purpose ".
+        "field_class=$field_class name=$name weight=$weight");
+
+    my $field = OpenILS::Elastic::BibSearch::BibField->new(
+        purpose => $purpose, 
+        field_class => $field_class, 
+        name => $name,
+        weight => $weight
+    );
+
+    push(@$fields, $field);
+}
+
 sub get_dynamic_fields {
     my $self = shift;
+    my $fields = [];
+
+    @seen_fields = (); # reset with each run
 
-    # elastic.bib_field has no primary key field, so retrieve_all won't work.
-    # Note the name value may be repeated across search group depending
-    # on local configuration.
-    return new_editor()->search_elastic_bib_field({name => {'!=' => undef}});
+    my $null_doc = XML::LibXML->load_xml(string => '<root/>');
+    my $result = $self->xsl_sheet->transform($null_doc, target => '"index-fields"');
+    my $output = $self->xsl_sheet->output_as_chars($result);
+
+    my @rows = split(/\n/, $output);
+    for my $row (@rows) {
+        my @parts = split(/ /, $row);
+        $self->add_dynamic_field($fields, @parts);
+    }
+
+    return $fields;
 }
 
+sub get_bib_data {
+    my ($self, $record_ids) = @_;
+
+    my $bib_data = [];
+    my $db_data = $self->get_bib_db_data($record_ids);
+
+    for my $db_rec (@$db_data) {
+
+        if ($db_rec->{deleted} == 1) {
+            # No need to extract index values.
+            push(@$bib_data, {deleted => 1});
+            next;
+        }
+
+        my $marc_doc = XML::LibXML->load_xml(string => $db_rec->{marc});
+        my $result = $self->xsl_sheet->transform($marc_doc, target => '"index-values"');
+        my $output = $self->xsl_sheet->output_as_chars($result);
+
+        my @rows = split(/\n/, $output);
+        for my $row (@rows) {
+            my ($purpose, $field_class, $name, @tokens) = split(/ /, $row);
+
+            $field_class = '' if ($field_class || '') eq '_';
+
+            my $value = join(' ', @tokens);
+
+            my $field = {
+                purpose => $purpose,
+                field_class => $field_class,
+                name => $name,
+                value => $value
+            };
+
+            # Stamp each field with the additional bib metadata.
+            $field->{$_} = $db_rec->{$_} for 
+                qw/id bib_source metarecord create_date edit_date deleted/;
+
+            push(@$bib_data, $field);
+        }
+    }
+
+    return $bib_data;
+}
+
+sub get_bib_db_data {
+    my ($self, $record_ids) = @_;
+
+    my $ids_str = join(',', @$record_ids);
+
+    my $sql = <<SQL;
+SELECT DISTINCT ON (bre.id)
+    bre.id, 
+    bre.create_date, 
+    bre.edit_date, 
+    bre.source AS bib_source,
+    bre.deleted,
+    bre.marc
+FROM biblio.record_entry bre
+LEFT JOIN metabib.metarecord_source_map mmrsm ON (mmrsm.source = bre.id)
+WHERE bre.id IN ($ids_str)
+SQL
+
+    return $self->get_db_rows($sql);
+}
 
 sub create_index_properties {
     my ($self) = @_;
@@ -252,25 +409,21 @@ sub create_index_properties {
 
     my $fields = $self->get_dynamic_fields;
 
-    $logger->info('ES ' . OpenSRF::Utils::JSON->perl2JSON($fields));
-
     for my $field (@$fields) {
-        
 
         my $field_name = $field->name;
         my $field_class = $field->field_class;
         $field_name = "$field_class|$field_name" if $field_class;
 
-        $logger->info("ES ONE FIELD name=$field_name: " . OpenSRF::Utils::JSON->perl2JSON($field));
-
         my $def;
 
         if ($field_class) {
-            if ($field->search_field eq 't') {
+            if ($field->search_field) {
 
                 # Use the same fields and analysis as the 'grouped' field.
                 $def = clone($properties->{$field_class});
-                $def->{copy_to} = [$field_class, $SHORT_GROUP_MAP{$field_class}];
+                # Copy grouped fields into their group parent field.
+                $def->{copy_to} = $field_class;
 
                 # Apply ranking boost to each analysis variation.
                 my $flds = $def->{fields};
@@ -284,9 +437,12 @@ sub create_index_properties {
 
             $def = {
                 type => 'keyword',
-                ignore_above => $IGNORE_ABOVE,
                 normalizer => 'custom_lowercase'
             };
+
+            # Long sorter values are not necessarily unexpected,
+            # e.g. long titles.
+            $def->{ignore_above} = $IGNORE_ABOVE unless $field->sorter;
         }
 
         if ($def) {
@@ -299,7 +455,7 @@ sub create_index_properties {
         # Search and facet fields can have the same name/group pair,
         # but are stored as separate fields in ES since the content
         # may vary between the two.
-        if ($field->facet_field eq 't') {
+        if ($field->facet_field) {
 
             # Facet fields are stored as separate fields, because their
             # content may differ from the matching search field.
@@ -360,58 +516,48 @@ sub create_index {
 
     # Create each mapping one at a time instead of en masse so we 
     # can more easily report when mapping creation fails.
-
     for my $field (keys %$properties) {
-        $logger->info("ES Creating index mapping for field $field");
-
-        eval { 
-            $self->es->indices->put_mapping({
-                index => $index_name,
-                type  => 'record',
-                body  => {
-                    dynamic => 'strict', 
-                    properties => {$field => $properties->{$field}}
-                }
-            });
-        };
-
-        if ($@) {
-            my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties->{$field});
-
-            $logger->error("ES failed to create index mapping: " .
-                "index=$index_name field=$field error=$@ mapping=$mapjson");
+        return 0 unless 
+            $self->create_one_field_index($field, $properties->{$field});
+    }
 
-            warn "$@\n\n";
-            return 0;
-        }
+    # Now that we've added the static (and dynamic) fields,
+    # add the shortened field_class aliases.
+    while (my ($field, $alias) = each %SHORT_GROUP_MAP) {
+        return 0 unless $self->create_one_field_index(
+            $alias, {type => 'alias', path => $field});
     }
 
     return 1;
 }
 
-# TODO: elastic.bib_record_properties needs to also pull values
-# from metabib.facet_entry
-# TODO: stamp each field with a 'purpose' (search, facet, filter, sorter)
-sub get_bib_data {
-    my ($self, $record_ids) = @_;
+sub create_one_field_index {
+    my ($self, $field, $properties) = @_;
+    my $index_name = $self->index_name;
+    $logger->info("ES Creating index mapping for field $field");
+
+    eval { 
+        $self->es->indices->put_mapping({
+            index => $index_name,
+            type  => 'record',
+            body  => {
+                dynamic => 'strict', 
+                properties => {$field => $properties}
+            }
+        });
+    };
 
-    my $ids_str = join(',', @$record_ids);
+    if ($@) {
+        my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties);
 
-    my $sql = <<SQL;
-SELECT DISTINCT ON (bre.id, field_class, name, value)
-    bre.id, 
-    bre.create_date, 
-    bre.edit_date, 
-    bre.source AS bib_source,
-    bre.deleted,
-    mmrsm.metarecord,
-    (elastic.bib_record_properties(bre.id)).*
-FROM biblio.record_entry bre
-LEFT JOIN metabib.metarecord_source_map mmrsm ON (mmrsm.source = bre.id)
-WHERE bre.id IN ($ids_str)
-SQL
+        $logger->error("ES failed to create index mapping: " .
+            "index=$index_name field=$field error=$@ mapping=$mapjson");
 
-    return $self->get_db_rows($sql);
+        warn "$@\n\n";
+        return 0;
+    }
+
+    return 1;
 }
 
 sub populate_bib_index_batch {
@@ -480,7 +626,9 @@ sub populate_bib_index_batch {
 
             $fname = "$fclass|$fname" if $fclass;
             $fname = "$fname|facet" if $field->{purpose} eq 'facet';
-            $value = $self->truncate_value($value);
+
+            my $trim = $field->{purpose} eq 'sorter' ? $TRIM_ABOVE : undef;
+            $value = $self->truncate_value($value, $trim);
 
             if ($fname eq 'identifier|isbn') {
                 index_isbns($body, $value);

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm b/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm
deleted file mode 100644 (file)
index 6178936..0000000
--- a/Open-ILS/src/perlmods/lib/OpenILS/Elastic/BibSearch/XSLT.pm
+++ /dev/null
@@ -1,199 +0,0 @@
-# ---------------------------------------------------------------
-# Copyright (C) 2020 King County Library System
-# Author: Bill Erickson <berickxx@gmail.com>
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 2
-# of the License, or (at your option) any later version.
-
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR code.  See the
-# GNU General Public License for more details.
-# ---------------------------------------------------------------
-package OpenILS::Elastic::BibSearch::BibField;
-# Helper class for modeling an elastic bib field.
-# This is what OpenILS::Elastic::BibSearch expects.
-
-sub new {
-    my ($class, %args) = @_;
-    return bless(\%args, $class);
-}
-sub name {
-    my $self = shift;
-    return $self->{name};
-}
-sub field_class {
-    my $self = shift;
-    return $self->{field_class};
-}
-sub search_field {
-    my $self = shift;
-    return $self->{purpose} eq 'search' ? 't' : 'f';
-}
-sub facet_field {
-    my $self = shift;
-    return $self->{purpose} eq 'facet' ? 't' : 'f';
-}
-sub sorter {
-    my $self = shift;
-    return $self->{purpose} eq 'sorter' ? 't' : 'f';
-}
-sub filter {
-    my $self = shift;
-    return $self->{purpose} eq 'filter' ? 't' : 'f';
-}
-sub weight {
-    my $self = shift;
-    return $self->{weight} || 1;
-}
-
-package OpenILS::Elastic::BibSearch::XSLT;
-use strict;
-use warnings;
-use XML::LibXML;
-use XML::LibXSLT;
-use OpenSRF::Utils::Logger qw/:logger/;
-use OpenILS::Utils::CStoreEditor qw/:funcs/;
-use OpenILS::Elastic::BibSearch;
-use base qw/OpenILS::Elastic::BibSearch/;
-
-
-sub xsl_file {
-    my ($self, $filename) = @_;
-    $self->{xsl_file} = $filename if $filename;
-    return $self->{xsl_file};
-}
-
-sub xsl_doc {
-    my ($self) = @_;
-
-    $self->{xsl_doc} = XML::LibXML->load_xml(location => $self->xsl_file)
-        unless $self->{xsl_doc};
-
-    return $self->{xsl_doc};
-}
-
-sub xsl_sheet {
-    my $self = shift;
-
-    $self->{xsl_sheet} = XML::LibXSLT->new->parse_stylesheet($self->xsl_doc)
-        unless $self->{xsl_sheet};
-
-    return $self->{xsl_sheet};
-}
-
-
-my @seen_fields;
-sub add_dynamic_field {
-    my ($self, $fields, $purpose, $field_class, $name, $weight) = @_;
-    return unless $name;
-
-    $weight = '' if !$weight || $weight eq '_';
-    $field_class = '' if !$field_class || $field_class eq '_';
-
-    my $tag = $purpose . $field_class . $name;
-    return if grep {$_ eq $tag} @seen_fields;
-    push(@seen_fields, $tag);
-
-    $logger->info("ES adding dynamic field purpose=$purpose ".
-        "field_class=$field_class name=$name weight=$weight");
-
-    my $field = OpenILS::Elastic::BibSearch::BibField->new(
-        purpose => $purpose, 
-        field_class => $field_class, 
-        name => $name,
-        weight => $weight
-    );
-
-    push(@$fields, $field);
-}
-
-sub get_dynamic_fields {
-    my $self = shift;
-    my $fields = [];
-
-    @seen_fields = (); # reset with each run
-
-    my $null_doc = XML::LibXML->load_xml(string => '<root/>');
-    my $result = $self->xsl_sheet->transform($null_doc, target => '"index-fields"');
-    my $output = $self->xsl_sheet->output_as_chars($result);
-
-    my @rows = split(/\n/, $output);
-    for my $row (@rows) {
-        my @parts = split(/ /, $row);
-        $self->add_dynamic_field($fields, @parts);
-    }
-
-    return $fields;
-}
-
-sub get_bib_data {
-    my ($self, $record_ids) = @_;
-
-    my $bib_data = [];
-    my $db_data = $self->get_bib_db_data($record_ids);
-
-    for my $db_rec (@$db_data) {
-
-        if ($db_rec->{deleted} == 1) {
-            # No need to extract index values.
-            push(@$bib_data, {deleted => 1});
-            next;
-        }
-
-        my $marc_doc = XML::LibXML->load_xml(string => $db_rec->{marc});
-        my $result = $self->xsl_sheet->transform($marc_doc, target => '"index-values"');
-        my $output = $self->xsl_sheet->output_as_chars($result);
-
-        my @rows = split(/\n/, $output);
-        for my $row (@rows) {
-            my ($purpose, $field_class, $name, @tokens) = split(/ /, $row);
-
-            $field_class = '' if ($field_class || '') eq '_';
-
-            my $value = join(' ', @tokens);
-
-            my $field = {
-                purpose => $purpose,
-                field_class => $field_class,
-                name => $name,
-                value => $value
-            };
-
-            # Stamp each field with the additional bib metadata.
-            $field->{$_} = $db_rec->{$_} for 
-                qw/id bib_source metarecord create_date edit_date deleted/;
-
-            push(@$bib_data, $field);
-        }
-    }
-
-    return $bib_data;
-}
-
-sub get_bib_db_data {
-    my ($self, $record_ids) = @_;
-
-    my $ids_str = join(',', @$record_ids);
-
-    my $sql = <<SQL;
-SELECT DISTINCT ON (bre.id)
-    bre.id, 
-    bre.create_date, 
-    bre.edit_date, 
-    bre.source AS bib_source,
-    bre.deleted,
-    bre.marc
-FROM biblio.record_entry bre
-LEFT JOIN metabib.metarecord_source_map mmrsm ON (mmrsm.source = bre.id)
-WHERE bre.id IN ($ids_str)
-SQL
-
-    return $self->get_db_rows($sql);
-}
-
-
-1;
-

diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
index 193002a..077101f 100644 (file)
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.elastic-search.sql
@@ -3,30 +3,10 @@ DROP SCHEMA IF EXISTS elastic CASCADE;
 
 BEGIN;
 
-ALTER TABLE config.record_attr_definition
-    ADD COLUMN elastic_field BOOLEAN NOT NULL DEFAULT FALSE;
-
-ALTER TABLE config.metabib_field
-    ADD COLUMN elastic_field BOOLEAN NOT NULL DEFAULT FALSE;
-
--- Provide a sweeping set of default elastic fields.
--- Likely this set of fields can be trimmed significantly for most sites,
--- since many of these fields will never be searched from the catalog.
--- Reducing the number of elastic_field's will improve indexing time, 
--- search time, and reduce Elastic disk space requirements.
-UPDATE config.record_attr_definition 
-    SET elastic_field = TRUE WHERE name NOT LIKE 'marc21_%';
-
-UPDATE config.metabib_field 
-    SET elastic_field = TRUE WHERE search_field OR facet_field;
-
 INSERT INTO config.global_flag (name, enabled, label) 
 VALUES (
     'elastic.bib_search.enabled', FALSE,
     'Elasticsearch Enable Bib Searching'
-), (
-    'elastic.bib_search.dynamic_properties', FALSE,
-    'Elasticsearch Dynamic Bib Record Properties'
 );
 
 CREATE SCHEMA elastic;
@@ -61,170 +41,6 @@ CREATE TABLE elastic.index (
     CONSTRAINT    valid_index_class CHECK (index_class IN ('bib-search'))
 );
 
-CREATE OR REPLACE VIEW elastic.bib_field AS
-    SELECT fields.* FROM (
-        SELECT 
-            NULL::INT AS metabib_field,
-            crad.name,
-            crad.label,
-            NULL AS field_class,
-            crad.sorter,
-            FALSE AS search_field,
-            FALSE AS facet_field,
-            1 AS weight
-        FROM config.record_attr_definition crad
-        WHERE crad.elastic_field
-        UNION
-        SELECT 
-            cmf.id AS metabib_field,
-            cmf.name,
-            cmf.label,
-            cmf.field_class,
-            FALSE AS sorter,
-            -- always treat identifier fields as non-search fields.
-            (cmf.field_class <> 'identifier' AND cmf.search_field) AS search_field,
-            cmf.facet_field,
-            cmf.weight
-        FROM config.metabib_field cmf 
-        WHERE cmf.elastic_field
-    ) fields;
-
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_attrs(bre_id BIGINT)
-RETURNS TABLE (
-    field_class TEXT,
-    name TEXT,
-    source BIGINT,
-    value TEXT
-)
-AS $FUNK$
-    SELECT DISTINCT record.* FROM (
-        SELECT 
-            NULL::TEXT AS field_class, 
-            crad.name, 
-            mrs.source, 
-            mrs.value
-        FROM metabib.record_sorter mrs
-        JOIN config.record_attr_definition crad ON (crad.name = mrs.attr)
-        WHERE mrs.source = $1 AND crad.elastic_field
-        UNION
-
-        -- record attributes
-        SELECT 
-            NULL::TEXT AS field_class, 
-            crad.name, 
-            mraf.id AS source, 
-            mraf.value
-        FROM metabib.record_attr_flat mraf
-        JOIN config.record_attr_definition crad ON (crad.name = mraf.attr)
-        WHERE mraf.id = $1 AND crad.elastic_field
-    ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_static_props(bre_id BIGINT)
-RETURNS TABLE (
-    field_class TEXT,
-    name TEXT,
-    source BIGINT,
-    value TEXT
-)
-AS $FUNK$
-    SELECT DISTINCT record.* FROM (
-        SELECT
-            cmf.field_class,
-            cmf.name, 
-            props.source, 
-            CASE WHEN cmf.joiner IS NOT NULL THEN
-                REGEXP_SPLIT_TO_TABLE(props.value, cmf.joiner)
-            ELSE
-                props.value
-            END AS value
-        FROM (
-            SELECT field, source, value 
-                FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.author_field_entry mafe WHERE mafe.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.subject_field_entry msfe WHERE msfe.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.identifier_field_entry mife WHERE mife.source = $1
-            UNION 
-            SELECT field, source, value 
-                FROM metabib.facet_entry mfe WHERE mfe.source = $1
-        ) props
-        JOIN config.metabib_field cmf ON (cmf.id = props.field)
-        WHERE cmf.elastic_field
-    ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_dynamic_props(bre_id BIGINT)
-RETURNS TABLE (
-    field_class TEXT,
-    name TEXT,
-    source BIGINT,
-    value TEXT
-)
-AS $FUNK$
-    SELECT DISTINCT record.* FROM (
-        SELECT
-            cmf.field_class,
-            cmf.name, 
-            props.source, 
-            CASE WHEN cmf.joiner IS NOT NULL THEN
-                REGEXP_SPLIT_TO_TABLE(props.value, cmf.joiner)
-            ELSE
-                props.value
-            END AS value
-        FROM biblio.extract_metabib_field_entry(
-            $1, ' ', '{facet,search}',
-            (SELECT ARRAY_AGG(id) FROM config.metabib_field WHERE elastic_field)
-        ) props
-        JOIN config.metabib_field cmf ON (cmf.id = props.field)
-    ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_properties(bre_id BIGINT) 
-    RETURNS TABLE (
-        field_class TEXT,
-        name TEXT,
-        source BIGINT,
-        value TEXT
-    )
-    AS $FUNK$
-DECLARE
-    props_func TEXT;
-BEGIN
-
-    PERFORM 1 FROM config.internal_flag cif WHERE 
-        cif.name = 'elastic.bib_search.dynamic_properties' AND cif.enabled;
-
-    IF FOUND THEN
-        props_func := 'elastic.bib_record_dynamic_props';
-    ELSE
-        props_func := 'elastic.bib_record_static_props';
-    END IF;
-
-    RETURN QUERY EXECUTE $$
-        SELECT DISTINCT record.* FROM (
-            SELECT * FROM elastic.bib_record_attrs($$ || QUOTE_LITERAL(bre_id) || $$)
-            UNION
-            SELECT * FROM $$ || props_func || '(' || QUOTE_LITERAL(bre_id) || $$)
-        ) record
-    $$;
-END $FUNK$ LANGUAGE PLPGSQL;
-        
-/* give me bibs I should upate */
-
 CREATE OR REPLACE VIEW elastic.bib_last_mod_date AS
     /**
      * Last update date for each bib, which is taken from most recent
@@ -262,43 +78,12 @@ COMMIT;
 
 DROP SCHEMA IF EXISTS elastic CASCADE;
 
-ALTER TABLE config.record_attr_definition DROP COLUMN elastic_field;
-
-ALTER TABLE config.metabib_field DROP COLUMN elastic_field;
-
 DELETE FROM config.global_flag WHERE name ~ 'elastic.*';
 
 */
 
 /*
 
--- Sample narrower set of elastic fields to avoid duplication and 
--- indexing data that will presumably never be searched in the catalog.
-
-UPDATE config.metabib_field SET elastic_field = FALSE
-WHERE 
-    (field_class = 'keyword' AND name <> 'keyword') OR
-    (field_class = 'subject' AND name = 'complete') OR
-    (field_class = 'author'  AND name = 'first_author')
-;
-
-UPDATE config.record_attr_definition SET elastic_field = FALSE
-WHERE name NOT IN (
-    'authorsort',
-    'date1',
-    'date2',
-    'bib_level',
-    'item_form',
-    'item_lang',
-    'item_type',
-    'lit_form',
-    'pubdate',
-    'search_format',
-    'titlesort',
-    'sr_format',
-    'vr_format'
-);
-
 -- Bill's elastic VM for testing.
 UPDATE elastic.node 
     SET host = 'elastic.gamma', port = 80, path = '/elastic/node1' 

diff --git a/Open-ILS/src/support-scripts/elastic-index.pl b/Open-ILS/src/support-scripts/elastic-index.pl
index a03ea9b..a728fe4 100755 (executable)
--- a/Open-ILS/src/support-scripts/elastic-index.pl
+++ b/Open-ILS/src/support-scripts/elastic-index.pl
@@ -10,6 +10,7 @@ use OpenILS::Elastic::BibSearch::XSLT;
 
 my $help;
 my $osrf_config = '/openils/conf/opensrf_core.xml';
+my $bib_transform = '/openils/var/xsl/elastic-bib-transform.xsl';
 my $cluster;
 my $create_index;
 my $delete_index;
@@ -22,7 +23,6 @@ my $start_record;
 my $stop_record;
 my $modified_since;
 my $max_duration;
-my $bib_transform;
 my $batch_size = 500;
 
 # Database settings read from ENV by default.
@@ -144,22 +144,12 @@ OpenILS::Utils::CStoreEditor::init();
 my $es;
 
 if ($index_class eq 'bib-search') {
-
-    if ($bib_transform) {
-        $es = OpenILS::Elastic::BibSearch::XSLT->new(
-            cluster => $cluster, 
-            index_name => $index_name,
-            write_mode => 1,
-            xsl_file => $bib_transform
-        );
-    } else {
-
-        $es = OpenILS::Elastic::BibSearch->new(
-            cluster => $cluster, 
-            index_name => $index_name,
-            write_mode => 1
-        );
-    }
+    $es = OpenILS::Elastic::BibSearch->new(
+        cluster => $cluster, 
+        index_name => $index_name,
+        xsl_file => $bib_transform,
+        write_mode => 1
+    );
 }
 
 if (!$es) {

diff --git a/Open-ILS/xsl/elastic-bib-transform.xsl b/Open-ILS/xsl/elastic-bib-transform.xsl
new file mode 100644 (file)
index 0000000..8e6b4b3
--- /dev/null
+++ b/Open-ILS/xsl/elastic-bib-transform.xsl
@@ -0,0 +1,1278 @@
+<xsl:stylesheet
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:marc="http://www.loc.gov/MARC21/slim"
+  version="1.0">
+  <xsl:output encoding="UTF-8" method="text"/>
+
+  <!-- 
+      Transform operates in one of two modes:
+
+      1. target == 'index-fields'
+
+      Prints one index definition per line without any record-specific
+      data, In this mode, any valid XML string/file (e.g. '<root />')
+      may be used for the transform.
+
+      Output:
+
+      $index_purpose $index_class $index_name $index_weight
+
+     - Fields that have no value should use '_' as the value.
+
+      e.g.
+
+      search title proper 5
+
+      2. target == 'index-values'
+
+      Prints one index value per line for data found by transforming
+      a MARCXML record.
+
+      Output:
+
+      $index_purpose $index_class $index_name $value
+
+      - $value is the only string in the output that may contain spaces.
+
+      e.g.
+
+      search subject topic South America
+      facet author personal Janey Jam "Jojo" Jones
+  -->
+
+  <xsl:template match="@*|node()">
+    <xsl:call-template name="compile_searches" />
+    <xsl:call-template name="compile_facets" />
+    <xsl:call-template name="compile_filters" />
+    <xsl:call-template name="compile_sorters" />
+  </xsl:template>
+
+  <xsl:template name="compile_searches">
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">650</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">topic</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">651</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">geographic</xsl:with-param>
+      <xsl:with-param name="index_subfields">avxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">655</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">genre</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">630</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">uniftitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">adfgklmnoprstvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">600</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">name</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdfgjklmnopqrstuvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">610</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">corpname</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdfgklmnoprstuvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">611</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">meeting</xsl:with-param>
+      <xsl:with-param name="index_subfields">acdefgjklnpqstuvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">490</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">800</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">tflmnoprs</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">810</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">tflmnoprs</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">830</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">adfgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">100</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">personal</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">110</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">111</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">meeting</xsl:with-param>
+      <xsl:with-param name="index_subfields">acdegng</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">700</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">added_personal</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">710</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="index_subfields">ab</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">711</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">meeting</xsl:with-param>
+      <xsl:with-param name="index_subfields">acde</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">400</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">added_personal</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcd</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcd</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">411</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">meeting</xsl:with-param>
+      <xsl:with-param name="index_subfields">acdegq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">010</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">lccn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">010</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">lccn</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">020</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">isbn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">020</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">isbn</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">022</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">issn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">022</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">issn</xsl:with-param>
+      <xsl:with-param name="index_subfields">y</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">022</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">issn</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">024</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">upc</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">024</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">upc</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">027</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">tech_number</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">027</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">tech_number</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">028</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">tech_number</xsl:with-param>
+      <xsl:with-param name="index_subfields">ab</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">074</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">sudoc</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">074</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">sudoc</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">086</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">sudoc</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">086</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">sudoc</xsl:with-param>
+      <xsl:with-param name="index_subfields">z</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">092</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">bibcn</xsl:with-param>
+      <xsl:with-param name="index_subfields">ab</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">099</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">bibcn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">099</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">bibcn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">130</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">uniform</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">210</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">abbreviated</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">222</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">magazine</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">240</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">uniform</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">245</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">maintitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+      <xsl:with-param name="weight">10</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">245</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">proper</xsl:with-param>
+      <xsl:with-param name="index_subfields">abefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">245</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">responsibility</xsl:with-param>
+      <xsl:with-param name="index_subfields">c</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">246</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">alternative</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgjklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">247</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">former</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">260</xsl:with-param>
+      <xsl:with-param name="field_class">keyword</xsl:with-param>
+      <xsl:with-param name="index_name">publisher</xsl:with-param>
+      <xsl:with-param name="index_subfields">b</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">264</xsl:with-param>
+      <xsl:with-param name="field_class">keyword</xsl:with-param>
+      <xsl:with-param name="index_name">publisher</xsl:with-param>
+      <xsl:with-param name="index_subfields">b</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">400</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcde</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">411</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">conference</xsl:with-param>
+      <xsl:with-param name="index_subfields">acdegq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">411</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">440</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">490</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">490</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">uniform</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">694</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">700</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">added</xsl:with-param>
+      <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">710</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">added</xsl:with-param>
+      <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">711</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">added</xsl:with-param>
+      <xsl:with-param name="index_subfields">fklnpst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">730</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">added</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">740</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">added</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">780</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">previous</xsl:with-param>
+      <xsl:with-param name="index_subfields">st</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">785</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">succeeding</xsl:with-param>
+      <xsl:with-param name="index_subfields">st</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">800</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">personal_series</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">800</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">810</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate_series</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">810</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">811</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">conference_series</xsl:with-param>
+      <xsl:with-param name="index_subfields">acdegnq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">811</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">fklnpstv</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">830</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">938</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">match_isbn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_search_entry">
+      <xsl:with-param name="tag">938</xsl:with-param>
+      <xsl:with-param name="field_class">identifier</xsl:with-param>
+      <xsl:with-param name="index_name">match_isbn</xsl:with-param>
+      <xsl:with-param name="index_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="keyword_full_entry" />
+  </xsl:template>
+
+  <xsl:template name="compile_facets">
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">650</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">topic</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcdvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">651</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">geographic</xsl:with-param>
+      <xsl:with-param name="facet_subfields">avxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">600</xsl:with-param>
+      <xsl:with-param name="field_class">subject</xsl:with-param>
+      <xsl:with-param name="index_name">name</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcdfgjklmnopqrstuvxyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">490</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">a</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">800</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">tflmnoprs</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">810</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">tflmnoprs</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">830</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">adfgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">100</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">personal</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcdq</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">110</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="facet_subfields">ab</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">710</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="facet_subfields">ab</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcd</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">400</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">corporate</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">410</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">411</xsl:with-param>
+      <xsl:with-param name="field_class">author</xsl:with-param>
+      <xsl:with-param name="index_name">conference</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">411</xsl:with-param>
+      <xsl:with-param name="field_class">title</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">440</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">490</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">694</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields"></xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">800</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">fgklmnoprst</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">810</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcdn</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">811</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">fklnpstv</xsl:with-param>
+    </xsl:call-template>
+    <xsl:call-template name="add_facet_entry">
+      <xsl:with-param name="tag">830</xsl:with-param>
+      <xsl:with-param name="field_class">series</xsl:with-param>
+      <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+      <xsl:with-param name="facet_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+    </xsl:call-template>
+  </xsl:template>
+
+  <xsl:template name="compile_sorters">
+    
+    <!-- author sort is the first 1XX value -->
+    <xsl:for-each select="marc:datafield[starts-with(@tag, '1')]">
+      <xsl:sort select="@tag"/>
+      <xsl:if test="position() = 1">
+        <xsl:call-template name="add_sorter_entry">
+          <xsl:with-param name="name">author</xsl:with-param>
+          <xsl:with-param name="value">
+            <xsl:call-template name="subfieldSelect"></xsl:call-template>
+          </xsl:with-param>
+        </xsl:call-template>
+      </xsl:if>
+    </xsl:for-each>
+
+    <!-- title sort is the 245a non-filing -->
+    <xsl:for-each select="marc:datafield[@tag='245']">
+      <xsl:variable name="full_title">
+        <xsl:call-template name="subfieldSelect">
+          <xsl:with-param name="codes">a</xsl:with-param>
+        </xsl:call-template>
+      </xsl:variable>
+      <xsl:variable name="offset">
+        <xsl:choose>
+          <xsl:when test="number(@ind2) = @ind2">
+            <xsl:value-of select="@ind2" />
+          </xsl:when>
+          <xsl:otherwise>
+            <xsl:text>0</xsl:text>
+          </xsl:otherwise>
+        </xsl:choose>
+      </xsl:variable>
+      <xsl:call-template name="add_sorter_entry">
+        <xsl:with-param name="name">title</xsl:with-param>
+        <xsl:with-param name="value" select="substring($full_title, $offset + 1)" />
+      </xsl:call-template>
+    </xsl:for-each>
+
+    <!-- pubdate is the same as the date1 filter -->
+    <xsl:call-template name="add_sorter_entry">
+      <xsl:with-param name="name">pubdate</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">7</xsl:with-param>
+          <xsl:with-param name="length">4</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+    </xsl:call-template>
+
+  </xsl:template>
+
+  <xsl:template name="compile_filters">
+  
+    <!-- start with filters that are not used within composite filters.
+         These can be added to the document inline. -->
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">date1</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">7</xsl:with-param>
+          <xsl:with-param name="length">4</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+      <xsl:with-param name="default_value">0000</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">date2</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">11</xsl:with-param>
+          <xsl:with-param name="length">4</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+      <xsl:with-param name="default_value">9999</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">lit_form</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">33</xsl:with-param>
+          <xsl:with-param name="length">1</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">item_lang</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">35</xsl:with-param>
+          <xsl:with-param name="length">3</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">audience</xsl:with-param>
+      <xsl:with-param name="value">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">008</xsl:with-param>
+          <xsl:with-param name="offset">22</xsl:with-param>
+          <xsl:with-param name="length">1</xsl:with-param>
+        </xsl:call-template>
+      </xsl:with-param>
+    </xsl:call-template>
+
+    <!-- Filters that may be used within composite filters are 
+         stored in a local variable so they can first be added
+         to the document, then used to compile composite filters -->
+
+    <xsl:variable name="item_type">
+      <xsl:call-template name="leader_value">
+        <xsl:with-param name="offset">6</xsl:with-param>
+        <xsl:with-param name="length">1</xsl:with-param>
+      </xsl:call-template>
+    </xsl:variable>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">item_type</xsl:with-param>
+      <xsl:with-param name="value" select="$item_type" />
+    </xsl:call-template>
+
+    <xsl:variable name="bib_level">
+      <xsl:call-template name="leader_value">
+        <xsl:with-param name="offset">7</xsl:with-param>
+        <xsl:with-param name="length">1</xsl:with-param>
+      </xsl:call-template>
+    </xsl:variable>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">bib_level</xsl:with-param>
+      <xsl:with-param name="value" select="$bib_level" />
+    </xsl:call-template>
+
+    <xsl:variable name="item_form">
+      <xsl:call-template name="controlfield_value">
+        <xsl:with-param name="tag">008</xsl:with-param>
+        <xsl:with-param name="offset">23</xsl:with-param>
+        <xsl:with-param name="length">1</xsl:with-param>
+      </xsl:call-template>
+    </xsl:variable>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">item_form</xsl:with-param>
+      <xsl:with-param name="value" select="$item_form" />
+    </xsl:call-template>
+
+    <xsl:variable name="category_of_material">
+      <xsl:call-template name="controlfield_value">
+        <xsl:with-param name="tag">007</xsl:with-param>
+        <xsl:with-param name="offset">0</xsl:with-param>
+        <xsl:with-param name="length">1</xsl:with-param>
+      </xsl:call-template>
+    </xsl:variable>
+
+    <xsl:variable name="vr_format">
+      <xsl:if test="$category_of_material = 'v'">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">007</xsl:with-param>
+          <xsl:with-param name="offset">4</xsl:with-param>
+          <xsl:with-param name="length">1</xsl:with-param>
+        </xsl:call-template>
+      </xsl:if>
+    </xsl:variable>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">vr_format</xsl:with-param>
+      <xsl:with-param name="value" select="$vr_format" />
+    </xsl:call-template>
+    
+    <xsl:variable name="sr_format">
+      <xsl:if test="$category_of_material = 's'">
+        <xsl:call-template name="controlfield_value">
+          <xsl:with-param name="tag">007</xsl:with-param>
+          <xsl:with-param name="offset">3</xsl:with-param>
+          <xsl:with-param name="length">1</xsl:with-param>
+        </xsl:call-template>
+      </xsl:if>
+    </xsl:variable>
+
+    <xsl:call-template name="add_filter_entry">
+      <xsl:with-param name="name">sr_format</xsl:with-param>
+      <xsl:with-param name="value" select="$sr_format" />
+    </xsl:call-template>
+    
+    <!-- use the extracted raw filters to create composite filters -->
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">blu-ray</xsl:with-param>
+      <xsl:with-param name="vr_format" select="$vr_format" />
+      <xsl:with-param name="vr_format_codes">s</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">book</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">at</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_type" />
+      <xsl:with-param name="item_form_not_codes">abcfoqrs</xsl:with-param>
+      <xsl:with-param name="bib_level" select="$bib_level" />
+      <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">braille</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">a</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">f</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">casaudiobook</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">i</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">l</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">casmusic</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">j</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">l</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">cdaudiobook</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">i</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">f</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">cdaudiobook</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">j</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">f</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">dvd</xsl:with-param>
+      <xsl:with-param name="vr_format" select="$vr_format" />
+      <xsl:with-param name="vr_format_codes">v</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">eaudio</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">i</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">ebook</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">at</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+      <xsl:with-param name="bib_level" select="$bib_level" />
+      <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">electronic</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">os</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">equip</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">r</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">evideo</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">g</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">kit</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">op</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">lpbook</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">at</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">d</xsl:with-param>
+      <xsl:with-param name="bib_level" select="$bib_level" />
+      <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">map</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">ef</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">microform</xsl:with-param>
+      <xsl:with-param name="item_form" select="$item_form" />
+      <xsl:with-param name="item_form_codes">abc</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">music</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">j</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">phonomusic</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">j</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">abcde</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">phonospoken</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">i</xsl:with-param>
+      <xsl:with-param name="sr_format" select="$sr_format" />
+      <xsl:with-param name="sr_format_codes">abcde</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">picture</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">k</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">serial</xsl:with-param>
+      <xsl:with-param name="bib_level" select="$bib_level" />
+      <xsl:with-param name="bib_level_codes">bs</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">score</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">cd</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">software</xsl:with-param>
+      <xsl:with-param name="item_type" select="$item_type" />
+      <xsl:with-param name="item_type_codes">m</xsl:with-param>
+    </xsl:call-template>
+
+    <xsl:call-template name="add_composite_filter_entry">
+      <xsl:with-param name="name">search_format</xsl:with-param>
+      <xsl:with-param name="value">vhs</xsl:with-param>
+      <xsl:with-param name="vr_format" select="$vr_format" />
+      <xsl:with-param name="vr_format_codes">b</xsl:with-param>
+    </xsl:call-template>
+
+  </xsl:template>
+
+  <xsl:template name="add_sorter_entry">
+    <xsl:param name="name"/>
+    <xsl:param name="value"/>
+    <xsl:text>sorter _ </xsl:text>
+    <xsl:value-of select="$name" />
+    <xsl:if test="$target = 'index-fields'">
+      <xsl:text> _ </xsl:text><!-- weight -->
+    </xsl:if>
+    <xsl:if test="$target = 'index-values'">
+      <xsl:text> </xsl:text>
+      <xsl:value-of select="$value" />
+    </xsl:if>
+    <xsl:text>&#xa;</xsl:text>
+  </xsl:template>
+
+  <xsl:template name="add_filter_entry">
+    <xsl:param name="name"/>
+    <xsl:param name="value"/>
+    <xsl:param name="default_value"/>
+    <xsl:text>filter _ </xsl:text>
+    <xsl:value-of select="$name" />
+    <xsl:if test="$target = 'index-fields'">
+      <xsl:text> _</xsl:text><!-- weight -->
+    </xsl:if>
+    <xsl:if test="$target = 'index-values'">
+      <xsl:text> </xsl:text>
+      <xsl:choose>
+        <xsl:when test="$default_value and translate($value, ' ', '') = ''">
+          <xsl:value-of select="$default_value" />
+        </xsl:when>
+        <xsl:otherwise>
+          <xsl:value-of select="$value" />
+        </xsl:otherwise>
+      </xsl:choose>
+    </xsl:if>
+    <xsl:text>&#xa;</xsl:text>
+  </xsl:template>
+
+  <xsl:template name="add_composite_filter_entry">
+    <xsl:param name="name"/>
+    <xsl:param name="value"/>
+    <xsl:param name="item_type"/>
+    <xsl:param name="item_type_codes"/>
+    <xsl:param name="item_form"/>
+    <xsl:param name="item_form_codes"/>
+    <xsl:param name="item_form_not_codes"/>
+    <xsl:param name="bib_level"/>
+    <xsl:param name="bib_level_codes"/>
+    <xsl:param name="vr_format"/>
+    <xsl:param name="vr_format_codes"/>
+    <xsl:param name="sr_format"/>
+    <xsl:param name="sr_format_codes"/>
+
+    <xsl:variable name="item_type_matches" select="
+      not($item_type_codes) or (  
+        $item_type != '' and
+        contains($item_type_codes, $item_type)
+      )  
+    "/>
+
+    <xsl:variable name="item_form_matches" select="
+      (
+        not($item_form_codes) or 
+        contains($item_form_codes, $item_form)
+      ) and (
+        not($item_form_not_codes) or
+        not(contains($item_form_not_codes, $item_form))
+      )
+    "/>
+
+    <xsl:variable name="bib_level_matches" select="
+      not($bib_level_codes) or (
+        $bib_level != '' and
+        contains($bib_level_codes, $bib_level)
+      )
+    "/>
+
+    <xsl:variable name="vr_format_matches" select="
+      not($vr_format_codes) or (
+        $vr_format != '' and 
+        contains($vr_format_codes, $vr_format)
+      )
+    "/>
+
+    <xsl:variable name="sr_format_matches" select="
+      not($sr_format_codes) or (
+        $sr_format != '' and
+        contains($sr_format_codes, $sr_format)
+      )
+    "/>
+
+    <xsl:if test="
+        $target = 'index-fields' or (
+        $item_type_matches and 
+        $item_form_matches and 
+        $bib_level_matches and
+        $sr_format_matches and
+        $vr_format_matches
+      )">
+      <xsl:call-template name="add_filter_entry">
+        <xsl:with-param name="name" select="$name" />
+        <xsl:with-param name="value" select="$value" />
+      </xsl:call-template>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template name="leader_value">
+    <xsl:param name="offset" /> <!-- zero-based -->
+    <xsl:param name="length" />
+    <xsl:for-each select="marc:leader">
+      <xsl:value-of select="substring(text(), $offset + 1, $length)"/>
+    </xsl:for-each>
+  </xsl:template>
+
+  <xsl:template name="controlfield_value">
+    <xsl:param name="tag" />
+    <xsl:param name="offset" /> <!-- zero-based -->
+    <xsl:param name="length" />
+    <xsl:for-each select="marc:controlfield[@tag=$tag]">
+      <xsl:value-of select="substring(text(), $offset + 1, $length)"/>
+    </xsl:for-each>
+  </xsl:template>
+
+  <!-- Produces a single value for the specific tab/subfields.
+       Should only be used in cases where a single value is expected. -->
+  <xsl:template name="datafield_value">
+    <xsl:param name="tag" />
+    <xsl:param name="subfields" />
+    <xsl:for-each select="marc:datafield[@tag=$tag]">
+      <xsl:call-template name="subfieldSelect">
+        <xsl:with-param name="codes">
+          <xsl:value-of select="$subfields" />
+        </xsl:with-param>
+      </xsl:call-template>
+      <xsl:text> </xsl:text>
+    </xsl:for-each>
+  </xsl:template>
+
+  <xsl:template name="subfieldSelect">
+    <xsl:param name="codes">abcdefghijklmnopqrstuvwxyz</xsl:param>
+    <xsl:param name="delimeter">
+      <xsl:text> </xsl:text>
+    </xsl:param>
+    <xsl:variable name="str">
+      <xsl:for-each select="marc:subfield">
+        <xsl:if test="contains($codes, @code)">
+          <xsl:value-of select="text()"/>
+          <xsl:value-of select="$delimeter"/>
+        </xsl:if>
+      </xsl:for-each>
+    </xsl:variable>
+    <xsl:value-of select="substring($str,1,string-length($str)-string-length($delimeter))"/>
+  </xsl:template>
+
+  <xsl:template name="add_search_entry">
+    <xsl:param name="tag" />
+    <xsl:param name="field_class" />
+    <xsl:param name="index_name" />
+    <xsl:param name="index_subfields" />
+    <xsl:param name="weight" />
+    <xsl:if test="$target = 'index-fields'">
+      <xsl:text>search </xsl:text>
+      <xsl:value-of select="$field_class" /><xsl:text> </xsl:text>
+      <xsl:value-of select="$index_name" /><xsl:text> </xsl:text>
+      <xsl:value-of select="$weight" />
+      <xsl:text>&#xa;</xsl:text><!-- newline -->
+    </xsl:if>
+    <xsl:if test="$target = 'index-values'">
+      <xsl:for-each select="marc:datafield[@tag=$tag] |
+        marc:datafield[@tag='880']/marc:subfield[@code='6'][starts-with(., $tag)]/..">
+        <xsl:text>search </xsl:text>
+        <xsl:value-of select="$field_class" /><xsl:text> </xsl:text>
+        <xsl:value-of select="$index_name" /><xsl:text> </xsl:text>
+        <xsl:call-template name="subfieldSelect">
+          <xsl:with-param name="codes">
+            <xsl:value-of select="$index_subfields" />
+          </xsl:with-param>
+        </xsl:call-template>
+        <xsl:text>&#xa;</xsl:text><!-- newline -->
+      </xsl:for-each>
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template name="add_facet_entry">
+    <xsl:param name="tag" />
+    <xsl:param name="field_class" />
+    <xsl:param name="index_name" />
+    <xsl:param name="facet_subfields" />
+    <xsl:if test="$target = 'index-fields'">
+      <xsl:text>facet </xsl:text>
+      <xsl:value-of select="$field_class"/>
+      <xsl:text> </xsl:text>
+      <xsl:value-of select="$index_name"/>
+      <xsl:text> _</xsl:text><!-- weight -->
+      <xsl:text>&#xa;</xsl:text><!-- newline -->
+    </xsl:if>
+    <xsl:if test="$target = 'index-values'">
+      <xsl:for-each select="marc:datafield[@tag=$tag] |
+        marc:datafield[@tag='880']/marc:subfield[@code='6'][starts-with(., $tag)]/..">
+        <xsl:text>facet </xsl:text>
+        <xsl:value-of select="$field_class"/><xsl:text> </xsl:text>
+        <xsl:value-of select="$index_name"/><xsl:text> </xsl:text>
+        <xsl:call-template name="subfieldSelect">
+          <xsl:with-param name="codes">
+            <xsl:value-of select="$facet_subfields" />
+          </xsl:with-param>
+        </xsl:call-template>
+        <xsl:text>&#xa;</xsl:text><!-- newline -->
+      </xsl:for-each>
+    </xsl:if>
+  </xsl:template>
+
+  <!-- Dumps practically the entire document into a single 
+       keyword|keyword index.
+  -->
+  <xsl:template name="keyword_full_entry">
+    <xsl:text>search keyword keyword </xsl:text>
+    <xsl:if test="$target = 'index-fields'">
+      <xsl:text>_</xsl:text><!-- weight -->
+    </xsl:if>
+    <xsl:if test="$target = 'index-values'">
+      <xsl:for-each select="marc:datafield">
+        <xsl:call-template name="subfieldSelect" />
+        <xsl:text> </xsl:text>
+      </xsl:for-each>
+    </xsl:if>
+    <xsl:text>&#xa;</xsl:text><!-- newline -->
+  </xsl:template>
+
+</xsl:stylesheet>
+
+