LP#1251394 Display field schema sync'ing

Copy flat/compressed/wide display entry views into base schema.
Synchronize a few remaining seed data bits.

Replace references to 'topic_subject' in wide display views with the new
'creators' field.

Remove ZZZZ.UNDO testing SQL file.

Signed-off-by: Bill Erickson <berickxx@gmail.com>
Signed-off-by: Mike Rylander <mrylander@gmail.com>

diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml
index 4d944a8..92a8e71 100644 (file)
--- a/Open-ILS/examples/fm_IDL.xml
+++ b/Open-ILS/examples/fm_IDL.xml
@@ -3807,7 +3807,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
                        <field name="title" reporter:label="Title" reporter:datatype="text"/>
                        <field name="author" reporter:label="Author" reporter:datatype="text"/>
                        <field name="subject" reporter:label="Subject" reporter:datatype="text"/>
-                       <field name="topic_subject" reporter:label="Topic Subject" reporter:datatype="text"/>
+                       <field name="creators" reporter:label="Creators" reporter:datatype="text"/>
                        <field name="isbn" reporter:label="ISBN" reporter:datatype="text"/>
                        <!-- TODO add all well-known fields -->
                </fields>

diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql
index e8f3f24..ff27a68 100644 (file)
--- a/Open-ILS/src/sql/Pg/002.schema.config.sql
+++ b/Open-ILS/src/sql/Pg/002.schema.config.sql
@@ -48,9 +48,11 @@ INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_browse_indexing');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_search_indexing');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_facet_indexing');
+INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_display_indexing');
 INSERT INTO config.internal_flag (name) VALUES ('serial.rematerialize_on_same_holding_code');
 INSERT INTO config.internal_flag (name) VALUES ('ingest.metarecord_mapping.preserve_on_delete');
 
+
 CREATE TABLE config.global_flag (
     label   TEXT    NOT NULL
 ) INHERITS (config.internal_flag);
@@ -226,6 +228,12 @@ $$;
 
 CREATE UNIQUE INDEX config_metabib_field_class_name_idx ON config.metabib_field (field_class, name);
 
+CREATE TABLE config.display_field_map (
+    name    TEXT   PRIMARY KEY,
+    field   INTEGER REFERENCES config.metabib_field (id),
+    multi   BOOLEAN DEFAULT FALSE
+);
+
 CREATE TABLE config.ts_config_list (
        id                      TEXT PRIMARY KEY,
        name            TEXT NOT NULL

diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index 0240622..f55905d 100644 (file)
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -195,6 +195,63 @@ CREATE INDEX metabib_display_entry_field_idx
 CREATE INDEX metabib_display_entry_source_idx 
     ON metabib.display_entry (source);
 
+CREATE VIEW metabib.flat_display_entry AS
+    /* One row per display entry fleshed with field info */
+    SELECT
+        mde.source,
+        cdfm.name,
+        cdfm.multi,
+        cmf.label,
+        cmf.id AS field,
+        mde.value
+    FROM metabib.display_entry mde
+    JOIN config.metabib_field cmf ON (cmf.id = mde.field)
+    JOIN config.display_field_map cdfm ON (cdfm.field = mde.field)
+;
+
+CREATE VIEW metabib.compressed_display_entry AS
+/* Like flat_display_entry except values are compressed into 
+   one row per display_field_map and JSON-ified.  */
+    SELECT 
+        source,
+        name,
+        multi,
+        label,
+        field,
+        CASE WHEN multi THEN
+            TO_JSON(ARRAY_AGG(value))
+        ELSE
+            TO_JSON(MIN(value))
+        END AS value
+    FROM metabib.flat_display_entry
+    GROUP BY 1, 2, 3, 4, 5
+;
+
+CREATE VIEW metabib.wide_display_entry AS
+/* Table-like view of well-known display fields.   
+   This VIEW expands as well-known display fields are added. */
+    SELECT 
+        bre.id AS source,
+        COALESCE(mcde_title.value, 'null') AS title,
+        COALESCE(mcde_author.value, 'null') AS author,
+        COALESCE(mcde_subject.value, 'null') AS subject,
+        COALESCE(mcde_creators.value, 'null') AS creators,
+        COALESCE(mcde_isbn.value, 'null') AS isbn
+    -- ensure one row per bre regardless of the presence of display entries
+    FROM biblio.record_entry bre 
+    LEFT JOIN metabib.compressed_display_entry mcde_title 
+        ON (bre.id = mcde_title.source AND mcde_title.name = 'title')
+    LEFT JOIN metabib.compressed_display_entry mcde_author 
+        ON (bre.id = mcde_author.source AND mcde_author.name = 'author')
+    LEFT JOIN metabib.compressed_display_entry mcde_subject 
+        ON (bre.id = mcde_subject.source AND mcde_subject.name = 'subject')
+    LEFT JOIN metabib.compressed_display_entry mcde_creators 
+        ON (bre.id = mcde_creators.source AND mcde_creators.name = 'creators')
+    LEFT JOIN metabib.compressed_display_entry mcde_isbn 
+        ON (bre.id = mcde_isbn.source AND mcde_isbn.name = 'isbn')
+;
+
+
 CREATE TABLE metabib.browse_entry (
     id BIGSERIAL PRIMARY KEY,
     value TEXT,

diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql
index b679ff6..2e452b9 100644 (file)
--- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql
@@ -60,7 +60,7 @@ CREATE VIEW metabib.wide_display_entry AS
         COALESCE(mcde_title.value, 'null') AS title,
         COALESCE(mcde_author.value, 'null') AS author,
         COALESCE(mcde_subject.value, 'null') AS subject,
-        COALESCE(mcde_topic_subject.value, 'null') AS topic_subject,
+        COALESCE(mcde_creators.value, 'null') AS creators,
         COALESCE(mcde_isbn.value, 'null') AS isbn
     -- ensure one row per bre regardless of any display fields
     FROM biblio.record_entry bre 
@@ -70,8 +70,8 @@ CREATE VIEW metabib.wide_display_entry AS
         ON (bre.id = mcde_author.source AND mcde_author.name = 'author')
     LEFT JOIN metabib.compressed_display_entry mcde_subject 
         ON (bre.id = mcde_subject.source AND mcde_subject.name = 'subject')
-    LEFT JOIN metabib.compressed_display_entry mcde_topic_subject 
-        ON (bre.id = mcde_topic_subject.source AND mcde_topic_subject.name = 'topic_subject')
+    LEFT JOIN metabib.compressed_display_entry mcde_creators 
+        ON (bre.id = mcde_creators.source AND mcde_creators.name = 'creators')
     LEFT JOIN metabib.compressed_display_entry mcde_isbn 
         ON (bre.id = mcde_isbn.source AND mcde_isbn.name = 'isbn')
 ;

diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql
index c1258e7..e633751 100644 (file)
--- a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql
@@ -11,7 +11,8 @@ UPDATE config.metabib_field SET display_field = TRUE WHERE id IN (6, 8, 16, 18);
 INSERT INTO config.metabib_field ( id, field_class, name, label,
     format, xpath, display_field, display_xpath ) VALUES
     (37, 'author', 'creator', oils_i18n_gettext(8, 'All Creators', 'cmf', 'label'),
-     'mods32', $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']]$$, TRUE, $$//*[local-name()='namePart']$$ ); -- /* to fool vim */;
+     'mods32', $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']]$$, 
+     TRUE, $$//*[local-name()='namePart']$$ ); -- /* to fool vim */;
 
 INSERT INTO config.display_field_map (name, field, multi) VALUES
     ('title', 6, FALSE),
@@ -24,5 +25,7 @@ INSERT INTO config.display_field_map (name, field, multi) VALUES
 COMMIT;
 
 -- REINGEST DISPLAY ENTRIES
-SELECT metabib.reingest_metabib_field_entries(id, TRUE, FALSE, TRUE, TRUE, '{6,8,16,18,37}'::INT[]) FROM biblio.record_entry WHERE NOT deleted AND id > 0;
+SELECT metabib.reingest_metabib_field_entries(id, TRUE, FALSE, TRUE, TRUE, 
+    (SELECT ARRAY_AGG(id)::INT[] FROM config.metabib_field WHERE display_field))
+    FROM biblio.record_entry WHERE NOT deleted AND id > 0;
 

diff --git a/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql
deleted file mode 100644 (file)
index 5997b97..0000000
--- a/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql
+++ /dev/null
@@ -1,308 +0,0 @@
--- XXX REVERT FILE -- DELETE THIS BEFORE MERGING XXX --
-
-DELETE FROM metabib.display_entry;
-DELETE FROM config.display_field_map;
-DELETE FROM config.metabib_field WHERE display_field; -- ASSUMES ALL NEW FIELDS
-
-DELETE FROM config.internal_flag WHERE name = 'ingest.skip_display_indexing';
-
-BEGIN;
-
-DROP FUNCTION metabib.reingest_metabib_field_entries(BIGINT, BOOL, BOOL, BOOL, BOOL);
-
-CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$
-DECLARE
-    fclass          RECORD;
-    ind_data        metabib.field_entry_template%ROWTYPE;
-    mbe_row         metabib.browse_entry%ROWTYPE;
-    mbe_id          BIGINT;
-    b_skip_facet    BOOL;
-    b_skip_browse   BOOL;
-    b_skip_search   BOOL;
-    value_prepped   TEXT;
-BEGIN
-
-    SELECT COALESCE(NULLIF(skip_facet, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name =  'ingest.skip_facet_indexing' AND enabled)) INTO b_skip_facet;
-    SELECT COALESCE(NULLIF(skip_browse, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name =  'ingest.skip_browse_indexing' AND enabled)) INTO b_skip_browse;
-    SELECT COALESCE(NULLIF(skip_search, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name =  'ingest.skip_search_indexing' AND enabled)) INTO b_skip_search;
-
-    PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled;
-    IF NOT FOUND THEN
-        IF NOT b_skip_search THEN
-            FOR fclass IN SELECT * FROM config.metabib_class LOOP
-                -- RAISE NOTICE 'Emptying out %', fclass.name;
-                EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id;
-            END LOOP;
-        END IF;
-        IF NOT b_skip_facet THEN
-            DELETE FROM metabib.facet_entry WHERE source = bib_id;
-        END IF;
-        IF NOT b_skip_browse THEN
-            DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id;
-        END IF;
-    END IF;
-
-    FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP
-
-       -- don't store what has been normalized away
-        CONTINUE WHEN ind_data.value IS NULL;
-
-        IF ind_data.field < 0 THEN
-            ind_data.field = -1 * ind_data.field;
-        END IF;
-
-        IF ind_data.facet_field AND NOT b_skip_facet THEN
-            INSERT INTO metabib.facet_entry (field, source, value)
-                VALUES (ind_data.field, ind_data.source, ind_data.value);
-        END IF;
-
-        IF ind_data.browse_field AND NOT b_skip_browse THEN
-            -- A caveat about this SELECT: this should take care of replacing
-            -- old mbe rows when data changes, but not if normalization (by
-            -- which I mean specifically the output of
-            -- evergreen.oils_tsearch2()) changes.  It may or may not be
-            -- expensive to add a comparison of index_vector to index_vector
-            -- to the WHERE clause below.
-
-            CONTINUE WHEN ind_data.sort_value IS NULL;
-
-            value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field);
-            SELECT INTO mbe_row * FROM metabib.browse_entry
-                WHERE value = value_prepped AND sort_value = ind_data.sort_value;
-
-            IF FOUND THEN
-                mbe_id := mbe_row.id;
-            ELSE
-                INSERT INTO metabib.browse_entry
-                    ( value, sort_value ) VALUES
-                    ( value_prepped, ind_data.sort_value );
-
-                mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS);
-            END IF;
-
-            INSERT INTO metabib.browse_entry_def_map (entry, def, source, authority)
-                VALUES (mbe_id, ind_data.field, ind_data.source, ind_data.authority);
-        END IF;
-
-        IF ind_data.search_field AND NOT b_skip_search THEN
-            -- Avoid inserting duplicate rows
-            EXECUTE 'SELECT 1 FROM metabib.' || ind_data.field_class ||
-                '_field_entry WHERE field = $1 AND source = $2 AND value = $3'
-                INTO mbe_id USING ind_data.field, ind_data.source, ind_data.value;
-                -- RAISE NOTICE 'Search for an already matching row returned %', mbe_id;
-            IF mbe_id IS NULL THEN
-                EXECUTE $$
-                INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value)
-                    VALUES ($$ ||
-                        quote_literal(ind_data.field) || $$, $$ ||
-                        quote_literal(ind_data.source) || $$, $$ ||
-                        quote_literal(ind_data.value) ||
-                    $$);$$;
-            END IF;
-        END IF;
-
-    END LOOP;
-
-    IF NOT b_skip_search THEN
-        PERFORM metabib.update_combined_index_vectors(bib_id);
-    END IF;
-
-    RETURN;
-END;
-$func$ LANGUAGE PLPGSQL;
-
-
-CREATE OR REPLACE FUNCTION biblio.extract_metabib_field_entry ( rid BIGINT, default_joiner TEXT ) RETURNS SETOF metabib.field_entry_template AS $func$
-DECLARE
-    bib     biblio.record_entry%ROWTYPE;
-    idx     config.metabib_field%ROWTYPE;
-    xfrm        config.xml_transform%ROWTYPE;
-    prev_xfrm   TEXT;
-    transformed_xml TEXT;
-    xml_node    TEXT;
-    xml_node_list   TEXT[];
-    facet_text  TEXT;
-    browse_text TEXT;
-    sort_value  TEXT;
-    raw_text    TEXT;
-    curr_text   TEXT;
-    joiner      TEXT := default_joiner; -- XXX will index defs supply a joiner?
-    authority_text TEXT;
-    authority_link BIGINT;
-    output_row  metabib.field_entry_template%ROWTYPE;
-BEGIN
-
-    -- Start out with no field-use bools set
-    output_row.browse_field = FALSE;
-    output_row.facet_field = FALSE;
-    output_row.search_field = FALSE;
-
-    -- Get the record
-    SELECT INTO bib * FROM biblio.record_entry WHERE id = rid;
-
-    -- Loop over the indexing entries
-    FOR idx IN SELECT * FROM config.metabib_field ORDER BY format LOOP
-
-        joiner := COALESCE(idx.joiner, default_joiner);
-
-        SELECT INTO xfrm * from config.xml_transform WHERE name = idx.format;
-
-        -- See if we can skip the XSLT ... it's expensive
-        IF prev_xfrm IS NULL OR prev_xfrm <> xfrm.name THEN
-            -- Can't skip the transform
-            IF xfrm.xslt <> '---' THEN
-                transformed_xml := oils_xslt_process(bib.marc,xfrm.xslt);
-            ELSE
-                transformed_xml := bib.marc;
-            END IF;
-
-            prev_xfrm := xfrm.name;
-        END IF;
-
-        xml_node_list := oils_xpath( idx.xpath, transformed_xml, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] );
-
-        raw_text := NULL;
-        FOR xml_node IN SELECT x FROM unnest(xml_node_list) AS x LOOP
-            CONTINUE WHEN xml_node !~ E'^\\s*<';
-
-            -- XXX much of this should be moved into oils_xpath_string...
-            curr_text := ARRAY_TO_STRING(evergreen.array_remove_item_by_value(evergreen.array_remove_item_by_value(
-                oils_xpath( '//text()', -- get the content of all the nodes within the main selected node
-                    REGEXP_REPLACE( xml_node, E'\\s+', ' ', 'g' ) -- Translate adjacent whitespace to a single space
-                ), ' '), ''),  -- throw away morally empty (bankrupt?) strings
-                joiner
-            );
-
-            CONTINUE WHEN curr_text IS NULL OR curr_text = '';
-
-            IF raw_text IS NOT NULL THEN
-                raw_text := raw_text || joiner;
-            END IF;
-
-            raw_text := COALESCE(raw_text,'') || curr_text;
-
-            -- autosuggest/metabib.browse_entry
-            IF idx.browse_field THEN
-
-                IF idx.browse_xpath IS NOT NULL AND idx.browse_xpath <> '' THEN
-                    browse_text := oils_xpath_string( idx.browse_xpath, xml_node, joiner, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] );
-                ELSE
-                    browse_text := curr_text;
-                END IF;
-
-                IF idx.browse_sort_xpath IS NOT NULL AND
-                    idx.browse_sort_xpath <> '' THEN
-
-                    sort_value := oils_xpath_string(
-                        idx.browse_sort_xpath, xml_node, joiner,
-                        ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]]
-                    );
-                ELSE
-                    sort_value := browse_text;
-                END IF;
-
-                output_row.field_class = idx.field_class;
-                output_row.field = idx.id;
-                output_row.source = rid;
-                output_row.value = BTRIM(REGEXP_REPLACE(browse_text, E'\\s+', ' ', 'g'));
-                output_row.sort_value :=
-                    public.naco_normalize(sort_value);
-
-                output_row.authority := NULL;
-
-                IF idx.authority_xpath IS NOT NULL AND idx.authority_xpath <> '' THEN
-                    authority_text := oils_xpath_string(
-                        idx.authority_xpath, xml_node, joiner,
-                        ARRAY[
-                            ARRAY[xfrm.prefix, xfrm.namespace_uri],
-                            ARRAY['xlink','http://www.w3.org/1999/xlink']
-                        ]
-                    );
-
-                    IF authority_text ~ '^\d+$' THEN
-                        authority_link := authority_text::BIGINT;
-                        PERFORM * FROM authority.record_entry WHERE id = authority_link;
-                        IF FOUND THEN
-                            output_row.authority := authority_link;
-                        END IF;
-                    END IF;
-
-                END IF;
-
-                output_row.browse_field = TRUE;
-                -- Returning browse rows with search_field = true for search+browse
-                -- configs allows us to retain granularity of being able to search
-                -- browse fields with "starts with" type operators (for example, for
-                -- titles of songs in music albums)
-                IF idx.search_field THEN
-                    output_row.search_field = TRUE;
-                END IF;
-                RETURN NEXT output_row;
-                output_row.browse_field = FALSE;
-                output_row.search_field = FALSE;
-                output_row.sort_value := NULL;
-            END IF;
-
-            -- insert raw node text for faceting
-            IF idx.facet_field THEN
-
-                IF idx.facet_xpath IS NOT NULL AND idx.facet_xpath <> '' THEN
-                    facet_text := oils_xpath_string( idx.facet_xpath, xml_node, joiner, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] );
-                ELSE
-                    facet_text := curr_text;
-                END IF;
-
-                output_row.field_class = idx.field_class;
-                output_row.field = -1 * idx.id;
-                output_row.source = rid;
-                output_row.value = BTRIM(REGEXP_REPLACE(facet_text, E'\\s+', ' ', 'g'));
-
-                output_row.facet_field = TRUE;
-                RETURN NEXT output_row;
-                output_row.facet_field = FALSE;
-            END IF;
-
-        END LOOP;
-
-        CONTINUE WHEN raw_text IS NULL OR raw_text = '';
-
-        -- insert combined node text for searching
-        IF idx.search_field THEN
-            output_row.field_class = idx.field_class;
-            output_row.field = idx.id;
-            output_row.source = rid;
-            output_row.value = BTRIM(REGEXP_REPLACE(raw_text, E'\\s+', ' ', 'g'));
-
-            output_row.search_field = TRUE;
-            RETURN NEXT output_row;
-            output_row.search_field = FALSE;
-        END IF;
-
-    END LOOP;
-
-END;
-
-$func$ LANGUAGE PLPGSQL;
-
-
-ALTER TYPE metabib.field_entry_template DROP ATTRIBUTE display_field;
-DROP TRIGGER display_field_force_nfc_tgr ON metabib.display_entry;
-DROP FUNCTION evergreen.display_field_force_nfc();
-DROP TRIGGER display_field_normalize_tgr ON metabib.display_entry;
-DROP FUNCTION metabib.display_field_normalize_trigger();
-DROP INDEX metabib.metabib_display_entry_source_idx;
-DROP INDEX metabib.metabib_display_entry_field_idx;
-DROP VIEW metabib.wide_display_entry;
-DROP VIEW metabib.compressed_display_entry;
-DROP VIEW metabib.flat_display_entry;
-DROP TABLE config.display_field_map;
-DROP TABLE metabib.display_entry;
-
-ALTER TABLE config.metabib_field 
-    DROP COLUMN display_xpath,
-    DROP COLUMN display_field;
-
-COMMIT;
-
-
-