From: Bill Erickson Date: Fri, 25 Aug 2017 14:50:44 +0000 (-0400) Subject: LP#1251394 Display field schema sync'ing X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=436345ea4808152bdeee1f737a4073f15b73f2df;p=working%2FEvergreen.git LP#1251394 Display field schema sync'ing Copy flat/compressed/wide display entry views into base schema. Synchronize a few remaining seed data bits. Replace references to 'topic_subject' in wide display views with the new 'creators' field. Remove ZZZZ.UNDO testing SQL file. Signed-off-by: Bill Erickson Signed-off-by: Mike Rylander --- diff --git a/Open-ILS/examples/fm_IDL.xml b/Open-ILS/examples/fm_IDL.xml index 4d944a8b37..92a8e7197e 100644 --- a/Open-ILS/examples/fm_IDL.xml +++ b/Open-ILS/examples/fm_IDL.xml @@ -3807,7 +3807,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - + diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql index e8f3f24673..ff27a68126 100644 --- a/Open-ILS/src/sql/Pg/002.schema.config.sql +++ b/Open-ILS/src/sql/Pg/002.schema.config.sql @@ -48,9 +48,11 @@ INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only'); INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_browse_indexing'); INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_search_indexing'); INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_facet_indexing'); +INSERT INTO config.internal_flag (name) VALUES ('ingest.skip_display_indexing'); INSERT INTO config.internal_flag (name) VALUES ('serial.rematerialize_on_same_holding_code'); INSERT INTO config.internal_flag (name) VALUES ('ingest.metarecord_mapping.preserve_on_delete'); + CREATE TABLE config.global_flag ( label TEXT NOT NULL ) INHERITS (config.internal_flag); @@ -226,6 +228,12 @@ $$; CREATE UNIQUE INDEX config_metabib_field_class_name_idx ON config.metabib_field (field_class, name); +CREATE TABLE config.display_field_map ( + name TEXT PRIMARY KEY, + field INTEGER REFERENCES config.metabib_field (id), + multi BOOLEAN DEFAULT FALSE +); + CREATE TABLE config.ts_config_list ( id TEXT PRIMARY KEY, name TEXT NOT NULL diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 02406228c5..f55905d4de 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -195,6 +195,63 @@ CREATE INDEX metabib_display_entry_field_idx CREATE INDEX metabib_display_entry_source_idx ON metabib.display_entry (source); +CREATE VIEW metabib.flat_display_entry AS + /* One row per display entry fleshed with field info */ + SELECT + mde.source, + cdfm.name, + cdfm.multi, + cmf.label, + cmf.id AS field, + mde.value + FROM metabib.display_entry mde + JOIN config.metabib_field cmf ON (cmf.id = mde.field) + JOIN config.display_field_map cdfm ON (cdfm.field = mde.field) +; + +CREATE VIEW metabib.compressed_display_entry AS +/* Like flat_display_entry except values are compressed into + one row per display_field_map and JSON-ified. */ + SELECT + source, + name, + multi, + label, + field, + CASE WHEN multi THEN + TO_JSON(ARRAY_AGG(value)) + ELSE + TO_JSON(MIN(value)) + END AS value + FROM metabib.flat_display_entry + GROUP BY 1, 2, 3, 4, 5 +; + +CREATE VIEW metabib.wide_display_entry AS +/* Table-like view of well-known display fields. + This VIEW expands as well-known display fields are added. */ + SELECT + bre.id AS source, + COALESCE(mcde_title.value, 'null') AS title, + COALESCE(mcde_author.value, 'null') AS author, + COALESCE(mcde_subject.value, 'null') AS subject, + COALESCE(mcde_creators.value, 'null') AS creators, + COALESCE(mcde_isbn.value, 'null') AS isbn + -- ensure one row per bre regardless of the presence of display entries + FROM biblio.record_entry bre + LEFT JOIN metabib.compressed_display_entry mcde_title + ON (bre.id = mcde_title.source AND mcde_title.name = 'title') + LEFT JOIN metabib.compressed_display_entry mcde_author + ON (bre.id = mcde_author.source AND mcde_author.name = 'author') + LEFT JOIN metabib.compressed_display_entry mcde_subject + ON (bre.id = mcde_subject.source AND mcde_subject.name = 'subject') + LEFT JOIN metabib.compressed_display_entry mcde_creators + ON (bre.id = mcde_creators.source AND mcde_creators.name = 'creators') + LEFT JOIN metabib.compressed_display_entry mcde_isbn + ON (bre.id = mcde_isbn.source AND mcde_isbn.name = 'isbn') +; + + CREATE TABLE metabib.browse_entry ( id BIGSERIAL PRIMARY KEY, value TEXT, diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql index b679ff65aa..2e452b97c9 100644 --- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.metabib-display-field.sql @@ -60,7 +60,7 @@ CREATE VIEW metabib.wide_display_entry AS COALESCE(mcde_title.value, 'null') AS title, COALESCE(mcde_author.value, 'null') AS author, COALESCE(mcde_subject.value, 'null') AS subject, - COALESCE(mcde_topic_subject.value, 'null') AS topic_subject, + COALESCE(mcde_creators.value, 'null') AS creators, COALESCE(mcde_isbn.value, 'null') AS isbn -- ensure one row per bre regardless of any display fields FROM biblio.record_entry bre @@ -70,8 +70,8 @@ CREATE VIEW metabib.wide_display_entry AS ON (bre.id = mcde_author.source AND mcde_author.name = 'author') LEFT JOIN metabib.compressed_display_entry mcde_subject ON (bre.id = mcde_subject.source AND mcde_subject.name = 'subject') - LEFT JOIN metabib.compressed_display_entry mcde_topic_subject - ON (bre.id = mcde_topic_subject.source AND mcde_topic_subject.name = 'topic_subject') + LEFT JOIN metabib.compressed_display_entry mcde_creators + ON (bre.id = mcde_creators.source AND mcde_creators.name = 'creators') LEFT JOIN metabib.compressed_display_entry mcde_isbn ON (bre.id = mcde_isbn.source AND mcde_isbn.name = 'isbn') ; diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql index c1258e7a5b..e63375162b 100644 --- a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql +++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.metabib-display-field.sql @@ -11,7 +11,8 @@ UPDATE config.metabib_field SET display_field = TRUE WHERE id IN (6, 8, 16, 18); INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, display_field, display_xpath ) VALUES (37, 'author', 'creator', oils_i18n_gettext(8, 'All Creators', 'cmf', 'label'), - 'mods32', $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']]$$, TRUE, $$//*[local-name()='namePart']$$ ); -- /* to fool vim */; + 'mods32', $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']]$$, + TRUE, $$//*[local-name()='namePart']$$ ); -- /* to fool vim */; INSERT INTO config.display_field_map (name, field, multi) VALUES ('title', 6, FALSE), @@ -24,5 +25,7 @@ INSERT INTO config.display_field_map (name, field, multi) VALUES COMMIT; -- REINGEST DISPLAY ENTRIES -SELECT metabib.reingest_metabib_field_entries(id, TRUE, FALSE, TRUE, TRUE, '{6,8,16,18,37}'::INT[]) FROM biblio.record_entry WHERE NOT deleted AND id > 0; +SELECT metabib.reingest_metabib_field_entries(id, TRUE, FALSE, TRUE, TRUE, + (SELECT ARRAY_AGG(id)::INT[] FROM config.metabib_field WHERE display_field)) + FROM biblio.record_entry WHERE NOT deleted AND id > 0; diff --git a/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql b/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql deleted file mode 100644 index 5997b973ad..0000000000 --- a/Open-ILS/src/sql/Pg/upgrade/ZZZZ.UNDO.metabib-display-field.sql +++ /dev/null @@ -1,308 +0,0 @@ --- XXX REVERT FILE -- DELETE THIS BEFORE MERGING XXX -- - -DELETE FROM metabib.display_entry; -DELETE FROM config.display_field_map; -DELETE FROM config.metabib_field WHERE display_field; -- ASSUMES ALL NEW FIELDS - -DELETE FROM config.internal_flag WHERE name = 'ingest.skip_display_indexing'; - -BEGIN; - -DROP FUNCTION metabib.reingest_metabib_field_entries(BIGINT, BOOL, BOOL, BOOL, BOOL); - -CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$ -DECLARE - fclass RECORD; - ind_data metabib.field_entry_template%ROWTYPE; - mbe_row metabib.browse_entry%ROWTYPE; - mbe_id BIGINT; - b_skip_facet BOOL; - b_skip_browse BOOL; - b_skip_search BOOL; - value_prepped TEXT; -BEGIN - - SELECT COALESCE(NULLIF(skip_facet, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_facet_indexing' AND enabled)) INTO b_skip_facet; - SELECT COALESCE(NULLIF(skip_browse, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_browse_indexing' AND enabled)) INTO b_skip_browse; - SELECT COALESCE(NULLIF(skip_search, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_search_indexing' AND enabled)) INTO b_skip_search; - - PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; - IF NOT FOUND THEN - IF NOT b_skip_search THEN - FOR fclass IN SELECT * FROM config.metabib_class LOOP - -- RAISE NOTICE 'Emptying out %', fclass.name; - EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; - END LOOP; - END IF; - IF NOT b_skip_facet THEN - DELETE FROM metabib.facet_entry WHERE source = bib_id; - END IF; - IF NOT b_skip_browse THEN - DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id; - END IF; - END IF; - - FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP - - -- don't store what has been normalized away - CONTINUE WHEN ind_data.value IS NULL; - - IF ind_data.field < 0 THEN - ind_data.field = -1 * ind_data.field; - END IF; - - IF ind_data.facet_field AND NOT b_skip_facet THEN - INSERT INTO metabib.facet_entry (field, source, value) - VALUES (ind_data.field, ind_data.source, ind_data.value); - END IF; - - IF ind_data.browse_field AND NOT b_skip_browse THEN - -- A caveat about this SELECT: this should take care of replacing - -- old mbe rows when data changes, but not if normalization (by - -- which I mean specifically the output of - -- evergreen.oils_tsearch2()) changes. It may or may not be - -- expensive to add a comparison of index_vector to index_vector - -- to the WHERE clause below. - - CONTINUE WHEN ind_data.sort_value IS NULL; - - value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field); - SELECT INTO mbe_row * FROM metabib.browse_entry - WHERE value = value_prepped AND sort_value = ind_data.sort_value; - - IF FOUND THEN - mbe_id := mbe_row.id; - ELSE - INSERT INTO metabib.browse_entry - ( value, sort_value ) VALUES - ( value_prepped, ind_data.sort_value ); - - mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS); - END IF; - - INSERT INTO metabib.browse_entry_def_map (entry, def, source, authority) - VALUES (mbe_id, ind_data.field, ind_data.source, ind_data.authority); - END IF; - - IF ind_data.search_field AND NOT b_skip_search THEN - -- Avoid inserting duplicate rows - EXECUTE 'SELECT 1 FROM metabib.' || ind_data.field_class || - '_field_entry WHERE field = $1 AND source = $2 AND value = $3' - INTO mbe_id USING ind_data.field, ind_data.source, ind_data.value; - -- RAISE NOTICE 'Search for an already matching row returned %', mbe_id; - IF mbe_id IS NULL THEN - EXECUTE $$ - INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value) - VALUES ($$ || - quote_literal(ind_data.field) || $$, $$ || - quote_literal(ind_data.source) || $$, $$ || - quote_literal(ind_data.value) || - $$);$$; - END IF; - END IF; - - END LOOP; - - IF NOT b_skip_search THEN - PERFORM metabib.update_combined_index_vectors(bib_id); - END IF; - - RETURN; -END; -$func$ LANGUAGE PLPGSQL; - - -CREATE OR REPLACE FUNCTION biblio.extract_metabib_field_entry ( rid BIGINT, default_joiner TEXT ) RETURNS SETOF metabib.field_entry_template AS $func$ -DECLARE - bib biblio.record_entry%ROWTYPE; - idx config.metabib_field%ROWTYPE; - xfrm config.xml_transform%ROWTYPE; - prev_xfrm TEXT; - transformed_xml TEXT; - xml_node TEXT; - xml_node_list TEXT[]; - facet_text TEXT; - browse_text TEXT; - sort_value TEXT; - raw_text TEXT; - curr_text TEXT; - joiner TEXT := default_joiner; -- XXX will index defs supply a joiner? - authority_text TEXT; - authority_link BIGINT; - output_row metabib.field_entry_template%ROWTYPE; -BEGIN - - -- Start out with no field-use bools set - output_row.browse_field = FALSE; - output_row.facet_field = FALSE; - output_row.search_field = FALSE; - - -- Get the record - SELECT INTO bib * FROM biblio.record_entry WHERE id = rid; - - -- Loop over the indexing entries - FOR idx IN SELECT * FROM config.metabib_field ORDER BY format LOOP - - joiner := COALESCE(idx.joiner, default_joiner); - - SELECT INTO xfrm * from config.xml_transform WHERE name = idx.format; - - -- See if we can skip the XSLT ... it's expensive - IF prev_xfrm IS NULL OR prev_xfrm <> xfrm.name THEN - -- Can't skip the transform - IF xfrm.xslt <> '---' THEN - transformed_xml := oils_xslt_process(bib.marc,xfrm.xslt); - ELSE - transformed_xml := bib.marc; - END IF; - - prev_xfrm := xfrm.name; - END IF; - - xml_node_list := oils_xpath( idx.xpath, transformed_xml, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] ); - - raw_text := NULL; - FOR xml_node IN SELECT x FROM unnest(xml_node_list) AS x LOOP - CONTINUE WHEN xml_node !~ E'^\\s*<'; - - -- XXX much of this should be moved into oils_xpath_string... - curr_text := ARRAY_TO_STRING(evergreen.array_remove_item_by_value(evergreen.array_remove_item_by_value( - oils_xpath( '//text()', -- get the content of all the nodes within the main selected node - REGEXP_REPLACE( xml_node, E'\\s+', ' ', 'g' ) -- Translate adjacent whitespace to a single space - ), ' '), ''), -- throw away morally empty (bankrupt?) strings - joiner - ); - - CONTINUE WHEN curr_text IS NULL OR curr_text = ''; - - IF raw_text IS NOT NULL THEN - raw_text := raw_text || joiner; - END IF; - - raw_text := COALESCE(raw_text,'') || curr_text; - - -- autosuggest/metabib.browse_entry - IF idx.browse_field THEN - - IF idx.browse_xpath IS NOT NULL AND idx.browse_xpath <> '' THEN - browse_text := oils_xpath_string( idx.browse_xpath, xml_node, joiner, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] ); - ELSE - browse_text := curr_text; - END IF; - - IF idx.browse_sort_xpath IS NOT NULL AND - idx.browse_sort_xpath <> '' THEN - - sort_value := oils_xpath_string( - idx.browse_sort_xpath, xml_node, joiner, - ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] - ); - ELSE - sort_value := browse_text; - END IF; - - output_row.field_class = idx.field_class; - output_row.field = idx.id; - output_row.source = rid; - output_row.value = BTRIM(REGEXP_REPLACE(browse_text, E'\\s+', ' ', 'g')); - output_row.sort_value := - public.naco_normalize(sort_value); - - output_row.authority := NULL; - - IF idx.authority_xpath IS NOT NULL AND idx.authority_xpath <> '' THEN - authority_text := oils_xpath_string( - idx.authority_xpath, xml_node, joiner, - ARRAY[ - ARRAY[xfrm.prefix, xfrm.namespace_uri], - ARRAY['xlink','http://www.w3.org/1999/xlink'] - ] - ); - - IF authority_text ~ '^\d+$' THEN - authority_link := authority_text::BIGINT; - PERFORM * FROM authority.record_entry WHERE id = authority_link; - IF FOUND THEN - output_row.authority := authority_link; - END IF; - END IF; - - END IF; - - output_row.browse_field = TRUE; - -- Returning browse rows with search_field = true for search+browse - -- configs allows us to retain granularity of being able to search - -- browse fields with "starts with" type operators (for example, for - -- titles of songs in music albums) - IF idx.search_field THEN - output_row.search_field = TRUE; - END IF; - RETURN NEXT output_row; - output_row.browse_field = FALSE; - output_row.search_field = FALSE; - output_row.sort_value := NULL; - END IF; - - -- insert raw node text for faceting - IF idx.facet_field THEN - - IF idx.facet_xpath IS NOT NULL AND idx.facet_xpath <> '' THEN - facet_text := oils_xpath_string( idx.facet_xpath, xml_node, joiner, ARRAY[ARRAY[xfrm.prefix, xfrm.namespace_uri]] ); - ELSE - facet_text := curr_text; - END IF; - - output_row.field_class = idx.field_class; - output_row.field = -1 * idx.id; - output_row.source = rid; - output_row.value = BTRIM(REGEXP_REPLACE(facet_text, E'\\s+', ' ', 'g')); - - output_row.facet_field = TRUE; - RETURN NEXT output_row; - output_row.facet_field = FALSE; - END IF; - - END LOOP; - - CONTINUE WHEN raw_text IS NULL OR raw_text = ''; - - -- insert combined node text for searching - IF idx.search_field THEN - output_row.field_class = idx.field_class; - output_row.field = idx.id; - output_row.source = rid; - output_row.value = BTRIM(REGEXP_REPLACE(raw_text, E'\\s+', ' ', 'g')); - - output_row.search_field = TRUE; - RETURN NEXT output_row; - output_row.search_field = FALSE; - END IF; - - END LOOP; - -END; - -$func$ LANGUAGE PLPGSQL; - - -ALTER TYPE metabib.field_entry_template DROP ATTRIBUTE display_field; -DROP TRIGGER display_field_force_nfc_tgr ON metabib.display_entry; -DROP FUNCTION evergreen.display_field_force_nfc(); -DROP TRIGGER display_field_normalize_tgr ON metabib.display_entry; -DROP FUNCTION metabib.display_field_normalize_trigger(); -DROP INDEX metabib.metabib_display_entry_source_idx; -DROP INDEX metabib.metabib_display_entry_field_idx; -DROP VIEW metabib.wide_display_entry; -DROP VIEW metabib.compressed_display_entry; -DROP VIEW metabib.flat_display_entry; -DROP TABLE config.display_field_map; -DROP TABLE metabib.display_entry; - -ALTER TABLE config.metabib_field - DROP COLUMN display_xpath, - DROP COLUMN display_field; - -COMMIT; - - -