From 97a3ab44152c2fbe6704f3f5fdfd167621dd3d5f Mon Sep 17 00:00:00 2001 From: Dan Wells Date: Thu, 13 Mar 2014 18:33:45 -0400 Subject: [PATCH] LP#1259665: Restore 800t to series indexing When we added browse support in 2.5, we made series indexing more specific. In the process, we lost support for searching 800t (this subfield is not particularly fit for browsing, as it has no non-filing character support). This commit restores 800t support for both searching and faceting. Signed-off-by: Dan Wells --- Open-ILS/src/sql/Pg/030.schema.metabib.sql | 2 + Open-ILS/src/sql/Pg/950.data.seed-values.sql | 4 +- .../XXXX.function.avoid_empty_metabib_extracts.sql | 102 +++++++++++++++++++++ .../Pg/upgrade/YYYY.data.revive_800_t_search.sql | 25 +++++ 4 files changed, 131 insertions(+), 2 deletions(-) create mode 100644 Open-ILS/src/sql/Pg/upgrade/XXXX.function.avoid_empty_metabib_extracts.sql create mode 100644 Open-ILS/src/sql/Pg/upgrade/YYYY.data.revive_800_t_search.sql diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index b8ad51a44e..0edba09342 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -824,6 +824,8 @@ BEGIN END IF; FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP + CONTINUE WHEN ind_data.value IS NULL; + IF ind_data.field < 0 THEN ind_data.field = -1 * ind_data.field; END IF; diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 9e0e3b1451..dde51fe386 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -110,8 +110,8 @@ INSERT INTO config.xml_transform VALUES ( 'mods33', 'http://www.loc.gov/mods/v3' INSERT INTO config.xml_transform VALUES ( 'marc21expand880', 'http://www.loc.gov/MARC21/slim', 'marc', '' ); -- Index Definitions -INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, facet_field, authority_xpath, browse_sort_xpath ) VALUES - (1, 'series', 'seriestitle', oils_i18n_gettext(1, 'Series Title', 'cmf', 'label'), 'mods32', $$//mods32:mods/mods32:relatedItem[@type="series"]/mods32:titleInfo[@type="nfi"]$$, TRUE, '//@xlink:href', $$*[local-name() != "nonSort"]$$ ); +INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, facet_field, authority_xpath, browse_xpath, browse_sort_xpath ) VALUES + (1, 'series', 'seriestitle', oils_i18n_gettext(1, 'Series Title', 'cmf', 'label'), 'mods32', $$//mods32:mods/mods32:relatedItem[@type="series"]/mods32:titleInfo$$, TRUE, '//@xlink:href', '*[@type="nfi"]', '*[@type="nfi"]/*[local-name() != "nonSort"]' ); INSERT INTO config.metabib_field ( id, field_class, name, label, format, xpath, authority_xpath ) VALUES (2, 'title', 'abbreviated', oils_i18n_gettext(2, 'Abbreviated Title', 'cmf', 'label'), 'mods32', $$//mods32:mods/mods32:titleInfo[mods32:title and (@type='abbreviated')]$$, '//@xlink:href' ); diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.function.avoid_empty_metabib_extracts.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.function.avoid_empty_metabib_extracts.sql new file mode 100644 index 0000000000..ee9f93eb1a --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.function.avoid_empty_metabib_extracts.sql @@ -0,0 +1,102 @@ +BEGIN; + +--SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version); + +CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$ +DECLARE + fclass RECORD; + ind_data metabib.field_entry_template%ROWTYPE; + mbe_row metabib.browse_entry%ROWTYPE; + mbe_id BIGINT; + b_skip_facet BOOL; + b_skip_browse BOOL; + b_skip_search BOOL; + value_prepped TEXT; +BEGIN + + SELECT COALESCE(NULLIF(skip_facet, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_facet_indexing' AND enabled)) INTO b_skip_facet; + SELECT COALESCE(NULLIF(skip_browse, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_browse_indexing' AND enabled)) INTO b_skip_browse; + SELECT COALESCE(NULLIF(skip_search, FALSE), EXISTS (SELECT enabled FROM config.internal_flag WHERE name = 'ingest.skip_search_indexing' AND enabled)) INTO b_skip_search; + + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + IF NOT b_skip_search THEN + FOR fclass IN SELECT * FROM config.metabib_class LOOP + -- RAISE NOTICE 'Emptying out %', fclass.name; + EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; + END LOOP; + END IF; + IF NOT b_skip_facet THEN + DELETE FROM metabib.facet_entry WHERE source = bib_id; + END IF; + IF NOT b_skip_browse THEN + DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id; + END IF; + END IF; + + FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP + CONTINUE WHEN ind_data.value IS NULL; + + IF ind_data.field < 0 THEN + ind_data.field = -1 * ind_data.field; + END IF; + + IF ind_data.facet_field AND NOT b_skip_facet THEN + INSERT INTO metabib.facet_entry (field, source, value) + VALUES (ind_data.field, ind_data.source, ind_data.value); + END IF; + + IF ind_data.browse_field AND NOT b_skip_browse THEN + -- A caveat about this SELECT: this should take care of replacing + -- old mbe rows when data changes, but not if normalization (by + -- which I mean specifically the output of + -- evergreen.oils_tsearch2()) changes. It may or may not be + -- expensive to add a comparison of index_vector to index_vector + -- to the WHERE clause below. + + value_prepped := metabib.browse_normalize(ind_data.value, ind_data.field); + SELECT INTO mbe_row * FROM metabib.browse_entry + WHERE value = value_prepped AND sort_value = ind_data.sort_value; + + IF FOUND THEN + mbe_id := mbe_row.id; + ELSE + INSERT INTO metabib.browse_entry + ( value, sort_value ) VALUES + ( value_prepped, ind_data.sort_value ); + + mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS); + END IF; + + INSERT INTO metabib.browse_entry_def_map (entry, def, source, authority) + VALUES (mbe_id, ind_data.field, ind_data.source, ind_data.authority); + END IF; + + IF ind_data.search_field AND NOT b_skip_search THEN + -- Avoid inserting duplicate rows + EXECUTE 'SELECT 1 FROM metabib.' || ind_data.field_class || + '_field_entry WHERE field = $1 AND source = $2 AND value = $3' + INTO mbe_id USING ind_data.field, ind_data.source, ind_data.value; + -- RAISE NOTICE 'Search for an already matching row returned %', mbe_id; + IF mbe_id IS NULL THEN + EXECUTE $$ + INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value) + VALUES ($$ || + quote_literal(ind_data.field) || $$, $$ || + quote_literal(ind_data.source) || $$, $$ || + quote_literal(ind_data.value) || + $$);$$; + END IF; + END IF; + + END LOOP; + + IF NOT b_skip_search THEN + PERFORM metabib.update_combined_index_vectors(bib_id); + END IF; + + RETURN; +END; +$func$ LANGUAGE PLPGSQL; + +COMMIT; diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.revive_800_t_search.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.revive_800_t_search.sql new file mode 100644 index 0000000000..dd0e1f0340 --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.revive_800_t_search.sql @@ -0,0 +1,25 @@ +BEGIN; + +--SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version); + +-- update metabib series config if stock +UPDATE config.metabib_field SET + xpath = '//mods32:mods/mods32:relatedItem[@type="series"]/mods32:titleInfo', + browse_xpath = '*[@type="nfi"]', + browse_sort_xpath = '*[@type="nfi"]/*[local-name() != "nonSort"]' +WHERE + id = 1 AND + xpath = '//mods32:mods/mods32:relatedItem[@type="series"]/mods32:titleInfo[@type="nfi"]' AND + browse_xpath IS NULL AND + browse_sort_xpath = '*[local-name() != "nonSort"]' +; + +COMMIT; + +\qecho This is a search and facet reingest of your bib records. +\qecho It may take a while. +\qecho You may cancel now without losing the effect of the rest of the +\qecho upgrade script, and arrange the reingest later. +\qecho . +SELECT metabib.reingest_metabib_field_entries(id, FALSE, TRUE, FALSE) + FROM biblio.record_entry; -- 2.11.0