From: Mike Rylander Date: Sat, 13 Oct 2012 15:38:55 +0000 (-0400) Subject: LP#1066326: Allow component partitioning of ingest X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=577909c863c4a9dc4cbacaba886cd08e0c9da624;p=evergreen%2Fpines.git LP#1066326: Allow component partitioning of ingest The component parts of the indexing process are fairly flexible. In some places they can be called with parameters that control the behavior, and other places will respond to global and/or internal flags. However, there is some inconsistency to which parts will react to each mechanism. Previous to this commit, there was no way to cause ingest to skip specific parts of the indexing process (search / facet / browse, in particular) even though the stored procedure responsible for this had the ability to perform, and parameters for the control of, this behavior. Now, when any of the parameters are false (the default, saying "do not skip") it will check the value of a related global flag and, if enabled, skip that part of the ingest. This will allow one to do perform massive, batch, parallel ingests of bib records, something that was effectively blocked by the way browse ingest is performed. By disabling browse ingest during the main record loading, and then performing only the browse ingest later, one can significantly reduce the time required to load massive batches of records. Signed-off-by: Mike Rylander Signed-off-by: Galen Charlton Signed-off-by: Mike Rylander --- diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index da9d4ddca1..fd041d158d 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -580,19 +580,27 @@ DECLARE ind_data metabib.field_entry_template%ROWTYPE; mbe_row metabib.browse_entry%ROWTYPE; mbe_id BIGINT; + b_skip_facet BOOL; + b_skip_browse BOOL; + b_skip_search BOOL; BEGIN + + SELECT COALESCE(NULLIF(skip_facet, FALSE), enabled) INTO b_skip_facet FROM config.internal_flag WHERE name = 'ingest.skip_facet_indexing'; + SELECT COALESCE(NULLIF(skip_browse, FALSE), enabled) INTO b_skip_browse FROM config.internal_flag WHERE name = 'ingest.skip_browse_indexing'; + SELECT COALESCE(NULLIF(skip_search, FALSE), enabled) INTO b_skip_search FROM config.internal_flag WHERE name = 'ingest.skip_search_indexing'; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; IF NOT FOUND THEN - IF NOT skip_search THEN + IF NOT b_skip_search THEN FOR fclass IN SELECT * FROM config.metabib_class LOOP -- RAISE NOTICE 'Emptying out %', fclass.name; EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; END LOOP; END IF; - IF NOT skip_facet THEN + IF NOT b_skip_facet THEN DELETE FROM metabib.facet_entry WHERE source = bib_id; END IF; - IF NOT skip_browse THEN + IF NOT b_skip_browse THEN DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id; END IF; END IF; @@ -602,12 +610,12 @@ BEGIN ind_data.field = -1 * ind_data.field; END IF; - IF ind_data.facet_field AND NOT skip_facet THEN + IF ind_data.facet_field AND NOT b_skip_facet THEN INSERT INTO metabib.facet_entry (field, source, value) VALUES (ind_data.field, ind_data.source, ind_data.value); END IF; - IF ind_data.browse_field AND NOT skip_browse THEN + IF ind_data.browse_field AND NOT b_skip_browse THEN -- A caveat about this SELECT: this should take care of replacing -- old mbe rows when data changes, but not if normalization (by -- which I mean specifically the output of @@ -627,7 +635,7 @@ BEGIN VALUES (mbe_id, ind_data.field, ind_data.source); END IF; - IF ind_data.search_field AND NOT skip_search THEN + IF ind_data.search_field AND NOT b_skip_search THEN EXECUTE $$ INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value) VALUES ($$ || diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 51d608fbd8..4792e0cb95 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -8929,6 +8929,39 @@ INSERT INTO config.global_flag (name, label, enabled) INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE VALUES ( + 'ingest.skip_browse_indexing', + oils_i18n_gettext( + 'ingest.skip_browse_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of browse data', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'ingest.skip_search_indexing', + oils_i18n_gettext( + 'ingest.skip_search_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of search data', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'ingest.skip_facet_indexing', + oils_i18n_gettext( + 'ingest.skip_facet_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of facet data', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( 'ingest.disable_authority_linking', oils_i18n_gettext( 'ingest.disable_authority_linking', diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.partitioned-ingest.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.partitioned-ingest.sql new file mode 100644 index 0000000000..d1da1138ab --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.partitioned-ingest.sql @@ -0,0 +1,115 @@ +BEGIN; + +SELECT evergreen.upgrade_deps_block_check('XXXX', :eg_version); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'ingest.skip_browse_indexing', + oils_i18n_gettext( + 'ingest.skip_browse_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of browse data', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'ingest.skip_search_indexing', + oils_i18n_gettext( + 'ingest.skip_search_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of search data', + 'cgf', + 'label' + ) + ); + +INSERT INTO config.global_flag (name, label) -- defaults to enabled=FALSE + VALUES ( + 'ingest.skip_facet_indexing', + oils_i18n_gettext( + 'ingest.skip_facet_indexing', + 'Bibliographic Record Ingest: Disable extraction and indexing of facet data', + 'cgf', + 'label' + ) + ); + +CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT, skip_facet BOOL DEFAULT FALSE, skip_browse BOOL DEFAULT FALSE, skip_search BOOL DEFAULT FALSE ) RETURNS VOID AS $func$ +DECLARE + fclass RECORD; + ind_data metabib.field_entry_template%ROWTYPE; + mbe_row metabib.browse_entry%ROWTYPE; + mbe_id BIGINT; + b_skip_facet BOOL; + b_skip_browse BOOL; + b_skip_search BOOL; +BEGIN + + SELECT COALESCE(NULLIF(skip_facet, FALSE), enabled) INTO b_skip_facet FROM config.internal_flag WHERE name = 'ingest.skip_facet_indexing'; + SELECT COALESCE(NULLIF(skip_browse, FALSE), enabled) INTO b_skip_browse FROM config.internal_flag WHERE name = 'ingest.skip_browse_indexing'; + SELECT COALESCE(NULLIF(skip_search, FALSE), enabled) INTO b_skip_search FROM config.internal_flag WHERE name = 'ingest.skip_search_indexing'; + + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + IF NOT b_skip_search THEN + FOR fclass IN SELECT * FROM config.metabib_class LOOP + -- RAISE NOTICE 'Emptying out %', fclass.name; + EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; + END LOOP; + END IF; + IF NOT b_skip_facet THEN + DELETE FROM metabib.facet_entry WHERE source = bib_id; + END IF; + IF NOT b_skip_browse THEN + DELETE FROM metabib.browse_entry_def_map WHERE source = bib_id; + END IF; + END IF; + + FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP + IF ind_data.field < 0 THEN + ind_data.field = -1 * ind_data.field; + END IF; + + IF ind_data.facet_field AND NOT b_skip_facet THEN + INSERT INTO metabib.facet_entry (field, source, value) + VALUES (ind_data.field, ind_data.source, ind_data.value); + END IF; + + IF ind_data.browse_field AND NOT b_skip_browse THEN + -- A caveat about this SELECT: this should take care of replacing + -- old mbe rows when data changes, but not if normalization (by + -- which I mean specifically the output of + -- evergreen.oils_tsearch2()) changes. It may or may not be + -- expensive to add a comparison of index_vector to index_vector + -- to the WHERE clause below. + SELECT INTO mbe_row * FROM metabib.browse_entry WHERE value = ind_data.value; + IF FOUND THEN + mbe_id := mbe_row.id; + ELSE + INSERT INTO metabib.browse_entry (value) VALUES + (metabib.browse_normalize(ind_data.value, ind_data.field)); + mbe_id := CURRVAL('metabib.browse_entry_id_seq'::REGCLASS); + END IF; + + INSERT INTO metabib.browse_entry_def_map (entry, def, source) + VALUES (mbe_id, ind_data.field, ind_data.source); + END IF; + + IF ind_data.search_field AND NOT b_skip_search THEN + EXECUTE $$ + INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value) + VALUES ($$ || + quote_literal(ind_data.field) || $$, $$ || + quote_literal(ind_data.source) || $$, $$ || + quote_literal(ind_data.value) || + $$);$$; + END IF; + + END LOOP; + + RETURN; +END; +$func$ LANGUAGE PLPGSQL; + +COMMIT;