From: gmc Date: Tue, 21 Sep 2010 14:19:31 +0000 (+0000) Subject: yet another bib ingest knob X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=1c30de857f602d12af6974a5e65f85e4f9884a43;p=evergreen%2Fbjwebb.git yet another bib ingest knob When enabled, the ingest.assume_inserts_only flag disables checking several tables for old index data to delete; skipping this makes a small but noticeable difference when doing an initial bib ingest with more than one process in parallel. Note that this flag should not be used when *re*ingesting records unless you clear the old rows beforehand. Signed-off-by: Galen Charlton git-svn-id: svn://svn.open-ils.org/ILS/trunk@17863 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql index 22c636262..bfb4fb781 100644 --- a/Open-ILS/src/sql/Pg/002.schema.config.sql +++ b/Open-ILS/src/sql/Pg/002.schema.config.sql @@ -58,6 +58,7 @@ INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_located_uri'); INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_full_rec'); INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_rec_descriptor'); INSERT INTO config.internal_flag (name) VALUES ('ingest.disable_metabib_field_entry'); +INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only'); CREATE TABLE config.global_flag ( label TEXT NOT NULL @@ -69,7 +70,7 @@ CREATE TABLE config.upgrade_log ( install_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW() ); -INSERT INTO config.upgrade_log (version) VALUES ('0410'); -- gmc +INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc CREATE TABLE config.bib_source ( id SERIAL PRIMARY KEY, diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 7be8ff3d6..8efc3da1e 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -707,7 +707,10 @@ $func$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$ BEGIN - DELETE FROM metabib.rec_descriptor WHERE record = bib_id; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + DELETE FROM metabib.rec_descriptor WHERE record = bib_id; + END IF; INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2) SELECT bib_id, biblio.marc21_extract_fixed_field( bib_id, 'Type' ), @@ -735,7 +738,10 @@ $func$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$ BEGIN - DELETE FROM metabib.real_full_rec WHERE record = bib_id; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + DELETE FROM metabib.real_full_rec WHERE record = bib_id; + END IF; INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value) SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id ); @@ -748,12 +754,14 @@ DECLARE fclass RECORD; ind_data metabib.field_entry_template%ROWTYPE; BEGIN - FOR fclass IN SELECT * FROM config.metabib_class LOOP - -- RAISE NOTICE 'Emptying out %', fclass.name; - EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; - END LOOP; - - DELETE FROM metabib.facet_entry WHERE source = bib_id; + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + FOR fclass IN SELECT * FROM config.metabib_class LOOP + -- RAISE NOTICE 'Emptying out %', fclass.name; + EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; + END LOOP; + DELETE FROM metabib.facet_entry WHERE source = bib_id; + END IF; FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP IF ind_data.field < 0 THEN diff --git a/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql b/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql new file mode 100644 index 000000000..03194fd28 --- /dev/null +++ b/Open-ILS/src/sql/Pg/upgrade/0411.schema.assume_inserts_only_flag.sql @@ -0,0 +1,86 @@ +BEGIN; + +INSERT INTO config.upgrade_log (version) VALUES ('0411'); -- gmc + +INSERT INTO config.internal_flag (name) VALUES ('ingest.assume_inserts_only'); + +CREATE OR REPLACE FUNCTION metabib.reingest_metabib_rec_descriptor( bib_id BIGINT ) RETURNS VOID AS $func$ +BEGIN + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + DELETE FROM metabib.rec_descriptor WHERE record = bib_id; + END IF; + INSERT INTO metabib.rec_descriptor (record, item_type, item_form, bib_level, control_type, enc_level, audience, lit_form, type_mat, cat_form, pub_status, item_lang, vr_format, date1, date2) + SELECT bib_id, + biblio.marc21_extract_fixed_field( bib_id, 'Type' ), + biblio.marc21_extract_fixed_field( bib_id, 'Form' ), + biblio.marc21_extract_fixed_field( bib_id, 'BLvl' ), + biblio.marc21_extract_fixed_field( bib_id, 'Ctrl' ), + biblio.marc21_extract_fixed_field( bib_id, 'ELvl' ), + biblio.marc21_extract_fixed_field( bib_id, 'Audn' ), + biblio.marc21_extract_fixed_field( bib_id, 'LitF' ), + biblio.marc21_extract_fixed_field( bib_id, 'TMat' ), + biblio.marc21_extract_fixed_field( bib_id, 'Desc' ), + biblio.marc21_extract_fixed_field( bib_id, 'DtSt' ), + biblio.marc21_extract_fixed_field( bib_id, 'Lang' ), + ( SELECT v.value + FROM biblio.marc21_physical_characteristics( bib_id) p + JOIN config.marc21_physical_characteristic_subfield_map s ON (s.id = p.subfield) + JOIN config.marc21_physical_characteristic_value_map v ON (v.id = p.value) + WHERE p.ptype = 'v' AND s.subfield = 'e' ), + biblio.marc21_extract_fixed_field( bib_id, 'Date1'), + biblio.marc21_extract_fixed_field( bib_id, 'Date2'); + + RETURN; +END; +$func$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION metabib.reingest_metabib_full_rec( bib_id BIGINT ) RETURNS VOID AS $func$ +BEGIN + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + DELETE FROM metabib.real_full_rec WHERE record = bib_id; + END IF; + INSERT INTO metabib.real_full_rec (record, tag, ind1, ind2, subfield, value) + SELECT record, tag, ind1, ind2, subfield, value FROM biblio.flatten_marc( bib_id ); + + RETURN; +END; +$func$ LANGUAGE PLPGSQL; + +CREATE OR REPLACE FUNCTION metabib.reingest_metabib_field_entries( bib_id BIGINT ) RETURNS VOID AS $func$ +DECLARE + fclass RECORD; + ind_data metabib.field_entry_template%ROWTYPE; +BEGIN + PERFORM * FROM config.internal_flag WHERE name = 'ingest.assume_inserts_only' AND enabled; + IF NOT FOUND THEN + FOR fclass IN SELECT * FROM config.metabib_class LOOP + -- RAISE NOTICE 'Emptying out %', fclass.name; + EXECUTE $$DELETE FROM metabib.$$ || fclass.name || $$_field_entry WHERE source = $$ || bib_id; + END LOOP; + DELETE FROM metabib.facet_entry WHERE source = bib_id; + END IF; + + FOR ind_data IN SELECT * FROM biblio.extract_metabib_field_entry( bib_id ) LOOP + IF ind_data.field < 0 THEN + ind_data.field = -1 * ind_data.field; + INSERT INTO metabib.facet_entry (field, source, value) + VALUES (ind_data.field, ind_data.source, ind_data.value); + ELSE + EXECUTE $$ + INSERT INTO metabib.$$ || ind_data.field_class || $$_field_entry (field, source, value) + VALUES ($$ || + quote_literal(ind_data.field) || $$, $$ || + quote_literal(ind_data.source) || $$, $$ || + quote_literal(ind_data.value) || + $$);$$; + END IF; + + END LOOP; + + RETURN; +END; +$func$ LANGUAGE PLPGSQL; + +COMMIT;