From: Mike Rylander Date: Mon, 20 Jan 2014 22:24:10 +0000 (-0500) Subject: Schema elements and ingest logic for composite attributes X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=1434b71cb98653d6f1a589a23a3a7ec0cfb955f5;p=working%2FEvergreen.git Schema elements and ingest logic for composite attributes Signed-off-by: Mike Rylander --- diff --git a/Open-ILS/src/sql/Pg/002.schema.config.sql b/Open-ILS/src/sql/Pg/002.schema.config.sql index ce5e56fc80..1ae319a335 100644 --- a/Open-ILS/src/sql/Pg/002.schema.config.sql +++ b/Open-ILS/src/sql/Pg/002.schema.config.sql @@ -773,6 +773,7 @@ CREATE TABLE config.record_attr_definition ( multi BOOL NOT NULL DEFAULT TRUE, -- will store all values from a record filter BOOL NOT NULL DEFAULT TRUE, -- becomes QP filter if true sorter BOOL NOT NULL DEFAULT FALSE, -- becomes QP sort() axis if true + composite BOOL NOT NULL DEFAULT FALSE, -- its values are derived from others -- For pre-extracted fields. Takes the first occurance, uses naive subfield ordering tag TEXT, -- LIKE format @@ -850,6 +851,11 @@ BEGIN END; $f$ LANGUAGE PLPGSQL; +CREATE TABLE config.composite_attr_entry_definition( + coded_value PRIMARY KEY NOT NULL REFERENCES config.coded_value_map (id) ON UPDATE CASCADE ON DELETE CASCADE, + definition TEXT NOT NULL -- JSON +); + -- List applied db patches that are deprecated by (and block the application of) my_db_patch CREATE OR REPLACE FUNCTION evergreen.upgrade_list_applied_deprecates ( my_db_patch TEXT ) RETURNS SETOF evergreen.patch AS $$ SELECT DISTINCT l.version diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 89ef57a79d..c23a3ac78b 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -278,6 +278,65 @@ CREATE TABLE metabib.uncontrolled_record_attr_value ( ); CREATE UNIQUE INDEX muv_once_idx ON metabib.uncontrolled_record_attr_value (attr,value); +CREATE VIEW metabib.record_attr_id_map AS + SELECT id, attr, value FROM metabib.uncontrolled_record_attr_value + UNION + SELECT c.id, c.ctype AS attr, c.code AS value + FROM config.coded_value_map c + JOIN config.record_attr_definition d ON (d.name = c.ctype AND NOT d.composite); + +CREATE VIEW metabib.composite_attr_id_map AS + SELECT c.id, c.ctype AS attr, c.code AS value + FROM config.coded_value_map c + JOIN config.record_attr_definition d ON (d.name = c.ctype AND d.composite); + +CREATE VIEW metabib.full_attr_id_map AS + SELECT id, attr, value FROM metabib.record_attr_id_map + UNION + SELECT id, attr, value FROM metabib.composite_attr_id_map; + + +CREATE FUNCTION metabib.compile_composite_attr ( cattr_id INT ) RETURNS query_int AS $func$ + + use JSON; + my $cid = shift; + + my $cattr = spi_exec_query( + "SELECT * FROM config.composite_attr_entry_defintion WHERE id = $cid" + )->{rows}[0]; + + die("Composite attribute not found with an id of $cid") unless $cattr; + + my $plan = spi_prepare('SELECT * FROM metabib.full_attr_id_map WHERE attr = $1 AND value = $2', qw/TEXT TEXT/); + my $def = from_json $cattr->{definition}; + + sub recurse { + my $d = shift; + my $j = '&'; + my @list; + + if (ref $d eq 'HASH') { # node or AND + if (exists $d->{_attr}) { # it is a node + return spi_query_prepared( + $plan, {limit => 1}, $d->{_attr}, $d->{_val} + )->{rows}[0]{id}; + } elsif (exists $d->{_not} && scalar(keys(%$d)) == 1) { # it is a NOT + return '!' . recurse($$d{_not}); + } else { # an AND list + @list = map { recurse($$d{$_}) } sort keys %$d; + } + } elsif (ref $d eq 'ARRAY') + $j = '|' + @list = map { recurse($_) } @$d; + } + return '(' . join($j,@list) . ')' if @list; + return ''; + } + + return recurse($def); + +$func$ STRICT STABLE IMMUTABLE LANGUAGE plperlu; + CREATE TABLE metabib.record_attr_vector_list ( source BIGINT PRIMARY KEY REFERENCES biblio.record_entry (id), vlist INT[] NOT NULL -- stores id from ccvm AND murav @@ -313,19 +372,11 @@ CREATE TYPE metabib.record_attr_type AS ( -- Back-compat view ... we're moving to an INTARRAY world CREATE VIEW metabib.record_attr_flat AS - SELECT DISTINCT v.source AS id, - c.ctype AS attr, - c.code AS value - FROM metabib.record_attr_vector_list v - LEFT JOIN config.coded_value_map c ON ( c.id = ANY( v.vlist ) ) - WHERE c.id IS NOT NULL - UNION ALL - SELECT DISTINCT v.source AS id, - u.attr, - u.value - FROM metabib.record_attr_vector_list v - LEFT JOIN metabib.uncontrolled_record_attr_value u ON ( u.id = ANY( v.vlist ) ) - WHERE u.id IS NOT NULL; + SELECT v.source AS id, + m.attr, + m.value + FROM metabib.full_attr_id_map m + JOIN metabib.record_attr_vector_list v ( m.id = ANY( v.vlist ) ) CREATE VIEW metabib.record_attr AS SELECT id, HSTORE( ARRAY_AGG( attr ), ARRAY_AGG( value ) ) AS attrs FROM metabib.record_attr_flat GROUP BY 1; @@ -1364,7 +1415,7 @@ BEGIN SELECT marc INTO rmarc FROM biblio.record_entry WHERE id = rid; END IF; - FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE name = ANY( attr_list ) ORDER BY format LOOP + FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE NOT composite AND name = ANY( attr_list ) ORDER BY format LOOP attr_value := '{}'::TEXT[]; norm_attr_value := '{}'::TEXT[]; @@ -1494,7 +1545,6 @@ BEGIN END LOOP; - IF ARRAY_LENGTH(attr_vector, 1) > 0 THEN /* We may need to rewrite the vlist to contain the intersection of new values for requested attrs and old values for ignored attrs. To @@ -1502,13 +1552,41 @@ BEGIN subtract any values that are valid for the requested attrs, and then add back the new set of attr values. */ - IF ARRAY_LENGTH(pattr_list, 1) > 0 THEN - SELECT vlist INTO attr_vector_tmp FROM metabib.record_attr_vector_list WHERE source = rid; - SELECT attr_vector_tmp - ARRAY_AGG(id) INTO attr_vector_tmp FROM metabib.uncontrolled_record_attr_value WHERE attr = ANY (pattr_list); - SELECT attr_vector_tmp - ARRAY_AGG(id) INTO attr_vector_tmp FROM config.coded_value_map WHERE ctype = ANY (pattr_list); - attr_vector := attr_vector || attr_vector_tmp; - END IF; + IF ARRAY_LENGTH(pattr_list, 1) > 0 THEN + SELECT vlist INTO attr_vector_tmp FROM metabib.record_attr_vector_list WHERE source = rid; + SELECT attr_vector_tmp - ARRAY_AGG(id) INTO attr_vector_tmp FROM metabib.full_attr_id_map WHERE attr = ANY (pattr_list); + attr_vector := attr_vector || attr_vector_tmp; + END IF; + + -- On to composite attributes, now that the record attrs have been pulled. Processed in name order, so later composite + -- attributes can depend on earlier ones. + FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP + + FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP + + tmp_val := metabib.compile_composite_attr( ccvm_row.id ); + NEXT WHEN tmp_val IS NULL OR tmp_val = ''; -- nothing to do + + IF attr_def.filter THEN + IF attr_vector @@ tmp_val::query_int THEN + attr_vector = attr_vector + intset(ccvm_row.id); + EXIT WHEN NOT attr_def.multi; + END IF; + END IF; + + IF attr_def.sorter THEN + IF attr_vector ~~ tmp_val THEN + DELETE FROM metabib.record_sorter WHERE source = rid AND attr = attr_def.name; + INSERT INTO metabib.record_sorter (source, attr, value) VALUES (rid, attr_def.name, ccvm_row.code); + END IF; + END IF; + + END LOOP; + + END LOOP; + + IF ARRAY_LENGTH(attr_vector, 1) > 0 THEN IF rdeleted THEN -- initial insert OR revivication DELETE FROM metabib.record_attr_vector_list WHERE source = rid; INSERT INTO metabib.record_attr_vector_list (source, vlist) VALUES (rid, attr_vector);