From: Mike Rylander Date: Tue, 18 Feb 2014 21:53:53 +0000 (-0500) Subject: LP1053397: Don't leak memory; Cache compiled ccraed values X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=e7a369e53b38e44b8dc16da3b485e1cfb8e4c1fd;p=working%2FEvergreen.git LP1053397: Don't leak memory; Cache compiled ccraed values We were leaking memory in a PLPGSQL function that compiles the query_int from a ccraed value. Stop doing that! Also, implement a self-invalidating (upon table update) cache for compiled composite attr defs. This brings concerto reingest down from 50+ seconds to under 10. Signed-off-by: Mike Rylander --- diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 7eb7dd9c58..8de64a1989 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -296,13 +296,43 @@ CREATE VIEW metabib.full_attr_id_map AS SELECT id, attr, value FROM metabib.composite_attr_id_map; +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_init () RETURNS BOOL AS $f$ + $_SHARED{metabib_compile_composite_attr_cache} = {} + if ! exists $_SHARED{metabib_compile_composite_attr_cache}; + return exists $_SHARED{metabib_compile_composite_attr_cache}; +$f$ LANGUAGE PLPERLU; + +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_disable () RETURNS BOOL AS $f$ + delete $_SHARED{metabib_compile_composite_attr_cache}; + return ! exists $_SHARED{metabib_compile_composite_attr_cache}; +$f$ LANGUAGE PLPERLU; + +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_invalidate () RETURNS BOOL AS $f$ + SELECT metabib.compile_composite_attr_cache_disable() AND metabib.compile_composite_attr_cache_init(); +$f$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION metabib.composite_attr_def_cache_inval_tgr () RETURNS TRIGGER AS $f$ +BEGIN + PERFORM metabib.compile_composite_attr_cache_invalidate(); +END; +$f$ LANGUAGE PLPGSQL; + +CREATE TRIGGER ccraed_cache_inval_tgr AFTER INSERT OR UPDATE OR DELETE ON config.composite_attr_entry_definition FOR EACH STATEMENT EXECUTE PROCEDURE metabib.composite_attr_def_cache_inval_tgr(); + CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RETURNS query_int AS $func$ use JSON::XS; - my $def = decode_json(shift); + + my $json = shift; + my $def = decode_json($json); die("Composite attribute definition not supplied") unless $def; + my $_cache = (exists $_SHARED{metabib_compile_composite_attr_cache}) ? 1 : 0; + + return $_SHARED{metabib_compile_composite_attr_cache}{$json} + if ($_cache && $_SHARED{metabib_compile_composite_attr_cache}{$json}); + sub recurse { my $d = shift; my $j = '&'; @@ -311,10 +341,11 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET if (ref $d eq 'HASH') { # node or AND if (exists $d->{_attr}) { # it is a node my $plan = spi_prepare('SELECT * FROM metabib.full_attr_id_map WHERE attr = $1 AND value = $2', qw/TEXT TEXT/); - return spi_exec_prepared( + my $id = spi_exec_prepared( $plan, {limit => 1}, $d->{_attr}, $d->{_val} )->{rows}[0]{id}; spi_freeplan($plan); + return $id; } elsif (exists $d->{_not} && scalar(keys(%$d)) == 1) { # it is a NOT return '!' . recurse($$d{_not}); } else { # an AND list @@ -331,7 +362,9 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET return ''; } - return recurse($def) || undef; + my $val = recurse($def) || undef; + $_SHARED{metabib_compile_composite_attr_cache}{$json} = $val if $_cache; + return $val; $func$ IMMUTABLE LANGUAGE plperlu; @@ -1563,6 +1596,7 @@ BEGIN -- On to composite attributes, now that the record attrs have been pulled. Processed in name order, so later composite -- attributes can depend on earlier ones. + PERFORM metabib.compile_composite_attr_cache_init(); FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP diff --git a/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql b/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql index 0fc6cd6733..4450566a7e 100644 --- a/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql +++ b/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql @@ -185,13 +185,43 @@ CREATE VIEW metabib.rec_descriptor AS (populate_record(NULL::metabib.rec_desc_type, attrs)).* FROM metabib.record_attr; +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_init () RETURNS BOOL AS $f$ + $_SHARED{metabib_compile_composite_attr_cache} = {} + if ! exists $_SHARED{metabib_compile_composite_attr_cache}; + return exists $_SHARED{metabib_compile_composite_attr_cache}; +$f$ LANGUAGE PLPERLU; + +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_disable () RETURNS BOOL AS $f$ + delete $_SHARED{metabib_compile_composite_attr_cache}; + return ! exists $_SHARED{metabib_compile_composite_attr_cache}; +$f$ LANGUAGE PLPERLU; + +CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_invalidate () RETURNS BOOL AS $f$ + SELECT metabib.compile_composite_attr_cache_disable() AND metabib.compile_composite_attr_cache_init(); +$f$ LANGUAGE SQL; + +CREATE OR REPLACE FUNCTION metabib.composite_attr_def_cache_inval_tgr () RETURNS TRIGGER AS $f$ +BEGIN + PERFORM metabib.compile_composite_attr_cache_invalidate(); +END; +$f$ LANGUAGE PLPGSQL; + +CREATE TRIGGER ccraed_cache_inval_tgr AFTER INSERT OR UPDATE OR DELETE ON config.composite_attr_entry_definition FOR EACH STATEMENT EXECUTE PROCEDURE metabib.composite_attr_def_cache_inval_tgr(); + CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RETURNS query_int AS $func$ use JSON::XS; - my $def = decode_json(shift); + + my $json = shift; + my $def = decode_json($json); die("Composite attribute definition not supplied") unless $def; + my $_cache = (exists $_SHARED{metabib_compile_composite_attr_cache}) ? 1 : 0; + + return $_SHARED{metabib_compile_composite_attr_cache}{$json} + if ($_cache && $_SHARED{metabib_compile_composite_attr_cache}{$json}); + sub recurse { my $d = shift; my $j = '&'; @@ -200,10 +230,11 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET if (ref $d eq 'HASH') { # node or AND if (exists $d->{_attr}) { # it is a node my $plan = spi_prepare('SELECT * FROM metabib.full_attr_id_map WHERE attr = $1 AND value = $2', qw/TEXT TEXT/); - return spi_exec_prepared( + my $id = spi_exec_prepared( $plan, {limit => 1}, $d->{_attr}, $d->{_val} )->{rows}[0]{id}; spi_freeplan($plan); + return $id; } elsif (exists $d->{_not} && scalar(keys(%$d)) == 1) { # it is a NOT return '!' . recurse($$d{_not}); } else { # an AND list @@ -220,7 +251,9 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET return ''; } - return recurse($def) || undef; + my $val = recurse($def) || undef; + $_SHARED{metabib_compile_composite_attr_cache}{$json} = $val if $_cache; + return $val; $func$ IMMUTABLE LANGUAGE plperlu; @@ -400,6 +433,7 @@ BEGIN -- On to composite attributes, now that the record attrs have been pulled. Processed in name order, so later composite -- attributes can depend on earlier ones. + PERFORM metabib.compile_composite_attr_cache_init(); FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP