LP1053397: Don't leak memory; Cache compiled ccraed values

We were leaking memory in a PLPGSQL function that compiles the
query_int from a ccraed value.  Stop doing that!

Also, implement a self-invalidating (upon table update) cache
for compiled composite attr defs.  This brings concerto reingest
down from 50+ seconds to under 10.

Signed-off-by: Mike Rylander <mrylander@gmail.com>
Signed-off-by: Dan Wells <dbw2@calvin.edu>

diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
index 7eb7dd9..e5c9673 100644 (file)
--- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql
+++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql
@@ -296,13 +296,44 @@ CREATE VIEW metabib.full_attr_id_map AS
     SELECT id, attr, value FROM metabib.composite_attr_id_map;
 
 
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_init () RETURNS BOOL AS $f$
+    $_SHARED{metabib_compile_composite_attr_cache} = {}
+        if ! exists $_SHARED{metabib_compile_composite_attr_cache};
+    return exists $_SHARED{metabib_compile_composite_attr_cache};
+$f$ LANGUAGE PLPERLU;
+
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_disable () RETURNS BOOL AS $f$
+    delete $_SHARED{metabib_compile_composite_attr_cache};
+    return ! exists $_SHARED{metabib_compile_composite_attr_cache};
+$f$ LANGUAGE PLPERLU;
+
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_invalidate () RETURNS BOOL AS $f$
+    SELECT metabib.compile_composite_attr_cache_disable() AND metabib.compile_composite_attr_cache_init();
+$f$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION metabib.composite_attr_def_cache_inval_tgr () RETURNS TRIGGER AS $f$
+BEGIN
+    PERFORM metabib.compile_composite_attr_cache_invalidate();
+    RETURN NULL;
+END;
+$f$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER ccraed_cache_inval_tgr AFTER INSERT OR UPDATE OR DELETE ON config.composite_attr_entry_definition FOR EACH STATEMENT EXECUTE PROCEDURE metabib.composite_attr_def_cache_inval_tgr();
+    
 CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RETURNS query_int AS $func$
 
     use JSON::XS;
-    my $def = decode_json(shift);
+
+    my $json = shift;
+    my $def = decode_json($json);
 
     die("Composite attribute definition not supplied") unless $def;
 
+    my $_cache = (exists $_SHARED{metabib_compile_composite_attr_cache}) ? 1 : 0;
+
+    return $_SHARED{metabib_compile_composite_attr_cache}{$json}
+        if ($_cache && $_SHARED{metabib_compile_composite_attr_cache}{$json});
+
     sub recurse {
         my $d = shift;
         my $j = '&';
@@ -311,10 +342,11 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET
         if (ref $d eq 'HASH') { # node or AND
             if (exists $d->{_attr}) { # it is a node
                 my $plan = spi_prepare('SELECT * FROM metabib.full_attr_id_map WHERE attr = $1 AND value = $2', qw/TEXT TEXT/);
-                return spi_exec_prepared(
+                my $id = spi_exec_prepared(
                     $plan, {limit => 1}, $d->{_attr}, $d->{_val}
                 )->{rows}[0]{id};
                 spi_freeplan($plan);
+                return $id;
             } elsif (exists $d->{_not} && scalar(keys(%$d)) == 1) { # it is a NOT
                 return '!' . recurse($$d{_not});
             } else { # an AND list
@@ -331,7 +363,9 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET
         return '';
     }
 
-    return recurse($def) || undef;
+    my $val = recurse($def) || undef;
+    $_SHARED{metabib_compile_composite_attr_cache}{$json} = $val if $_cache;
+    return $val;
 
 $func$ IMMUTABLE LANGUAGE plperlu;
 
@@ -1563,6 +1597,7 @@ BEGIN
 
     -- On to composite attributes, now that the record attrs have been pulled.  Processed in name order, so later composite
     -- attributes can depend on earlier ones.
+    PERFORM metabib.compile_composite_attr_cache_init();
     FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP
 
         FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP

diff --git a/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql b/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql
index 0fc6cd6..c04d8f4 100644 (file)
--- a/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql
+++ b/Open-ILS/src/sql/Pg/upgrade/QQQQ.MVF_CRA-upgrade.sql
@@ -185,13 +185,44 @@ CREATE VIEW metabib.rec_descriptor AS
             (populate_record(NULL::metabib.rec_desc_type, attrs)).*
       FROM  metabib.record_attr;
 
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_init () RETURNS BOOL AS $f$
+    $_SHARED{metabib_compile_composite_attr_cache} = {}
+        if ! exists $_SHARED{metabib_compile_composite_attr_cache};
+    return exists $_SHARED{metabib_compile_composite_attr_cache};
+$f$ LANGUAGE PLPERLU;
+
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_disable () RETURNS BOOL AS $f$
+    delete $_SHARED{metabib_compile_composite_attr_cache};
+    return ! exists $_SHARED{metabib_compile_composite_attr_cache};
+$f$ LANGUAGE PLPERLU;
+
+CREATE OR REPLACE FUNCTION metabib.compile_composite_attr_cache_invalidate () RETURNS BOOL AS $f$
+    SELECT metabib.compile_composite_attr_cache_disable() AND metabib.compile_composite_attr_cache_init();
+$f$ LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION metabib.composite_attr_def_cache_inval_tgr () RETURNS TRIGGER AS $f$
+BEGIN
+    PERFORM metabib.compile_composite_attr_cache_invalidate();
+    RETURN NULL;
+END;
+$f$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER ccraed_cache_inval_tgr AFTER INSERT OR UPDATE OR DELETE ON config.composite_attr_entry_definition FOR EACH STATEMENT EXECUTE PROCEDURE metabib.composite_attr_def_cache_inval_tgr();
+    
 CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RETURNS query_int AS $func$
 
     use JSON::XS;
-    my $def = decode_json(shift);
+
+    my $json = shift;
+    my $def = decode_json($json);
 
     die("Composite attribute definition not supplied") unless $def;
 
+    my $_cache = (exists $_SHARED{metabib_compile_composite_attr_cache}) ? 1 : 0;
+
+    return $_SHARED{metabib_compile_composite_attr_cache}{$json}
+        if ($_cache && $_SHARED{metabib_compile_composite_attr_cache}{$json});
+
     sub recurse {
         my $d = shift;
         my $j = '&';
@@ -200,10 +231,11 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET
         if (ref $d eq 'HASH') { # node or AND
             if (exists $d->{_attr}) { # it is a node
                 my $plan = spi_prepare('SELECT * FROM metabib.full_attr_id_map WHERE attr = $1 AND value = $2', qw/TEXT TEXT/);
-                return spi_exec_prepared(
+                my $id = spi_exec_prepared(
                     $plan, {limit => 1}, $d->{_attr}, $d->{_val}
                 )->{rows}[0]{id};
                 spi_freeplan($plan);
+                return $id;
             } elsif (exists $d->{_not} && scalar(keys(%$d)) == 1) { # it is a NOT
                 return '!' . recurse($$d{_not});
             } else { # an AND list
@@ -220,7 +252,9 @@ CREATE OR REPLACE FUNCTION metabib.compile_composite_attr ( cattr_def TEXT ) RET
         return '';
     }
 
-    return recurse($def) || undef;
+    my $val = recurse($def) || undef;
+    $_SHARED{metabib_compile_composite_attr_cache}{$json} = $val if $_cache;
+    return $val;
 
 $func$ IMMUTABLE LANGUAGE plperlu;
 
@@ -400,6 +434,7 @@ BEGIN
 
     -- On to composite attributes, now that the record attrs have been pulled.  Processed in name order, so later composite
     -- attributes can depend on earlier ones.
+    PERFORM metabib.compile_composite_attr_cache_init();
     FOR attr_def IN SELECT * FROM config.record_attr_definition WHERE composite AND name = ANY( attr_list ) ORDER BY name LOOP
 
         FOR ccvm_row IN SELECT * FROM config.coded_value_map c WHERE c.ctype = attr_def.name ORDER BY value LOOP