Stamping upgrades for Link Checker
authorMike Rylander <mrylander@gmail.com>
Thu, 14 Feb 2013 19:24:00 +0000 (14:24 -0500)
committerMike Rylander <mrylander@gmail.com>
Thu, 14 Feb 2013 19:24:00 +0000 (14:24 -0500)
Signed-off-by: Mike Rylander <mrylander@gmail.com>
Open-ILS/src/sql/Pg/002.schema.config.sql
Open-ILS/src/sql/Pg/upgrade/0752.schema.url_verify.sql [new file with mode: 0644]
Open-ILS/src/sql/Pg/upgrade/0753.functions.url_verify.sql [new file with mode: 0644]
Open-ILS/src/sql/Pg/upgrade/0754.data.url_verify.sql [new file with mode: 0644]
Open-ILS/src/sql/Pg/upgrade/XXXX.schema.url_verify.sql [deleted file]
Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql [deleted file]
Open-ILS/src/sql/Pg/upgrade/ZZZZ.data.url_verify.sql [deleted file]

index 8463598..0419423 100644 (file)
@@ -87,7 +87,7 @@ CREATE TRIGGER no_overlapping_deps
     BEFORE INSERT OR UPDATE ON config.db_patch_dependencies
     FOR EACH ROW EXECUTE PROCEDURE evergreen.array_overlap_check ('deprecates');
 
-INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0751', :eg_version); -- berick/senator
+INSERT INTO config.upgrade_log (version, applied_to) VALUES ('0754', :eg_version); -- senator/miker
 
 CREATE TABLE config.bib_source (
        id              SERIAL  PRIMARY KEY,
diff --git a/Open-ILS/src/sql/Pg/upgrade/0752.schema.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/0752.schema.url_verify.sql
new file mode 100644 (file)
index 0000000..ccf0372
--- /dev/null
@@ -0,0 +1,96 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('0752', :eg_version);
+
+INSERT INTO container.biblio_record_entry_bucket_type (code, label) VALUES ('url_verify', 'URL Verification Queue');
+
+DROP SCHEMA IF EXISTS url_verify CASCADE;
+
+CREATE SCHEMA url_verify;
+
+CREATE TABLE url_verify.session (
+    id          SERIAL                      PRIMARY KEY,
+    name        TEXT                        NOT NULL,
+    owning_lib  INT                         NOT NULL REFERENCES actor.org_unit (id) DEFERRABLE INITIALLY DEFERRED,
+    creator     INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
+    container   INT                         NOT NULL REFERENCES container.biblio_record_entry_bucket (id) DEFERRABLE INITIALLY DEFERRED,
+    create_time TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
+    search      TEXT                        NOT NULL,
+    CONSTRAINT uvs_name_once_per_lib UNIQUE (name, owning_lib)
+);
+
+CREATE TABLE url_verify.url_selector (
+    id      SERIAL  PRIMARY KEY,
+    xpath   TEXT    NOT NULL,
+    session INT     NOT NULL REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
+    CONSTRAINT tag_once_per_sess UNIQUE (xpath, session)
+);
+
+CREATE TABLE url_verify.url (
+    id              SERIAL  PRIMARY KEY,
+    redirect_from   INT     REFERENCES url_verify.url(id) DEFERRABLE INITIALLY DEFERRED,
+    item            INT     REFERENCES container.biblio_record_entry_bucket_item (id) DEFERRABLE INITIALLY DEFERRED,
+    url_selector    INT     REFERENCES url_verify.url_selector (id) DEFERRABLE INITIALLY DEFERRED,
+    session         INT     REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
+    tag             TEXT,
+    subfield        TEXT,
+    ord             INT,
+    full_url        TEXT    NOT NULL,
+    scheme          TEXT,
+    username        TEXT,
+    password        TEXT,
+    host            TEXT,
+    domain          TEXT,
+    tld             TEXT,
+    port            TEXT,
+    path            TEXT,
+    page            TEXT,
+    query           TEXT,
+    fragment        TEXT,
+    CONSTRAINT redirect_or_from_item CHECK (
+        redirect_from IS NOT NULL OR (
+            item         IS NOT NULL AND
+            url_selector IS NOT NULL AND
+            tag          IS NOT NULL AND
+            subfield     IS NOT NULL AND
+            ord          IS NOT NULL
+        )
+    )
+);
+
+CREATE TABLE url_verify.verification_attempt (
+    id          SERIAL                      PRIMARY KEY,
+    usr         INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
+    session     INT                         NOT NULL REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
+    start_time  TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
+    finish_time TIMESTAMP WITH TIME ZONE
+);
+CREATE TABLE url_verify.url_verification (
+    id          SERIAL                      PRIMARY KEY,
+    url         INT                         NOT NULL REFERENCES url_verify.url (id) DEFERRABLE INITIALLY DEFERRED,
+    attempt     INT                         NOT NULL REFERENCES url_verify.verification_attempt (id) DEFERRABLE INITIALLY DEFERRED,
+    req_time    TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
+    res_time    TIMESTAMP WITH TIME ZONE, 
+    res_code    INT                         CHECK (res_code BETWEEN 100 AND 999), -- we know > 599 will never be valid HTTP code, but we use 9XX for other stuff
+    res_text    TEXT, 
+    redirect_to INT                         REFERENCES url_verify.url (id) DEFERRABLE INITIALLY DEFERRED -- if redirected
+);
+
+CREATE TABLE config.filter_dialog_interface (
+    key         TEXT                        PRIMARY KEY,
+    description TEXT
+);
+
+CREATE TABLE config.filter_dialog_filter_set (
+    id          SERIAL                      PRIMARY KEY,
+    name        TEXT                        NOT NULL,
+    owning_lib  INT                         NOT NULL REFERENCES actor.org_unit (id) DEFERRABLE INITIALLY DEFERRED,
+    creator     INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
+    create_time TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
+    interface   TEXT                        NOT NULL REFERENCES config.filter_dialog_interface (key) DEFERRABLE INITIALLY DEFERRED,
+    filters     TEXT                        NOT NULL CHECK (evergreen.is_json(filters)),
+    CONSTRAINT cfdfs_name_once_per_lib UNIQUE (name, owning_lib)
+);
+COMMIT;
diff --git a/Open-ILS/src/sql/Pg/upgrade/0753.functions.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/0753.functions.url_verify.sql
new file mode 100644 (file)
index 0000000..026b4a2
--- /dev/null
@@ -0,0 +1,100 @@
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('0753', :eg_version);
+
+CREATE OR REPLACE FUNCTION url_verify.parse_url (url_in TEXT) RETURNS url_verify.url AS $$
+
+use Rose::URI;
+
+my $url_in = shift;
+my $url = Rose::URI->new($url_in);
+
+my %parts = map { $_ => $url->$_ } qw/scheme username password host port path query fragment/;
+
+$parts{full_url} = $url_in;
+($parts{domain} = $parts{host}) =~ s/^[^.]+\.//;
+($parts{tld} = $parts{domain}) =~ s/(?:[^.]+\.)+//;
+($parts{page} = $parts{path}) =~ s#(?:[^/]*/)+##;
+
+return \%parts;
+
+$$ LANGUAGE PLPERLU;
+
+CREATE OR REPLACE FUNCTION url_verify.ingest_url () RETURNS TRIGGER AS $$
+DECLARE
+    tmp_row url_verify.url%ROWTYPE;
+BEGIN
+    SELECT * INTO tmp_row FROM url_verify.parse_url(NEW.full_url);
+
+    NEW.scheme          := tmp_row.scheme;
+    NEW.username        := tmp_row.username;
+    NEW.password        := tmp_row.password;
+    NEW.host            := tmp_row.host;
+    NEW.domain          := tmp_row.domain;
+    NEW.tld             := tmp_row.tld;
+    NEW.port            := tmp_row.port;
+    NEW.path            := tmp_row.path;
+    NEW.page            := tmp_row.page;
+    NEW.query           := tmp_row.query;
+    NEW.fragment        := tmp_row.fragment;
+
+    RETURN NEW;
+END;
+$$ LANGUAGE PLPGSQL;
+
+CREATE TRIGGER ingest_url_tgr
+    BEFORE INSERT ON url_verify.url
+    FOR EACH ROW EXECUTE PROCEDURE url_verify.ingest_url(); 
+
+CREATE OR REPLACE FUNCTION url_verify.extract_urls ( session_id INT, item_id INT ) RETURNS INT AS $$
+DECLARE
+    last_seen_tag TEXT;
+    current_tag TEXT;
+    current_sf TEXT;
+    current_url TEXT;
+    current_ord INT;
+    current_url_pos INT;
+    current_selector url_verify.url_selector%ROWTYPE;
+BEGIN
+    current_ord := 1;
+
+    FOR current_selector IN SELECT * FROM url_verify.url_selector s WHERE s.session = session_id LOOP
+        current_url_pos := 1;
+        LOOP
+            SELECT  (XPATH(current_selector.xpath || '/text()', b.marc::XML))[current_url_pos]::TEXT INTO current_url
+              FROM  biblio.record_entry b
+                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
+              WHERE c.id = item_id;
+
+            EXIT WHEN current_url IS NULL;
+
+            SELECT  (XPATH(current_selector.xpath || '/../@tag', b.marc::XML))[current_url_pos]::TEXT INTO current_tag
+              FROM  biblio.record_entry b
+                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
+              WHERE c.id = item_id;
+
+            IF current_tag IS NULL THEN
+                current_tag := last_seen_tag;
+            ELSE
+                last_seen_tag := current_tag;
+            END IF;
+
+            SELECT  (XPATH(current_selector.xpath || '/@code', b.marc::XML))[current_url_pos]::TEXT INTO current_sf
+              FROM  biblio.record_entry b
+                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
+              WHERE c.id = item_id;
+
+            INSERT INTO url_verify.url (session, item, url_selector, tag, subfield, ord, full_url)
+              VALUES ( session_id, item_id, current_selector.id, current_tag, current_sf, current_ord, current_url);
+
+            current_url_pos := current_url_pos + 1;
+            current_ord := current_ord + 1;
+        END LOOP;
+    END LOOP;
+
+    RETURN current_ord - 1;
+END;
+$$ LANGUAGE PLPGSQL;
+
+COMMIT;
+
diff --git a/Open-ILS/src/sql/Pg/upgrade/0754.data.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/0754.data.url_verify.sql
new file mode 100644 (file)
index 0000000..eecbca8
--- /dev/null
@@ -0,0 +1,195 @@
+
+-- NOTE: beware the use of bare perm IDs in the update_perm's below and in 
+-- the 950 seed data file.  Update before merge to match current perm IDs! XXX
+
+BEGIN;
+
+SELECT evergreen.upgrade_deps_block_check('0754', :eg_version);
+
+INSERT INTO permission.perm_list (id, code, description) 
+    VALUES ( 
+        543, 
+        'URL_VERIFY',
+        oils_i18n_gettext(
+            543, 
+            'Allows a user to process and verify ULSs', 
+            'ppl', 
+            'description'
+        )
+    );
+
+
+INSERT INTO permission.perm_list (id, code, description) 
+    VALUES ( 
+        544, 
+        544,
+        oils_i18n_gettext(
+            544, 
+            'Allows a user to configure URL verification org unit settings',
+            'ppl', 
+            'description'
+        )
+    );
+
+
+INSERT INTO permission.perm_list (id, code, description) 
+    VALUES ( 
+        545, 
+        'SAVED_FILTER_DIALOG_FILTERS',
+        oils_i18n_gettext(
+            545, 
+            'Allows users to save and load sets of filters for filter dialogs, available in certain staff interfaces',
+            'ppl', 
+            'description'
+        )
+    );
+
+
+INSERT INTO config.settings_group (name, label)
+    VALUES (
+        'url_verify',
+        oils_i18n_gettext(
+            'url_verify',
+            'URL Verify',
+            'csg',
+            'label'
+        )
+    );
+
+INSERT INTO config.org_unit_setting_type
+    (name, grp, label, description, datatype, update_perm)
+    VALUES (
+        'url_verify.url_verification_delay',
+        'url_verify',
+        oils_i18n_gettext(
+            'url_verify.url_verification_delay',
+            'Number of seconds to wait between URL test attempts.',
+            'coust',
+            'label'
+        ),
+        oils_i18n_gettext(
+            'url_verify.url_verification_delay',
+            'Throttling mechanism for batch URL verification runs.  Each running process will wait this number of seconds after a URL test before performing the next.',
+            'coust',
+            'description'
+        ),
+        'integer',
+        544
+    );
+
+INSERT INTO config.org_unit_setting_type
+    (name, grp, label, description, datatype, update_perm)
+    VALUES (
+        'url_verify.url_verification_max_redirects',
+        'url_verify',
+        oils_i18n_gettext(
+            'url_verify.url_verification_max_redirects',
+            'Maximum redirect lookups',
+            'coust',
+            'label'
+        ),
+        oils_i18n_gettext(
+            'url_verify.url_verification_max_redirects',
+            'For URLs returning 3XX redirects, this is the maximum number of redirects we will follow before giving up.',
+            'coust',
+            'description'
+        ),
+        'integer',
+        544
+    );
+
+INSERT INTO config.org_unit_setting_type
+    (name, grp, label, description, datatype, update_perm)
+    VALUES (
+        'url_verify.url_verification_max_wait',
+        'url_verify',
+        oils_i18n_gettext(
+            'url_verify.url_verification_max_wait',
+            'Maximum wait time (in seconds) for a URL to lookup',
+            'coust',
+            'label'
+        ),
+        oils_i18n_gettext(
+            'url_verify.url_verification_max_wait',
+            'If we exceed the wait time, the URL is marked as a "timeout" and the system moves on to the next URL',
+            'coust',
+            'description'
+        ),
+        'integer',
+        544
+    );
+
+
+INSERT INTO config.org_unit_setting_type
+    (name, grp, label, description, datatype, update_perm)
+    VALUES (
+        'url_verify.verification_batch_size',
+        'url_verify',
+        oils_i18n_gettext(
+            'url_verify.verification_batch_size',
+            'Number of URLs to test in parallel',
+            'coust',
+            'label'
+        ),
+        oils_i18n_gettext(
+            'url_verify.verification_batch_size',
+            'URLs are tested in batches.  This number defines the size of each batch and it directly relates to the number of back-end processes performing URL verification.',
+            'coust',
+            'description'
+        ),
+        'integer',
+        544
+    );
+
+
+INSERT INTO config.filter_dialog_interface (key, description) VALUES (
+    'url_verify',
+    oils_i18n_gettext(
+        'url_verify',
+        'All Link Checker filter dialogs',
+        'cfdi',
+        'description'
+    )
+);
+
+
+INSERT INTO config.usr_setting_type (name,grp,opac_visible,label,description,datatype) VALUES (
+    'ui.grid_columns.url_verify.select_urls',
+    'gui',
+    FALSE,
+    oils_i18n_gettext(
+        'ui.grid_columns.url_verify.select_urls',
+        'Link Checker''s URL Selection interface''s saved columns',
+        'cust',
+        'label'
+    ),
+    oils_i18n_gettext(
+        'ui.grid_columns.url_verify.select_urls',
+        'Link Checker''s URL Selection interface''s saved columns',
+        'cust',
+        'description'
+    ),
+    'string'
+);
+
+INSERT INTO config.usr_setting_type (name,grp,opac_visible,label,description,datatype) VALUES (
+    'ui.grid_columns.url_verify.review_attempt',
+    'gui',
+    FALSE,
+    oils_i18n_gettext(
+        'ui.grid_columns.url_verify.review_attempt',
+        'Link Checker''s Review Attempt interface''s saved columns',
+        'cust',
+        'label'
+    ),
+    oils_i18n_gettext(
+        'ui.grid_columns.url_verify.review_attempt',
+        'Link Checker''s Review Attempt interface''s saved columns',
+        'cust',
+        'description'
+    ),
+    'string'
+);
+
+COMMIT;
+
diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.url_verify.sql
deleted file mode 100644 (file)
index ddddd41..0000000
+++ /dev/null
@@ -1,94 +0,0 @@
-BEGIN;
-
-INSERT INTO container.biblio_record_entry_bucket_type (code, label) VALUES ('url_verify', 'URL Verification Queue');
-
-DROP SCHEMA IF EXISTS url_verify CASCADE;
-
-CREATE SCHEMA url_verify;
-
-CREATE TABLE url_verify.session (
-    id          SERIAL                      PRIMARY KEY,
-    name        TEXT                        NOT NULL,
-    owning_lib  INT                         NOT NULL REFERENCES actor.org_unit (id) DEFERRABLE INITIALLY DEFERRED,
-    creator     INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
-    container   INT                         NOT NULL REFERENCES container.biblio_record_entry_bucket (id) DEFERRABLE INITIALLY DEFERRED,
-    create_time TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
-    search      TEXT                        NOT NULL,
-    CONSTRAINT uvs_name_once_per_lib UNIQUE (name, owning_lib)
-);
-
-CREATE TABLE url_verify.url_selector (
-    id      SERIAL  PRIMARY KEY,
-    xpath   TEXT    NOT NULL,
-    session INT     NOT NULL REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
-    CONSTRAINT tag_once_per_sess UNIQUE (xpath, session)
-);
-
-CREATE TABLE url_verify.url (
-    id              SERIAL  PRIMARY KEY,
-    redirect_from   INT     REFERENCES url_verify.url(id) DEFERRABLE INITIALLY DEFERRED,
-    item            INT     REFERENCES container.biblio_record_entry_bucket_item (id) DEFERRABLE INITIALLY DEFERRED,
-    url_selector    INT     REFERENCES url_verify.url_selector (id) DEFERRABLE INITIALLY DEFERRED,
-    session         INT     REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
-    tag             TEXT,
-    subfield        TEXT,
-    ord             INT,
-    full_url        TEXT    NOT NULL,
-    scheme          TEXT,
-    username        TEXT,
-    password        TEXT,
-    host            TEXT,
-    domain          TEXT,
-    tld             TEXT,
-    port            TEXT,
-    path            TEXT,
-    page            TEXT,
-    query           TEXT,
-    fragment        TEXT,
-    CONSTRAINT redirect_or_from_item CHECK (
-        redirect_from IS NOT NULL OR (
-            item         IS NOT NULL AND
-            url_selector IS NOT NULL AND
-            tag          IS NOT NULL AND
-            subfield     IS NOT NULL AND
-            ord          IS NOT NULL
-        )
-    )
-);
-
-CREATE TABLE url_verify.verification_attempt (
-    id          SERIAL                      PRIMARY KEY,
-    usr         INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
-    session     INT                         NOT NULL REFERENCES url_verify.session (id) DEFERRABLE INITIALLY DEFERRED,
-    start_time  TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
-    finish_time TIMESTAMP WITH TIME ZONE
-);
-CREATE TABLE url_verify.url_verification (
-    id          SERIAL                      PRIMARY KEY,
-    url         INT                         NOT NULL REFERENCES url_verify.url (id) DEFERRABLE INITIALLY DEFERRED,
-    attempt     INT                         NOT NULL REFERENCES url_verify.verification_attempt (id) DEFERRABLE INITIALLY DEFERRED,
-    req_time    TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
-    res_time    TIMESTAMP WITH TIME ZONE, 
-    res_code    INT                         CHECK (res_code BETWEEN 100 AND 999), -- we know > 599 will never be valid HTTP code, but we use 9XX for other stuff
-    res_text    TEXT, 
-    redirect_to INT                         REFERENCES url_verify.url (id) DEFERRABLE INITIALLY DEFERRED -- if redirected
-);
-
-CREATE TABLE config.filter_dialog_interface (
-    key         TEXT                        PRIMARY KEY,
-    description TEXT
-);
-
-CREATE TABLE config.filter_dialog_filter_set (
-    id          SERIAL                      PRIMARY KEY,
-    name        TEXT                        NOT NULL,
-    owning_lib  INT                         NOT NULL REFERENCES actor.org_unit (id) DEFERRABLE INITIALLY DEFERRED,
-    creator     INT                         NOT NULL REFERENCES actor.usr (id) DEFERRABLE INITIALLY DEFERRED,
-    create_time TIMESTAMP WITH TIME ZONE    NOT NULL DEFAULT NOW(),
-    interface   TEXT                        NOT NULL REFERENCES config.filter_dialog_interface (key) DEFERRABLE INITIALLY DEFERRED,
-    filters     TEXT                        NOT NULL CHECK (evergreen.is_json(filters)),
-    CONSTRAINT cfdfs_name_once_per_lib UNIQUE (name, owning_lib)
-);
-COMMIT;
diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.functions.url_verify.sql
deleted file mode 100644 (file)
index e80cb25..0000000
+++ /dev/null
@@ -1,98 +0,0 @@
-BEGIN;
-
-CREATE OR REPLACE FUNCTION url_verify.parse_url (url_in TEXT) RETURNS url_verify.url AS $$
-
-use Rose::URI;
-
-my $url_in = shift;
-my $url = Rose::URI->new($url_in);
-
-my %parts = map { $_ => $url->$_ } qw/scheme username password host port path query fragment/;
-
-$parts{full_url} = $url_in;
-($parts{domain} = $parts{host}) =~ s/^[^.]+\.//;
-($parts{tld} = $parts{domain}) =~ s/(?:[^.]+\.)+//;
-($parts{page} = $parts{path}) =~ s#(?:[^/]*/)+##;
-
-return \%parts;
-
-$$ LANGUAGE PLPERLU;
-
-CREATE OR REPLACE FUNCTION url_verify.ingest_url () RETURNS TRIGGER AS $$
-DECLARE
-    tmp_row url_verify.url%ROWTYPE;
-BEGIN
-    SELECT * INTO tmp_row FROM url_verify.parse_url(NEW.full_url);
-
-    NEW.scheme          := tmp_row.scheme;
-    NEW.username        := tmp_row.username;
-    NEW.password        := tmp_row.password;
-    NEW.host            := tmp_row.host;
-    NEW.domain          := tmp_row.domain;
-    NEW.tld             := tmp_row.tld;
-    NEW.port            := tmp_row.port;
-    NEW.path            := tmp_row.path;
-    NEW.page            := tmp_row.page;
-    NEW.query           := tmp_row.query;
-    NEW.fragment        := tmp_row.fragment;
-
-    RETURN NEW;
-END;
-$$ LANGUAGE PLPGSQL;
-
-CREATE TRIGGER ingest_url_tgr
-    BEFORE INSERT ON url_verify.url
-    FOR EACH ROW EXECUTE PROCEDURE url_verify.ingest_url(); 
-
-CREATE OR REPLACE FUNCTION url_verify.extract_urls ( session_id INT, item_id INT ) RETURNS INT AS $$
-DECLARE
-    last_seen_tag TEXT;
-    current_tag TEXT;
-    current_sf TEXT;
-    current_url TEXT;
-    current_ord INT;
-    current_url_pos INT;
-    current_selector url_verify.url_selector%ROWTYPE;
-BEGIN
-    current_ord := 1;
-
-    FOR current_selector IN SELECT * FROM url_verify.url_selector s WHERE s.session = session_id LOOP
-        current_url_pos := 1;
-        LOOP
-            SELECT  (XPATH(current_selector.xpath || '/text()', b.marc::XML))[current_url_pos]::TEXT INTO current_url
-              FROM  biblio.record_entry b
-                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
-              WHERE c.id = item_id;
-
-            EXIT WHEN current_url IS NULL;
-
-            SELECT  (XPATH(current_selector.xpath || '/../@tag', b.marc::XML))[current_url_pos]::TEXT INTO current_tag
-              FROM  biblio.record_entry b
-                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
-              WHERE c.id = item_id;
-
-            IF current_tag IS NULL THEN
-                current_tag := last_seen_tag;
-            ELSE
-                last_seen_tag := current_tag;
-            END IF;
-
-            SELECT  (XPATH(current_selector.xpath || '/@code', b.marc::XML))[current_url_pos]::TEXT INTO current_sf
-              FROM  biblio.record_entry b
-                    JOIN container.biblio_record_entry_bucket_item c ON (c.target_biblio_record_entry = b.id)
-              WHERE c.id = item_id;
-
-            INSERT INTO url_verify.url (session, item, url_selector, tag, subfield, ord, full_url)
-              VALUES ( session_id, item_id, current_selector.id, current_tag, current_sf, current_ord, current_url);
-
-            current_url_pos := current_url_pos + 1;
-            current_ord := current_ord + 1;
-        END LOOP;
-    END LOOP;
-
-    RETURN current_ord - 1;
-END;
-$$ LANGUAGE PLPGSQL;
-
-COMMIT;
-
diff --git a/Open-ILS/src/sql/Pg/upgrade/ZZZZ.data.url_verify.sql b/Open-ILS/src/sql/Pg/upgrade/ZZZZ.data.url_verify.sql
deleted file mode 100644 (file)
index 968361b..0000000
+++ /dev/null
@@ -1,193 +0,0 @@
-
--- NOTE: beware the use of bare perm IDs in the update_perm's below and in 
--- the 950 seed data file.  Update before merge to match current perm IDs! XXX
-
-BEGIN;
-
-INSERT INTO permission.perm_list (id, code, description) 
-    VALUES ( 
-        543, 
-        'URL_VERIFY',
-        oils_i18n_gettext(
-            543, 
-            'Allows a user to process and verify ULSs', 
-            'ppl', 
-            'description'
-        )
-    );
-
-
-INSERT INTO permission.perm_list (id, code, description) 
-    VALUES ( 
-        544, 
-        544,
-        oils_i18n_gettext(
-            544, 
-            'Allows a user to configure URL verification org unit settings',
-            'ppl', 
-            'description'
-        )
-    );
-
-
-INSERT INTO permission.perm_list (id, code, description) 
-    VALUES ( 
-        545, 
-        'SAVED_FILTER_DIALOG_FILTERS',
-        oils_i18n_gettext(
-            545, 
-            'Allows users to save and load sets of filters for filter dialogs, available in certain staff interfaces',
-            'ppl', 
-            'description'
-        )
-    );
-
-
-INSERT INTO config.settings_group (name, label)
-    VALUES (
-        'url_verify',
-        oils_i18n_gettext(
-            'url_verify',
-            'URL Verify',
-            'csg',
-            'label'
-        )
-    );
-
-INSERT INTO config.org_unit_setting_type
-    (name, grp, label, description, datatype, update_perm)
-    VALUES (
-        'url_verify.url_verification_delay',
-        'url_verify',
-        oils_i18n_gettext(
-            'url_verify.url_verification_delay',
-            'Number of seconds to wait between URL test attempts.',
-            'coust',
-            'label'
-        ),
-        oils_i18n_gettext(
-            'url_verify.url_verification_delay',
-            'Throttling mechanism for batch URL verification runs.  Each running process will wait this number of seconds after a URL test before performing the next.',
-            'coust',
-            'description'
-        ),
-        'integer',
-        544
-    );
-
-INSERT INTO config.org_unit_setting_type
-    (name, grp, label, description, datatype, update_perm)
-    VALUES (
-        'url_verify.url_verification_max_redirects',
-        'url_verify',
-        oils_i18n_gettext(
-            'url_verify.url_verification_max_redirects',
-            'Maximum redirect lookups',
-            'coust',
-            'label'
-        ),
-        oils_i18n_gettext(
-            'url_verify.url_verification_max_redirects',
-            'For URLs returning 3XX redirects, this is the maximum number of redirects we will follow before giving up.',
-            'coust',
-            'description'
-        ),
-        'integer',
-        544
-    );
-
-INSERT INTO config.org_unit_setting_type
-    (name, grp, label, description, datatype, update_perm)
-    VALUES (
-        'url_verify.url_verification_max_wait',
-        'url_verify',
-        oils_i18n_gettext(
-            'url_verify.url_verification_max_wait',
-            'Maximum wait time (in seconds) for a URL to lookup',
-            'coust',
-            'label'
-        ),
-        oils_i18n_gettext(
-            'url_verify.url_verification_max_wait',
-            'If we exceed the wait time, the URL is marked as a "timeout" and the system moves on to the next URL',
-            'coust',
-            'description'
-        ),
-        'integer',
-        544
-    );
-
-
-INSERT INTO config.org_unit_setting_type
-    (name, grp, label, description, datatype, update_perm)
-    VALUES (
-        'url_verify.verification_batch_size',
-        'url_verify',
-        oils_i18n_gettext(
-            'url_verify.verification_batch_size',
-            'Number of URLs to test in parallel',
-            'coust',
-            'label'
-        ),
-        oils_i18n_gettext(
-            'url_verify.verification_batch_size',
-            'URLs are tested in batches.  This number defines the size of each batch and it directly relates to the number of back-end processes performing URL verification.',
-            'coust',
-            'description'
-        ),
-        'integer',
-        544
-    );
-
-
-INSERT INTO config.filter_dialog_interface (key, description) VALUES (
-    'url_verify',
-    oils_i18n_gettext(
-        'url_verify',
-        'All Link Checker filter dialogs',
-        'cfdi',
-        'description'
-    )
-);
-
-
-INSERT INTO config.usr_setting_type (name,grp,opac_visible,label,description,datatype) VALUES (
-    'ui.grid_columns.url_verify.select_urls',
-    'gui',
-    FALSE,
-    oils_i18n_gettext(
-        'ui.grid_columns.url_verify.select_urls',
-        'Link Checker''s URL Selection interface''s saved columns',
-        'cust',
-        'label'
-    ),
-    oils_i18n_gettext(
-        'ui.grid_columns.url_verify.select_urls',
-        'Link Checker''s URL Selection interface''s saved columns',
-        'cust',
-        'description'
-    ),
-    'string'
-);
-
-INSERT INTO config.usr_setting_type (name,grp,opac_visible,label,description,datatype) VALUES (
-    'ui.grid_columns.url_verify.review_attempt',
-    'gui',
-    FALSE,
-    oils_i18n_gettext(
-        'ui.grid_columns.url_verify.review_attempt',
-        'Link Checker''s Review Attempt interface''s saved columns',
-        'cust',
-        'label'
-    ),
-    oils_i18n_gettext(
-        'ui.grid_columns.url_verify.review_attempt',
-        'Link Checker''s Review Attempt interface''s saved columns',
-        'cust',
-        'description'
-    ),
-    'string'
-);
-
-COMMIT;
-