From 5f1247385514bda1ce2b9bce06612b311bd71f11 Mon Sep 17 00:00:00 2001 From: Mike Rylander Date: Fri, 9 Feb 2018 15:19:31 -0500 Subject: [PATCH] LP#1744385: Various improvements based on testing * Improve author and publisher display fields * Reimplement Simple Record Extracts using Display Fields * Address potential failure in combined search collection * Provide useful helper function for inspecting transformed records * Implement MODS transforms specified by LoC documentation but not actually included in the stylesheet * Expand highlighting use to publisher, edition, and physical description * Reifying schema from updates Signed-off-by: Mike Rylander Signed-off-by: Kathy Lussier Signed-off-by: Dan Wells --- Open-ILS/src/sql/Pg/030.schema.metabib.sql | 40 +++++----- Open-ILS/src/sql/Pg/950.data.seed-values.sql | 24 +++++- Open-ILS/src/sql/Pg/999.functions.global.sql | 4 + Open-ILS/src/sql/Pg/reporter-schema.sql | 35 +++------ .../upgrade/WWWW.data.display-field-seed-data.sql | 85 ++++++++++++++++------ .../Pg/upgrade/XXXX.schema.highlight_search.sql | 6 ++ .../YYYY.data.mods-title-punctuation-change.sql | 82 ++++++++++++++++++++- .../src/templates/opac/parts/record/summary.tt2 | 20 ++++- 8 files changed, 223 insertions(+), 73 deletions(-) diff --git a/Open-ILS/src/sql/Pg/030.schema.metabib.sql b/Open-ILS/src/sql/Pg/030.schema.metabib.sql index 2f37b555d4..bc72c89601 100644 --- a/Open-ILS/src/sql/Pg/030.schema.metabib.sql +++ b/Open-ILS/src/sql/Pg/030.schema.metabib.sql @@ -245,25 +245,25 @@ CREATE VIEW metabib.wide_display_entry AS This VIEW expands as well-known display fields are added. */ SELECT bre.id AS source, - COALESCE(mcde_title.value, 'null') AS title, - COALESCE(mcde_author.value, 'null') AS author, - COALESCE(mcde_subject_geographic.value, 'null') AS subject_geographic, - COALESCE(mcde_subject_name.value, 'null') AS subject_name, - COALESCE(mcde_subject_temporal.value, 'null') AS subject_temporal, - COALESCE(mcde_subject_topic.value, 'null') AS subject_topic, - COALESCE(mcde_creators.value, 'null') AS creators, - COALESCE(mcde_isbn.value, 'null') AS isbn, - COALESCE(mcde_issn.value, 'null') AS issn, - COALESCE(mcde_upc.value, 'null') AS upc, - COALESCE(mcde_tcn.value, 'null') AS tcn, - COALESCE(mcde_edition.value, 'null') AS edition, - COALESCE(mcde_physical_description.value, 'null') AS physical_description, - COALESCE(mcde_publisher.value, 'null') AS publisher, - COALESCE(mcde_series_title.value, 'null') AS series_title, - COALESCE(mcde_abstract.value, 'null') AS abstract, - COALESCE(mcde_toc.value, 'null') AS toc, - COALESCE(mcde_pubdate.value, 'null') AS pubdate, - COALESCE(mcde_type_of_resource.value, 'null') AS type_of_resource + COALESCE(mcde_title.value, 'null')::TEXT AS title, + COALESCE(mcde_author.value, 'null')::TEXT AS author, + COALESCE(mcde_subject_geographic.value, 'null')::TEXT AS subject_geographic, + COALESCE(mcde_subject_name.value, 'null')::TEXT AS subject_name, + COALESCE(mcde_subject_temporal.value, 'null')::TEXT AS subject_temporal, + COALESCE(mcde_subject_topic.value, 'null')::TEXT AS subject_topic, + COALESCE(mcde_creators.value, 'null')::TEXT AS creators, + COALESCE(mcde_isbn.value, 'null')::TEXT AS isbn, + COALESCE(mcde_issn.value, 'null')::TEXT AS issn, + COALESCE(mcde_upc.value, 'null')::TEXT AS upc, + COALESCE(mcde_tcn.value, 'null')::TEXT AS tcn, + COALESCE(mcde_edition.value, 'null')::TEXT AS edition, + COALESCE(mcde_physical_description.value, 'null')::TEXT AS physical_description, + COALESCE(mcde_publisher.value, 'null')::TEXT AS publisher, + COALESCE(mcde_series_title.value, 'null')::TEXT AS series_title, + COALESCE(mcde_abstract.value, 'null')::TEXT AS abstract, + COALESCE(mcde_toc.value, 'null')::TEXT AS toc, + COALESCE(mcde_pubdate.value, 'null')::TEXT AS pubdate, + COALESCE(mcde_type_of_resource.value, 'null')::TEXT AS type_of_resource FROM biblio.record_entry bre LEFT JOIN metabib.compressed_display_entry mcde_title ON (bre.id = mcde_title.source AND mcde_title.name = 'title') @@ -1027,6 +1027,8 @@ BEGIN WHERE record = $1 AND metabib_field = $2 $$ USING bib_id, vfield, rdata; + WHEN OTHERS THEN + -- ignore and move on END; END LOOP; END; diff --git a/Open-ILS/src/sql/Pg/950.data.seed-values.sql b/Open-ILS/src/sql/Pg/950.data.seed-values.sql index 94ea0307be..a66a12d52b 100644 --- a/Open-ILS/src/sql/Pg/950.data.seed-values.sql +++ b/Open-ILS/src/sql/Pg/950.data.seed-values.sql @@ -307,6 +307,26 @@ VALUES ( ) ; +INSERT INTO config.metabib_field (id, field_class, name, format, + label, xpath, display_xpath, display_field, search_field, browse_field) +VALUES ( + 51, 'author', 'first_author', 'mods32', + oils_i18n_gettext(51, 'Author', 'cmf', 'label'), + $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']][1]$$, + $$//*[local-name()='namePart']$$, + TRUE, TRUE, FALSE +); + +INSERT INTO config.metabib_field (id, field_class, name, format, + label, xpath, display_xpath, display_field, search_field, browse_field) +VALUES ( + 52, 'keyword', 'origin_info', 'marcxml', + oils_i18n_gettext(52, 'Origin Info', 'cmf', 'label'), + $$//*[@tag='260']$$, + $$//*[local-name()='subfield' and contains('abc',@code)]$$, + TRUE, TRUE, FALSE +); + INSERT INTO config.metabib_field_virtual_map (real, virtual) SELECT id, 45 @@ -340,7 +360,7 @@ SELECT SETVAL('config.metabib_field_id_seq', GREATEST(1000, (SELECT MAX(id) FROM INSERT INTO config.display_field_map (name, field, multi) VALUES ('title', 6, FALSE), - ('author', 8, FALSE), + ('author', 51, FALSE), ('creators', 37, TRUE), ('subject', 16, TRUE), ('isbn', 18, TRUE), @@ -354,7 +374,7 @@ INSERT INTO config.display_field_map (name, field, multi) VALUES ('tcn', 26, FALSE), ('edition', 38, FALSE), ('physical_description',39, TRUE), - ('publisher', 40, FALSE), + ('publisher', 52, FALSE), ('abstract', 41, FALSE), ('toc', 42, FALSE), ('type_of_resource', 43, FALSE), diff --git a/Open-ILS/src/sql/Pg/999.functions.global.sql b/Open-ILS/src/sql/Pg/999.functions.global.sql index dd36aa9ecb..44157c120b 100644 --- a/Open-ILS/src/sql/Pg/999.functions.global.sql +++ b/Open-ILS/src/sql/Pg/999.functions.global.sql @@ -2178,4 +2178,8 @@ BEGIN END; $$ LANGUAGE PLPGSQL; +-- Handy function for transforming marc to a variant available on config.xml_transform +CREATE OR REPLACE FUNCTION evergreen.marc_to (marc text, xfrm text) RETURNS TEXT AS $$ + SELECT evergreen.xml_pretty_print(xslt_process($1,xslt)::XML)::TEXT FROM config.xml_transform WHERE name = $2; +$$ LANGUAGE SQL; diff --git a/Open-ILS/src/sql/Pg/reporter-schema.sql b/Open-ILS/src/sql/Pg/reporter-schema.sql index 28d9d44251..ce302a6f9b 100644 --- a/Open-ILS/src/sql/Pg/reporter-schema.sql +++ b/Open-ILS/src/sql/Pg/reporter-schema.sql @@ -154,37 +154,20 @@ SELECT r.id, r.quality, r.tcn_source, r.tcn_value, - CONCAT_WS(' ', FIRST(title.value),FIRST(title_np.val)) AS title, - FIRST(author.value) AS author, - STRING_AGG(DISTINCT publisher.value, ', ') AS publisher, - STRING_AGG(DISTINCT SUBSTRING(pubdate.value FROM $$\d+$$), ', ') AS pubdate, - CASE WHEN ARRAY_AGG( DISTINCT REPLACE(SUBSTRING(isbn.value FROM $$^\S+$$), '-', '') ) = '{NULL}' + evergreen.oils_json_to_text(d.title) AS title, + evergreen.oils_json_to_text(d.author) AS author, + evergreen.oils_json_to_text(d.publisher) AS publisher, + evergreen.oils_json_to_text(d.pubdate) AS pubdate, + CASE WHEN d.isbn = 'null' THEN NULL - ELSE ARRAY_AGG( DISTINCT REPLACE(SUBSTRING(isbn.value FROM $$^\S+$$), '-', '') ) + ELSE (SELECT ARRAY(SELECT json_array_elements_text(d.isbn::JSON))) END AS isbn, - CASE WHEN ARRAY_AGG( DISTINCT REGEXP_REPLACE(issn.value, E'^\\S*(\\d{4})[-\\s](\\d{3,4}x?)', E'\\1 \\2') ) = '{NULL}' + CASE WHEN d.issn = 'null' THEN NULL - ELSE ARRAY_AGG( DISTINCT REGEXP_REPLACE(issn.value, E'^\\S*(\\d{4})[-\\s](\\d{3,4}x?)', E'\\1 \\2') ) + ELSE (SELECT ARRAY(SELECT json_array_elements_text(d.issn::JSON))) END AS issn FROM biblio.record_entry r - LEFT JOIN metabib.full_rec title ON (r.id = title.record AND title.tag = '245' AND title.subfield = 'a') - LEFT JOIN ( -- Grab 245 N and P subfields in the order that they appear. - SELECT b.record, string_agg(val, ' ') AS val FROM ( - SELECT title_np.record, title_np.value AS val - FROM metabib.full_rec title_np - WHERE - title_np.tag = '245' - AND title_np.subfield IN ('p','n') - ORDER BY title_np.id - ) b - GROUP BY 1 - ) title_np ON (title_np.record=r.id) - LEFT JOIN metabib.full_rec author ON (r.id = author.record AND author.tag IN ('100','110','111') AND author.subfield = 'a') - LEFT JOIN metabib.full_rec publisher ON (r.id = publisher.record AND (publisher.tag = '260' OR (publisher.tag = '264' AND publisher.ind2 = '1')) AND publisher.subfield = 'b') - LEFT JOIN metabib.full_rec pubdate ON (r.id = pubdate.record AND (pubdate.tag = '260' OR (pubdate.tag = '264' AND pubdate.ind2 = '1')) AND pubdate.subfield = 'c') - LEFT JOIN metabib.full_rec isbn ON (r.id = isbn.record AND isbn.tag IN ('024', '020') AND isbn.subfield IN ('a','z')) - LEFT JOIN metabib.full_rec issn ON (r.id = issn.record AND issn.tag = '022' AND issn.subfield = 'a') - GROUP BY 1,2,3,4,5; + JOIN metabib.wide_display_entry d ON (r.id = d.source); CREATE TABLE reporter.materialized_simple_record AS SELECT * FROM reporter.old_super_simple_record WHERE 1=0; ALTER TABLE reporter.materialized_simple_record ADD PRIMARY KEY (id); diff --git a/Open-ILS/src/sql/Pg/upgrade/WWWW.data.display-field-seed-data.sql b/Open-ILS/src/sql/Pg/upgrade/WWWW.data.display-field-seed-data.sql index b44afd4ed4..fae9b210f9 100644 --- a/Open-ILS/src/sql/Pg/upgrade/WWWW.data.display-field-seed-data.sql +++ b/Open-ILS/src/sql/Pg/upgrade/WWWW.data.display-field-seed-data.sql @@ -99,6 +99,26 @@ VALUES ( ) ; +INSERT INTO config.metabib_field (id, field_class, name, format, + label, xpath, display_xpath, display_field, search_field, browse_field) +VALUES ( + 51, 'author', 'first_author', 'mods32', + oils_i18n_gettext(51, 'Author', 'cmf', 'label'), + $$//mods32:mods/mods32:name[mods32:role/mods32:roleTerm[text()='creator']][1]$$, + $$//*[local-name()='namePart']$$, + TRUE, TRUE, FALSE +); + +INSERT INTO config.metabib_field (id, field_class, name, format, + label, xpath, display_xpath, display_field, search_field, browse_field) +VALUES ( + 52, 'keyword', 'origin_info', 'marcxml', + oils_i18n_gettext(52, 'Origin Info', 'cmf', 'label'), + $$//*[@tag='260']$$, + $$//*[local-name()='subfield' and contains('abc',@code)]$$, + TRUE, TRUE, FALSE +); + -- Modify existing config.metabib_field entries @@ -126,38 +146,40 @@ INSERT INTO config.display_field_map (name, field, multi) VALUES ('tcn', 26, FALSE), ('edition', 38, FALSE), ('physical_description',39, TRUE), - ('publisher', 40, FALSE), + ('publisher', 52, FALSE), ('abstract', 41, FALSE), ('toc', 42, FALSE), ('type_of_resource', 43, FALSE), ('pubdate', 44, FALSE) ; +UPDATE config.display_field_map SET field = 51 WHERE name = 'author'; + -- Add a column to wide-display-entry per well-known field -DROP VIEW metabib.wide_display_entry; +DROP VIEW IF EXISTS metabib.wide_display_entry; CREATE VIEW metabib.wide_display_entry AS SELECT bre.id AS source, - COALESCE(mcde_title.value, 'null') AS title, - COALESCE(mcde_author.value, 'null') AS author, - COALESCE(mcde_subject_geographic.value, 'null') AS subject_geographic, - COALESCE(mcde_subject_name.value, 'null') AS subject_name, - COALESCE(mcde_subject_temporal.value, 'null') AS subject_temporal, - COALESCE(mcde_subject_topic.value, 'null') AS subject_topic, - COALESCE(mcde_creators.value, 'null') AS creators, - COALESCE(mcde_isbn.value, 'null') AS isbn, - COALESCE(mcde_issn.value, 'null') AS issn, - COALESCE(mcde_upc.value, 'null') AS upc, - COALESCE(mcde_tcn.value, 'null') AS tcn, - COALESCE(mcde_edition.value, 'null') AS edition, - COALESCE(mcde_physical_description.value, 'null') AS physical_description, - COALESCE(mcde_publisher.value, 'null') AS publisher, - COALESCE(mcde_series_title.value, 'null') AS series_title, - COALESCE(mcde_abstract.value, 'null') AS abstract, - COALESCE(mcde_toc.value, 'null') AS toc, - COALESCE(mcde_pubdate.value, 'null') AS pubdate, - COALESCE(mcde_type_of_resource.value, 'null') AS type_of_resource + COALESCE(mcde_title.value, 'null')::TEXT AS title, + COALESCE(mcde_author.value, 'null')::TEXT AS author, + COALESCE(mcde_subject_geographic.value, 'null')::TEXT AS subject_geographic, + COALESCE(mcde_subject_name.value, 'null')::TEXT AS subject_name, + COALESCE(mcde_subject_temporal.value, 'null')::TEXT AS subject_temporal, + COALESCE(mcde_subject_topic.value, 'null')::TEXT AS subject_topic, + COALESCE(mcde_creators.value, 'null')::TEXT AS creators, + COALESCE(mcde_isbn.value, 'null')::TEXT AS isbn, + COALESCE(mcde_issn.value, 'null')::TEXT AS issn, + COALESCE(mcde_upc.value, 'null')::TEXT AS upc, + COALESCE(mcde_tcn.value, 'null')::TEXT AS tcn, + COALESCE(mcde_edition.value, 'null')::TEXT AS edition, + COALESCE(mcde_physical_description.value, 'null')::TEXT AS physical_description, + COALESCE(mcde_publisher.value, 'null')::TEXT AS publisher, + COALESCE(mcde_series_title.value, 'null')::TEXT AS series_title, + COALESCE(mcde_abstract.value, 'null')::TEXT AS abstract, + COALESCE(mcde_toc.value, 'null')::TEXT AS toc, + COALESCE(mcde_pubdate.value, 'null')::TEXT AS pubdate, + COALESCE(mcde_type_of_resource.value, 'null')::TEXT AS type_of_resource FROM biblio.record_entry bre LEFT JOIN metabib.compressed_display_entry mcde_title ON (bre.id = mcde_title.source AND mcde_title.name = 'title') @@ -207,5 +229,26 @@ CREATE VIEW metabib.wide_display_entry AS AND mcde_type_of_resource.name = 'type_of_resource') ; +CREATE OR REPLACE VIEW reporter.old_super_simple_record AS +SELECT r.id, + r.fingerprint, + r.quality, + r.tcn_source, + r.tcn_value, + evergreen.oils_json_to_text(d.title) AS title, + evergreen.oils_json_to_text(d.author) AS author, + evergreen.oils_json_to_text(d.publisher) AS publisher, + evergreen.oils_json_to_text(d.pubdate) AS pubdate, + CASE WHEN d.isbn = 'null' + THEN NULL + ELSE (SELECT ARRAY(SELECT json_array_elements_text(d.isbn::JSON))) + END AS isbn, + CASE WHEN d.issn = 'null' + THEN NULL + ELSE (SELECT ARRAY(SELECT json_array_elements_text(d.issn::JSON))) + END AS issn + FROM biblio.record_entry r + JOIN metabib.wide_display_entry d ON (r.id = d.source); + COMMIT; diff --git a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.highlight_search.sql b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.highlight_search.sql index 2fc357b7bc..7a4e5517a6 100644 --- a/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.highlight_search.sql +++ b/Open-ILS/src/sql/Pg/upgrade/XXXX.schema.highlight_search.sql @@ -322,6 +322,8 @@ BEGIN WHERE record = $1 AND metabib_field = $2 $$ USING bib_id, vfield, rdata; + WHEN OTHERS THEN + -- ignore and move on END; END LOOP; END; @@ -555,5 +557,9 @@ BEGIN END; $function$ LANGUAGE plpgsql; +CREATE OR REPLACE FUNCTION evergreen.marc_to (marc text, xfrm text) RETURNS TEXT AS $$ + SELECT evergreen.xml_pretty_print(xslt_process($1,xslt)::XML)::TEXT FROM config.xml_transform WHERE name = $2; +$$ LANGUAGE SQL; + COMMIT; diff --git a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.mods-title-punctuation-change.sql b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.mods-title-punctuation-change.sql index 5efb1ff967..eb7e34e7e8 100644 --- a/Open-ILS/src/sql/Pg/upgrade/YYYY.data.mods-title-punctuation-change.sql +++ b/Open-ILS/src/sql/Pg/upgrade/YYYY.data.mods-title-punctuation-change.sql @@ -1358,6 +1358,45 @@ Added Log Comment + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1467,7 +1506,7 @@ Added Log Comment - + @@ -4717,6 +4756,45 @@ Revision 1.2 - Added Log Comment 2003/03/24 19:37:42 ckeith + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -4823,7 +4901,7 @@ Revision 1.2 - Added Log Comment 2003/03/24 19:37:42 ckeith + + select="marc:datafield[@tag=501 or @tag=507 or @tag=513 or @tag=514 or @tag=515 or @tag=516 or @tag=522 or @tag=524 or @tag=525 or @tag=526 or @tag=535 or @tag=536 or @tag=540 or @tag=541 or @tag=544 or @tag=545 or @tag=546 or @tag=547 or @tag=550 or @tag=552 or @tag=555 or @tag=556 or @tag=561 or @tag=562 or @tag=565 or @tag=567 or @tag=580 or @tag=581 or @tag=584 or @tag=585 or @tag=586]">+ diff --git a/Open-ILS/src/templates/opac/parts/record/summary.tt2 b/Open-ILS/src/templates/opac/parts/record/summary.tt2 index 15d2a37fb0..374182fa87 100644 --- a/Open-ILS/src/templates/opac/parts/record/summary.tt2 +++ b/Open-ILS/src/templates/opac/parts/record/summary.tt2 @@ -333,13 +333,22 @@ END; [%- END %] [%- END %] - [%- IF attrs.phys_desc %] + [%- IF attrs.hl.physical_description.size %] +
  • + [% l("Physical Description:") %] + [% attrs.hl.physical_description.join('
    ') %]     +
    • + [%- ELSIF attrs.phys_desc %]
  • [% l("Physical Description:") %] [% attrs.phys_desc | html %]
    • [%- END %] - [%- IF attrs.edition %] + [%- IF attrs.hl.edition %] +
  • + [% l("Edition:") %] + [% attrs.hl.edition %] + [%- ELSIF attrs.edition %]
  • [% l("Edition:") %] [% attrs.edition | html %] @@ -360,7 +369,12 @@ END; -%]
    • [%- END %] - [%- IF attrs.publisher %] + [%- IF attrs.hl.publisher %] +
  • + [% l("Publisher:") %] + [% attrs.hl.publisher %] +
    • + [%- ELSIF attrs.publisher %]
  • [% l("Publisher:") %] -- 2.11.0