--- /dev/null
+= Elasticsearch Examples
+
+== Bib Transform Testing
+
+[source,sh]
+----------------------------------------------------------------------------
+sudo apt install xsltproc
+
+xsltproc ../../xsl/elastic-bib-transform.xsl bib-248-marc.xml
+
+xsltproc ../../xsl/elastic-bib-transform.xsl bib-233-marc.xml
+----------------------------------------------------------------------------
+
+
--- /dev/null
+<?xml version="1.0"?>
+<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.loc.gov/MARC21/slim" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader>07649cim a2200913 i 4500</leader>
+ <controlfield tag="001">233</controlfield>
+ <controlfield tag="003">CONS</controlfield>
+ <controlfield tag="005">20140128084328.0</controlfield>
+ <controlfield tag="008">140128s2013 nyuopk|zqdefhi n | ita d</controlfield>
+ <datafield tag="010" ind1=" " ind2=" ">
+ <subfield code="a"> 2013565186</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">9781480328532</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">1480328537</subfield>
+ </datafield>
+ <datafield tag="024" ind1="1" ind2=" ">
+ <subfield code="a">884088883249</subfield>
+ </datafield>
+ <datafield tag="028" ind1="3" ind2="2">
+ <subfield code="a">HL50498721</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(bk.)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">HL50490487</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(cd.)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">HL50486260</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(cd.)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">63011108</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(diction coach 1)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">63011109</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(diction coach 2)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">63014792</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(CD 1)</subfield>
+ </datafield>
+ <datafield tag="028" ind1="0" ind2="2">
+ <subfield code="a">63014793</subfield>
+ <subfield code="b">Hal Leonard</subfield>
+ <subfield code="q">(CD 2)</subfield>
+ </datafield>
+ <datafield tag="035" ind1=" " ind2=" ">
+ <subfield code="a">(OCoLC)ocn826076986</subfield>
+ </datafield>
+ <datafield tag="035" ind1=" " ind2=" ">
+ <subfield code="a">(OCoLC)826076986</subfield>
+ </datafield>
+ <datafield tag="040" ind1=" " ind2=" ">
+ <subfield code="a">YDXCP</subfield>
+ <subfield code="b">eng</subfield>
+ <subfield code="e">rda</subfield>
+ <subfield code="c">YDXCP</subfield>
+ <subfield code="d">CLE</subfield>
+ <subfield code="d">NUI</subfield>
+ <subfield code="d">MYG</subfield>
+ <subfield code="d">DLC</subfield>
+ </datafield>
+ <datafield tag="041" ind1="0" ind2=" ">
+ <subfield code="a">ita</subfield>
+ <subfield code="a">ger</subfield>
+ <subfield code="a">fre</subfield>
+ <subfield code="a">eng</subfield>
+ <subfield code="e">ita</subfield>
+ <subfield code="e">ger</subfield>
+ <subfield code="e">fre</subfield>
+ <subfield code="e">eng</subfield>
+ <subfield code="g">eng</subfield>
+ </datafield>
+ <datafield tag="042" ind1=" " ind2=" ">
+ <subfield code="a">lccopycat</subfield>
+ </datafield>
+ <datafield tag="048" ind1=" " ind2=" ">
+ <subfield code="b">vf01</subfield>
+ <subfield code="a">ka01</subfield>
+ </datafield>
+ <datafield tag="050" ind1="0" ind2="0">
+ <subfield code="a">M1507.A+</subfield>
+ </datafield>
+ <datafield tag="100" ind1="0" ind2="0">
+ <subfield code="a">Pickins, Slim</subfield>
+ <subfield code="b">More Stuff</subfield>
+ </datafield>
+ <datafield tag="245" ind1="0" ind2="4">
+ <subfield code="a">The Arias for bass :</subfield>
+ <subfield code="b">complete package : with diction coach and accompaniment CDs /</subfield>
+ <subfield code="c">compiled and edited by Robert L. Larsen.</subfield>
+ </datafield>
+ <datafield tag="264" ind1=" " ind2="1">
+ <subfield code="a">New York, NY :</subfield>
+ <subfield code="b">G. Schirmer, Inc.,</subfield>
+ <subfield code="c">2013.</subfield>
+ </datafield>
+ <datafield tag="264" ind1=" " ind2="2">
+ <subfield code="a">Milwaukee, WI :</subfield>
+ <subfield code="b">Distributed by Hal Leonard Corporation</subfield>
+ </datafield>
+ <datafield tag="300" ind1=" " ind2=" ">
+ <subfield code="a">1 score (263 pages) ;</subfield>
+ <subfield code="c">31 cm +</subfield>
+ <subfield code="e">4 sound discs (digital ; 4 3/4 in.)</subfield>
+ </datafield>
+ <datafield tag="336" ind1=" " ind2=" ">
+ <subfield code="a">notated music</subfield>
+ <subfield code="b">ntm</subfield>
+ <subfield code="2">rdacontent</subfield>
+ </datafield>
+ <datafield tag="336" ind1=" " ind2=" ">
+ <subfield code="a">performed music</subfield>
+ <subfield code="b">prm</subfield>
+ <subfield code="2">rdacontent</subfield>
+ </datafield>
+ <datafield tag="337" ind1=" " ind2=" ">
+ <subfield code="a">unmediated</subfield>
+ <subfield code="b">n</subfield>
+ <subfield code="2">rdamedia</subfield>
+ </datafield>
+ <datafield tag="337" ind1=" " ind2=" ">
+ <subfield code="a">audio</subfield>
+ <subfield code="b">s</subfield>
+ <subfield code="2">rdamedia</subfield>
+ </datafield>
+ <datafield tag="338" ind1=" " ind2=" ">
+ <subfield code="a">volume</subfield>
+ <subfield code="b">nc</subfield>
+ <subfield code="2">rdacarrier</subfield>
+ </datafield>
+ <datafield tag="338" ind1=" " ind2=" ">
+ <subfield code="a">audio disc</subfield>
+ <subfield code="b">sd</subfield>
+ <subfield code="2">rdacarrier</subfield>
+ </datafield>
+ <datafield tag="490" ind1="1" ind2=" ">
+ <subfield code="a">G. Schirmer opera anthology</subfield>
+ </datafield>
+ <datafield tag="546" ind1=" " ind2=" ">
+ <subfield code="b">staff notation</subfield>
+ </datafield>
+ <datafield tag="546" ind1=" " ind2=" ">
+ <subfield code="a">Italian, French, German, and English words; non-English texts also printed with English translations.</subfield>
+ </datafield>
+ <datafield tag="500" ind1=" " ind2=" ">
+ <subfield code="a">Opera arias; acc. arr. for piano.</subfield>
+ </datafield>
+ <datafield tag="500" ind1=" " ind2=" ">
+ <subfield code="a">William Billingham, pianist on CDs.</subfield>
+ </datafield>
+ <datafield tag="500" ind1=" " ind2=" ">
+ <subfield code="a">disc 1-2 diction coach -- disc 3-4 accompaniment CDs.</subfield>
+ </datafield>
+ <datafield tag="505" ind1="0" ind2=" ">
+ <subfield code="a">Il barbiere di Siviglia. La calunnia / Gioachino Rossini -- La Bohè̀me. Vecchia zimarra, senti / Giacomo Puccini -- La Cenerentola. Miei rampolli femminini / Gioachino Rossini -- Don Giovanni. Madamina! Il catalogo è questo / Wolfgang Amadeus Mozart -- Don Pasquale. Ah! Un foco insolito / Gaetano Donizetti -- Die Entführung aus dem Serail. O, wie will ich triumphiren / Wolfgang Amadeus Mozart -- Ernani. Infelice! E tuo credevi / Giuseppe Verdi -- Eugene Onegin. Gremin's aria / Pyotr Il'yich Tchaikovsky -- Faust. Le veau d'or ; Vous qui faites l'endormie / Charles Gounod -- Der Freischütz. Schweig'! Schweig'! Damit dich niemand warnt / Carl Maria von Weber -- Les huguenots. Pour le couvents c'est fini (Piff, paff) / Giacomo Meyerbeer -- La jolie fille de Perth. Quand la flamme de l'amour / Georges Bizet -- Lucia di Lammermoor. Dalle stanze ove Lucia / Gaetano Donizetti -- Die lustigen Weiber von Windsor. Als Büblein klein / Otto Nicolai -- Macbeth. Come dal ciel precipita / Giuseppe Verdi -- Manon. Épouse quelque brave fille / Jules Massenet -- The mother of us all. What what is it / Virgil Thomson -- Le nozze di Figaro. La vendetta ; Se vuol ballare ; Non più andrai ; Aprite un po' quegl'occhi / Wolfgang Amadeus Mozart -- Simon Boccanegra. Il lacerato spirito / Giuseppe Verdi -- La sonnambula. Vi ravviso / Vincenzo Bellini -- Street scene. Let things be like they always was / Kurt Weill -- I vespri siciliani. O tu, Palermo / Giuseppe Verdi -- Die Zauberflöte. O Isis und Osiris ; In diesen heil'gen Hallen / Wolfgang Amadeus Mozart.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Operas</subfield>
+ <subfield code="v">Excerpts</subfield>
+ <subfield code="v">Vocal scores with piano.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Recorded accompaniments (Low voice)</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2=" ">
+ <subfield code="a">Larsen, Robert L.,</subfield>
+ <subfield code="d">1934-</subfield>
+ <subfield code="e">editor,</subfield>
+ <subfield code="e">compiler.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2=" ">
+ <subfield code="a">Billingham, William,</subfield>
+ <subfield code="e">performer.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Rossini, Gioacchino,</subfield>
+ <subfield code="d">1792-1868.</subfield>
+ <subfield code="t">Barbiere di Siviglia.</subfield>
+ <subfield code="p">Calunnia è un venticello.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Puccini, Giacomo,</subfield>
+ <subfield code="d">1858-1924.</subfield>
+ <subfield code="t">Bohème.</subfield>
+ <subfield code="p">Vecchia zimarra.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Rossini, Gioacchino,</subfield>
+ <subfield code="d">1792-1868.</subfield>
+ <subfield code="t">Cenerentola.</subfield>
+ <subfield code="p">Miei rampolli femminini.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Don Giovanni.</subfield>
+ <subfield code="p">Madamina, il catalogo è questo.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Donizetti, Gaetano,</subfield>
+ <subfield code="d">1797-1848.</subfield>
+ <subfield code="t">Don Pasquale.</subfield>
+ <subfield code="p">Foco insolito.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Entführung aus dem Serail.</subfield>
+ <subfield code="p">Ha! wie will ich triumphieren.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Verdi, Giuseppe,</subfield>
+ <subfield code="d">1813-1901.</subfield>
+ <subfield code="t">Ernani.</subfield>
+ <subfield code="p">Infelice! e tu credevi.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Tchaikovsky, Peter Ilich,</subfield>
+ <subfield code="d">1840-1893.</subfield>
+ <subfield code="t">Evgeniĭ Onegin.</subfield>
+ <subfield code="p">Arii︠a︡ kni︠a︡zi︠a︡.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Gounod, Charles,</subfield>
+ <subfield code="d">1818-1893.</subfield>
+ <subfield code="t">Faust.</subfield>
+ <subfield code="p">Veau d'or est toujours debout.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Gounod, Charles,</subfield>
+ <subfield code="d">1818-1893.</subfield>
+ <subfield code="t">Faust.</subfield>
+ <subfield code="p">Vous qui faites l'endormie.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Weber, Carl Maria von,</subfield>
+ <subfield code="d">1786-1826.</subfield>
+ <subfield code="t">Freischütz.</subfield>
+ <subfield code="p">Schweig', schweig'! damit dich niemand warnt.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Meyerbeer, Giacomo,</subfield>
+ <subfield code="d">1791-1864.</subfield>
+ <subfield code="t">Huguenots.</subfield>
+ <subfield code="p">Piff, paff.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Bizet, Georges,</subfield>
+ <subfield code="d">1838-1875.</subfield>
+ <subfield code="t">Jolie fille de Perth.</subfield>
+ <subfield code="p">Quand la flamme de l'amour.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Donizetti, Gaetano,</subfield>
+ <subfield code="d">1797-1848.</subfield>
+ <subfield code="t">Lucia di Lammermoor.</subfield>
+ <subfield code="p">Dalle stanze ove Lucia.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Nicolai, Otto,</subfield>
+ <subfield code="d">1810-1849.</subfield>
+ <subfield code="t">Lustigen Weiber von Windsor.</subfield>
+ <subfield code="p">Als Büblein klein.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Verdi, Giuseppe,</subfield>
+ <subfield code="d">1813-1901.</subfield>
+ <subfield code="t">Macbeth.</subfield>
+ <subfield code="p">Come dal ciel precipita.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Massenet, Jules,</subfield>
+ <subfield code="d">1842-1912.</subfield>
+ <subfield code="t">Manon.</subfield>
+ <subfield code="p">Épouse quelque brave fille.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Thomson, Virgil,</subfield>
+ <subfield code="d">1896-1989.</subfield>
+ <subfield code="t">Mother of us all.</subfield>
+ <subfield code="p">What what is it.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Nozze di Figaro.</subfield>
+ <subfield code="p">Vendetta.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Nozze di Figaro.</subfield>
+ <subfield code="p">Se vuol ballare, signor contino.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Nozze di Figaro.</subfield>
+ <subfield code="p">Non più andrai farfallone.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Nozze di Figaro.</subfield>
+ <subfield code="p">Aprite un po' quegl' occhi.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Verdi, Giuseppe,</subfield>
+ <subfield code="d">1813-1901.</subfield>
+ <subfield code="t">Simon Boccanegra.</subfield>
+ <subfield code="p">Lacerato spirito.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Bellini, Vincenzo,</subfield>
+ <subfield code="d">1801-1835.</subfield>
+ <subfield code="t">Sonnambula.</subfield>
+ <subfield code="p">Vi ravviso, o luoghi ameni.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Weill, Kurt,</subfield>
+ <subfield code="d">1900-1950.</subfield>
+ <subfield code="t">Street scene.</subfield>
+ <subfield code="p">Let things be like they always was.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Verdi, Giuseppe,</subfield>
+ <subfield code="d">1813-1901.</subfield>
+ <subfield code="t">Vêpres siciliennes.</subfield>
+ <subfield code="p">Et toi Palerme.</subfield>
+ <subfield code="l">Italian.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Zauberflöte.</subfield>
+ <subfield code="p">O Isis und Osiris (Aria and chorus)</subfield>
+ <subfield code="p">O Isis und Osiris.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="700" ind1="1" ind2="2">
+ <subfield code="i">Contains (expression):</subfield>
+ <subfield code="a">Mozart, Wolfgang Amadeus,</subfield>
+ <subfield code="d">1756-1791.</subfield>
+ <subfield code="t">Zauberflöte.</subfield>
+ <subfield code="p">In diesen heil'gen Hallen.</subfield>
+ <subfield code="s">Vocal score.</subfield>
+ </datafield>
+ <datafield tag="830" ind1=" " ind2="0">
+ <subfield code="a">G. Schirmer opera anthology.</subfield>
+ </datafield>
+ <datafield tag="906" ind1=" " ind2=" ">
+ <subfield code="a">0</subfield>
+ <subfield code="b">par</subfield>
+ <subfield code="c">copycat</subfield>
+ <subfield code="d">2</subfield>
+ <subfield code="e">ncip</subfield>
+ <subfield code="f">20</subfield>
+ <subfield code="g">y-genmusic</subfield>
+ </datafield>
+ <datafield tag="925" ind1="0" ind2=" ">
+ <subfield code="a">acquire</subfield>
+ <subfield code="b">2 shelf copies</subfield>
+ <subfield code="x">policy default</subfield>
+ </datafield>
+ <datafield tag="955" ind1=" " ind2=" ">
+ <subfield code="a">vl34 2014-01-28 z-client</subfield>
+ <subfield code="i">vl34 2014-01-28</subfield>
+ <subfield code="e">vl34 2014-01-28 4 sound disc to MBRS for shelf label</subfield>
+ <subfield code="t">vl34 2014-01-28 copy 2, 4 sound disc to MBRS for shelf label</subfield>
+ </datafield>
+ <datafield tag="901" ind1=" " ind2=" ">
+ <subfield code="a">233</subfield>
+ <subfield code="b">AUTOGEN</subfield>
+ <subfield code="c">233</subfield>
+ <subfield code="t">biblio</subfield>
+ </datafield>
+ <datafield tag="998" ind1=" " ind2=" ">
+ <subfield code="d">v</subfield>
+ </datafield>
+
+</record>
--- /dev/null
+<?xml version="1.0"?>
+<record xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.loc.gov/MARC21/slim" xsi:schemaLocation="http://www.loc.gov/MARC21/slim http://www.loc.gov/standards/marcxml/schema/MARC21slim.xsd">
+ <leader>00975pam a2200337 a 4500</leader>
+ <controlfield tag="001">248</controlfield>
+ <controlfield tag="003">CONS</controlfield>
+ <controlfield tag="005">20110823130500.0</controlfield>
+ <controlfield tag="008">110422s2011 nyu 000 1 eng </controlfield>
+ <datafield tag="010" ind1=" " ind2=" ">
+ <subfield code="a"> 2011015247</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">9780307887436 :</subfield>
+ <subfield code="c">$24.00</subfield>
+ </datafield>
+ <datafield tag="020" ind1=" " ind2=" ">
+ <subfield code="a">030788743X :</subfield>
+ <subfield code="c">$24.00</subfield>
+ </datafield>
+ <datafield tag="035" ind1=" " ind2=" ">
+ <subfield code="a">(DLC) 2011015247</subfield>
+ </datafield>
+ <datafield tag="040" ind1=" " ind2=" ">
+ <subfield code="a">DLC</subfield>
+ <subfield code="c">DLC</subfield>
+ <subfield code="d">NjBwBT</subfield>
+ <subfield code="d">GCmBT</subfield>
+ </datafield>
+ <datafield tag="042" ind1=" " ind2=" ">
+ <subfield code="a">pcc</subfield>
+ </datafield>
+ <datafield tag="050" ind1="0" ind2="0">
+ <subfield code="a">PS3603.L548</subfield>
+ <subfield code="b">R43 2011</subfield>
+ </datafield>
+ <datafield tag="082" ind1="0" ind2="0">
+ <subfield code="a">813/.6</subfield>
+ <subfield code="2">22</subfield>
+ </datafield>
+ <datafield tag="100" ind1="1" ind2=" ">
+ <subfield code="a">Cline, Ernest.</subfield>
+ </datafield>
+ <datafield tag="245" ind1="1" ind2="0">
+ <subfield code="a">Ready player one /</subfield>
+ <subfield code="c">Ernest Cline.</subfield>
+ </datafield>
+ <datafield tag="250" ind1=" " ind2=" ">
+ <subfield code="a">1st ed.</subfield>
+ </datafield>
+ <datafield tag="260" ind1=" " ind2=" ">
+ <subfield code="a">New York :</subfield>
+ <subfield code="b">Crown Publishers,</subfield>
+ <subfield code="c">c2011.</subfield>
+ </datafield>
+ <datafield tag="300" ind1=" " ind2=" ">
+ <subfield code="a">374 p. ;</subfield>
+ <subfield code="c">25 cm.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Regression (Civilization)</subfield>
+ <subfield code="v">Fiction.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Virtual reality</subfield>
+ <subfield code="v">Fiction.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Utopias</subfield>
+ <subfield code="v">Fiction.</subfield>
+ </datafield>
+ <datafield tag="650" ind1=" " ind2="0">
+ <subfield code="a">Puzzles</subfield>
+ <subfield code="v">Fiction.</subfield>
+ </datafield>
+ <datafield tag="655" ind1=" " ind2="7">
+ <subfield code="a">Fantasy fiction.</subfield>
+ <subfield code="2">gsafd</subfield>
+ </datafield>
+ <datafield tag="850" ind1=" " ind2=" ">
+ <subfield code="b">1</subfield>
+ </datafield>
+ <datafield tag="901" ind1=" " ind2=" ">
+ <subfield code="a">248</subfield>
+ <subfield code="b">AUTOGEN</subfield>
+ <subfield code="c">248</subfield>
+ <subfield code="t">biblio</subfield>
+ </datafield>
+</record>
--- /dev/null
+<?xml version="1.0" encoding="UTF-8" ?>
+
+<elasticsearch xmlns="http://evergreen-ils.org/spec/elasticsearch/v1">
+
+ <index class="bib-search">
+
+ <transform>/openils/var/xsl/elastic-bib-transform.xsl</transform>
+ <number_of_shards>1</number_of_shards>
+
+ <fields>
+ <!--
+ Attributes:
+ search_group="string" | optional; for grouped search fields (e.g. title)
+ name="string" | required
+ search_field="boolean" | defaults to false
+ facet_field="boolean" | defaults to false
+ filter="boolean" | defaults to false
+ sorter="boolean" | defaults to false
+ weight="integer" | defaults to 1
+ -->
+
+ <field search_group="title" name="maintitle" search_field="true" weight="10"/>
+ <field search_group="title" name="combined" search_field="true"/>
+ <field search_group="author" name="personal" search_field="true" weight="5"/>
+ <field search_group="author" name="combined" search_field="true" facet_field="true"/>
+ <field search_group="subject" name="combined" search_field="true" facet_field="true"/>
+ <field search_group="series" name="seriestitle" search_field="true" facet_field="true"/>
+ <field search_group="keyword" name="title" search_field="true" weight="10"/>
+ <field search_group="keyword" name="author" search_field="true" weight="5"/>
+ <field search_group="keyword" name="keyword" search_field="true"/>
+ <field search_group="keyword" name="publisher" search_field="true"/>
+ <field search_group="identifier" name="bibcn" search_field="true"/>
+ <field search_group="identifier" name="isbn" search_field="true"/>
+ <field search_group="identifier" name="issn" search_field="true"/>
+ <field search_group="identifier" name="lccn" search_field="true"/>
+ <field search_group="identifier" name="sudoc" search_field="true"/>
+ <field search_group="identifier" name="tech_number" search_field="true"/>
+ <field search_group="identifier" name="upc" search_field="true"/>
+ <field search_group="identifier" name="tcn" search_field="true"/>
+ <field search_group="identifier" name="bibid" search_field="true"/>
+
+ <!-- filters -->
+ <field name="mattype" filter="true"/>
+ <field name="audience" filter="true"/>
+ <field name="bib_level" filter="true"/>
+ <field name="date1" filter="true"/>
+ <field name="date2" filter="true"/>
+ <field name="item_form" filter="true"/>
+ <field name="item_lang" filter="true"/>
+ <field name="item_type" filter="true"/>
+ <field name="lit_form" filter="true"/>
+ <field name="search_format" filter="true"/>
+ <field name="sr_format" filter="true"/>
+ <field name="vr_format" filter="true"/>
+
+ <!-- sorters -->
+ <field name="authorsort" sorter="true"/>
+ <field name="titlesort" sorter="true"/>
+ <field name="pubdate" sorter="true"/>
+ </fields>
+ </index>
+</elasticsearch>
+
use OpenSRF::Utils::JSON;
use OpenSRF::Utils::Logger qw/:logger/;
use OpenILS::Utils::Fieldmapper;
-use OpenSRF::Utils::SettingsClient;
use OpenILS::Utils::CStoreEditor q/:funcs/;
use OpenILS::Elastic::BibSearch;
use Digest::MD5 qw(md5_hex);
return if $init_done;
$init_done = 1;
- my $e = new_editor();
+ # NOTE: after things stabilize and maybe load balancing, etc. is
+ # tested and working, we could maintain a global $es so the
+ # connection is cached instead of reconnecting on every search call.
+ my $es = OpenILS::Elastic::BibSearch->new;
+ $es->connect;
- # no pkey
- $bib_fields = $e->search_elastic_bib_field({name => {'!=' => undef}});
+ $bib_fields = $es->bib_fields;
+ my $e = new_editor();
my $stats = $e->json_query({
select => {ccs => ['id', 'opac_visible', 'is_available']},
from => 'ccs',
Org unit based item presence and availability filtering may
optionally be added to the query. See search options
below.
-
- See [ select * from elastic.bib_field where search_field; ]
- for full-text search fields and classes.
/,
params => [
{ type => 'object',
$elastic_query->{size} = 1000;
}
- my $es = OpenILS::Elastic::BibSearch->new('main');
-
+ # NOTE: after things stabilize and maybe load balancing, etc. is
+ # tested and working, we could maintain a global $es so the
+ # connection is cached instead of reconnecting on every search call.
+ my $es = OpenILS::Elastic::BibSearch->new;
$es->connect;
+
my $results = $es->search($elastic_query);
$logger->debug("ES elasticsearch returned: ".
}
# Format ES search aggregations to match the API response facet structure
-# {$cmf_id => {"Value" => $count}, $cmf_id2 => {"Value Two" => $count2}, ...}
+# {$field_id => {"Value" => $count}, $field_id2 => {"Value Two" => $count2}, ...}
sub format_facets {
my $aggregations = shift;
my $facets = {};
for my $fname (keys %$aggregations) {
- my ($field_class, $name) = split(/\|/, $fname);
+ my ($search_group, $name) = split(/\|/, $fname);
my ($bib_field) = grep {
- $_->name eq $name && $_->search_group eq $field_class
+ $_->name eq $name && $_->search_group eq $search_group
} @$bib_fields;
- my $hash = $facets->{$bib_field->metabib_field} = {};
+ my $hash = $facets->{$bib_field->id} = {};
my $values = $aggregations->{$fname}->{buckets};
for my $bucket (@$values) {
sub add_elastic_facet_aggregations {
my ($elastic_query) = @_;
- my @facet_fields = grep {$_->facet_field eq 't'} @$bib_fields;
+ my @facet_fields = grep {$_->facet_field} @$bib_fields;
return unless @facet_fields;
$elastic_query->{aggs} = {};
my $fgrp = $facet->search_group;
$fname = "$fgrp|$fname" if $fgrp;
- $elastic_query->{aggs}{$fname} = {terms => {field => "$fname.facet"}};
+ $elastic_query->{aggs}{$fname} = {terms => {field => "$fname|facet"}};
}
}
use warnings;
use DBI;
use Time::HiRes qw/time/;
-use OpenSRF::Utils::Logger qw/:logger/;
-use OpenILS::Utils::CStoreEditor qw/:funcs/;
+use XML::LibXML;
+use XML::LibXML::XPathContext;
use Search::Elasticsearch;
use OpenSRF::Utils::JSON;
+use OpenSRF::Utils::Logger qw/:logger/;
+use OpenILS::Utils::CStoreEditor qw/:funcs/;
+use OpenILS::Utils::Fieldmapper;
use Data::Dumper;
$Data::Dumper::Indent = 0;
+# For parsing the Elasticsearch configuration file
+my $ES_NAMESPACE = 'http://evergreen-ils.org/spec/elasticsearch/v1';
+
sub new {
- my ($class, $cluster) = @_;
+ my ($class, %args) = @_;
- my $self = {
- cluster => $cluster,
- indices => [],
- marc_fields => []
- };
+ my $self = {%args};
+
+ $self->{cluster} = 'main' unless $args{cluster};
return bless($self, $class);
}
sub indices {
my $self = shift;
- return $self->{indices};
+ return $self->{indices} if $self->{indices};
+
+ my $def;
+ eval {
+ # All open indices
+ $def = $self->es->indices->get(
+ index => $self->index_class . '-*',
+ expand_wildcards => 'open'
+ );
+ };
+
+ if ($@) {
+ $logger->error("ES index lookup failed: $@");
+ return {};
+ }
+
+ return $self->{indices} = $def;
}
sub es {
}
sub index_name {
- die "Index name must be provided by sub-class\n";
+ my ($self) = @_;
+ return $self->{index_name};
+}
+
+my $xpc;
+sub xpath_context {
+ if (!$xpc) {
+ $xpc = XML::LibXML::XPathContext->new;
+ $xpc->registerNs('es', $ES_NAMESPACE);
+ }
+ return $xpc;
+}
+
+# In maintenance mode we are working with specific indexes.
+# Otherwise all actions target the index alias which is index_class.
+sub index_target {
+ my ($self) = @_;
+ return $self->maintenance_mode ? $self->index_name : $self->index_class;
+}
+
+sub index_class {
+ die "index_class() should be implemented by sub-classes\n";
+}
+
+# Are we modifying indexes or just read/writing indexed data?
+sub maintenance_mode {
+ my $self = shift;
+ return $self->{maintenance_mode};
}
sub language_analyzers {
my $db_pass = $self->{db_pass};
my $db_appn = $self->{db_appn} || 'Elastic Indexer';
- # TODO Add application_name to dsn
+ my $dsn =
+ "dbi:Pg:db=$db_name;host=$db_host;port=$db_port;application_name='$db_appn';";
- my $dsn = "dbi:Pg:db=$db_name;host=$db_host;port=$db_port";
$logger->debug("ES connecting to DB $dsn");
$self->{db} = DBI->connect(
# load the config via cstore.
sub load_config {
- my $self = shift;
+ my ($self) = @_;
+
my $e = new_editor();
my $cluster = $self->cluster;
- $self->{nodes} = $e->search_elastic_node({cluster => $cluster, active => 't'});
+ my @nodes = $self->{nodes} ? @{$self->{nodes}} : ();
+
+ if (@nodes) {
+
+ $logger->info("ES overriding nodes with @nodes");
+ $self->{nodes} = \@nodes;
+
+ } else {
+
+ my %active = $self->maintenance_mode ? () : (active => 't');
+ my $nodes = $e->search_elastic_node({cluster => $cluster, %active});
+
+ $self->{nodes} = [
+ map {
+ sprintf("%s://%s:%d%s", $_->proto, $_->host, $_->port, $_->path)
+ } @$nodes
+ ];
+ }
unless (@{$self->nodes}) {
$logger->error("ES no nodes defined for cluster $cluster");
return;
}
- $self->{indices} = $e->search_elastic_index({cluster => $cluster});
-
- unless (@{$self->indices}) {
- $logger->error("ES no indices defined for cluster $cluster");
+ if (!$self->index_class) {
+ $logger->error("ES index_class required to initialize");
return;
}
}
-sub connect {
+sub load_es_config {
my ($self) = @_;
- $self->load_config;
- my @nodes;
- for my $server (@{$self->nodes}) {
- push(@nodes, {
- scheme => $server->proto,
- host => $server->host,
- port => $server->port,
- path => $server->path
- });
+ my $cluster = $self->cluster;
+
+ if (!$self->indices || !keys(%{$self->indices})) {
+ $logger->info("ES no usable indices defined for cluster $cluster");
+ return unless $self->maintenance_mode;
}
- $logger->debug("ES connecting to ".scalar(@nodes)." nodes");
+ if (!$self->index_name) {
+ # Default to the index that has an alias matching our index_class
+
+ for my $name (keys %{$self->indices}) {
+ if ($self->index_is_active($name)) {
+ $logger->info("ES defaulting to active index $name");
+ $self->{index_name} = $name;
+ }
+ }
+ }
+
+ # Load the main ES config file
+
+ # TODO: 'dirs' option for 'conf'
+ #my $client = OpenSRF::Utils::SettingsClient->new;
+ #my $dir = $client->config_value("dirs", "conf");
- eval { $self->{es} = Search::Elasticsearch->new(nodes => \@nodes) };
+ my $doc;
+ my $filename = $self->{es_config_file}
+ || '/openils/conf/elastic-config.xml';
+
+ eval { $doc = XML::LibXML->load_xml(location => $filename) };
+
+ if ($@ || !$doc) {
+ my $msg = "ES could not parse elastic config file: $filename $@";
+ $logger->error($msg);
+ die "$msg\n";
+ }
+
+ $self->{es_config} = $doc->documentElement;
+}
+
+sub es_config {
+ my $self = shift;
+ return $self->{es_config};
+}
+
+sub active_index {
+ my $self = shift;
+ my $indices = $self->indices;
+ for my $name (keys %{$indices}) {
+ return $name if $self->index_is_active($name);
+ }
+ return undef;
+}
+
+# True if the named index has an alias matching our index class
+sub index_is_active {
+ my ($self, $name) = @_;
+
+ my $conf = $self->indices->{$name};
+ return 0 unless $conf;
+
+ my @aliases = keys %{$conf->{aliases}};
+ return 1 if grep {$_ eq $self->index_class} @aliases;
+
+ return 0;
+}
+
+
+sub index_config {
+ my $self = shift;
+ my $class = $self->index_class;
+
+ if (!$self->es_config) {
+ $logger->error("ES cannot load index config without a config file");
+ return undef;
+ }
+
+ my @conf;
+ eval {
+ @conf = $xpc->findnodes(
+ "//es:elasticsearch/es:index[\@class='$class']",
+ $self->es_config
+ );
+ };
+
+ if ($@ || !@conf) {
+ my $msg = "ES failed to locate config for index class '$class' $@";
+ $logger->error($msg);
+ die "$msg\n";
+ }
+
+ return $conf[0];
+}
+
+sub connect {
+ my ($self) = @_;
+
+ $self->load_config;
+
+ my @nodes = @{$self->nodes};
+ $logger->info("ES connecting to nodes: @nodes");
+
+ eval {
+ $self->{es} = Search::Elasticsearch->new(
+ client => '6_0::Direct',
+ nodes => \@nodes
+ );
+ };
if ($@) {
$logger->error("ES failed to connect to @nodes: $@");
return;
}
+
+ $self->load_es_config;
+}
+
+# Activates the currently loaded index while deactivating any active
+# index with the same cluster and index_class.
+# Applies an alias to the activated index equal to the index class.
+sub activate_index {
+ my ($self) = @_;
+
+ my $index = $self->index_name;
+
+ if (!$self->es->indices->exists(index => $index)) {
+ $logger->warn("ES cannot activate index '$index' which does not exist");
+ return;
+ }
+
+ my $from_index = $self->active_index;
+
+ # When activating an index, point the main alias toward the
+ # newly active index.
+ return $self->migrate_alias($self->index_class, $from_index, $index);
+}
+
+
+# Migrate an alias from one index to another.
+# If either from_index or to_index are not defined, then only half
+# of the migration (i.e. remove or add) is performed.
+sub migrate_alias {
+ my ($self, $alias, $from_index, $to_index) = @_;
+ return undef unless $alias && ($from_index || $to_index);
+
+ my @actions;
+
+ $from_index ||= '';
+ $to_index ||= '';
+ $logger->info("ES migrating alias [$alias] from $from_index to $to_index");
+
+ if ($from_index) {
+ push(@actions, {remove => {alias => $alias, index => $from_index}});
+ }
+
+ if ($to_index) {
+ push(@actions, {add => {alias => $alias, index => $to_index}});
+ }
+
+ eval {
+ $self->es->indices->update_aliases({body => {actions => \@actions}});
+ };
+
+ if ($@) {
+ $logger->error("ES alias migration [$alias] failed $@");
+ return undef;
+ }
+
+ return 1;
}
sub delete_index {
$logger->warn("ES index '$index' ".
"does not exist in cluster '".$self->cluster."'");
}
+
+ delete $self->indices->{$index};
+
+ return 1;
}
# Remove multiple documents from the index by ID.
eval {
$result = $self->es->delete_by_query(
- index => $self->index_name,
+ index => $self->index_target,
type => 'record',
body => {query => {terms => {_id => $ids}}}
);
return $result;
}
+# Returns true if a document with the requested ID exists.
+sub document_exists {
+ my ($self, $id) = @_;
+
+ my $result;
+
+ eval {
+ $result = $self->es->index(
+ index => $self->index_target,
+ type => 'record',
+ id => $id,
+ );
+ };
+
+
+ if ($@) {
+ $logger->error("ES document_exists failed with $@");
+ return undef;
+ }
+
+ return $result ? 1 : 0;
+}
+
+# Create or replace a document.
sub index_document {
my ($self, $id, $body) = @_;
eval {
$result = $self->es->index(
- index => $self->index_name,
+ index => $self->index_target,
type => 'record',
id => $id,
body => $body
return $result;
}
+# Index a new document
+# This will fail if the document already exists.
+sub create_document {
+ my ($self, $id, $body) = @_;
+
+ my $result;
+
+ eval {
+ $result = $self->es->create(
+ index => $self->index_target,
+ type => 'record',
+ id => $id,
+ body => $body
+ );
+ };
+
+ if ($@) {
+ $logger->error("ES create_document failed with $@");
+ return undef;
+ }
+
+ if ($result->{failed}) {
+ $logger->error("ES create document $id failed " . Dumper($result));
+ return undef;
+ }
+
+ $logger->debug("ES create => $id succeeded");
+ return $result;
+}
+
+
+# Partial document update
+# This will fail if the document does not exist.
+sub update_document {
+ my ($self, $id, $body) = @_;
+
+ my $result;
+
+ eval {
+ $result = $self->es->update(
+ index => $self->index_target,
+ type => 'record',
+ id => $id,
+ body => {doc => $body}
+ );
+ };
+
+ if ($@) {
+ $logger->error("ES update_document failed with $@");
+ return undef;
+ }
+
+ if ($result->{failed}) {
+ $logger->error("ES update document $id failed " . Dumper($result));
+ return undef;
+ }
+
+ $logger->debug("ES update => $id succeeded");
+ return $result;
+}
+
sub search {
my ($self, $query) = @_;
eval {
my $start_time = time;
$result = $self->es->search(
- index => $self->index_name,
+ index => $self->index_target,
body => $query
);
$duration = time - $start_time;
# Avoid trying to index such data by lazily chopping it off
# at 1/4 the limit to accomodate all UTF-8 chars.
sub truncate_value {
- my ($self, $value) = @_;
+ my ($self, $value, $length) = @_;
+ $length = 8190 unless $length;
return substr($value, 0, 8190);
}
sub get_index_def {
- my ($self) = @_;
- return $self->es->indices->get(index => $self->index_name);
+ my ($self, $name) = @_;
+ $name ||= $self->index_name;
+
+ my $def;
+ eval { $def = $self->es->indices->get(index => $name) };
+
+ if ($@) {
+ $logger->error("ES cannot find index def for $name");
+ return undef;
+ }
+
+ return $def;
}
-package OpenILS::Elastic::BibSearch;
# ---------------------------------------------------------------
-# Copyright (C) 2019 King County Library System
+# Copyright (C) 2019-2020 King County Library System
# Author: Bill Erickson <berickxx@gmail.com>
#
# This program is free software; you can redistribute it and/or
# MERCHANTABILITY or FITNESS FOR A PARTICULAR code. See the
# GNU General Public License for more details.
# ---------------------------------------------------------------
+package OpenILS::Elastic::BibField;
+use strict;
+use warnings;
+
+sub new {
+ my ($class, %args) = @_;
+ my $self = {%args};
+ return bless($self, $class);
+}
+sub id {
+ my $self = shift;
+ return $self->search_group ?
+ $self->search_group . '|' . $self->name : $self->name;
+}
+sub search_group {
+ my $self = shift;
+ return $self->{search_group};
+}
+sub name {
+ my $self = shift;
+ return $self->{name};
+}
+sub search_field {
+ my $self = shift;
+ return $self->{search_field};
+}
+sub facet_field {
+ my $self = shift;
+ return $self->{facet_field};
+}
+sub weight {
+ my $self = shift;
+ return $self->{weight};
+}
+sub filter {
+ my $self = shift;
+ return $self->{filter};
+}
+sub sorter {
+ my $self = shift;
+ return $self->{sorter};
+}
+
+package OpenILS::Elastic::BibSearch;
use strict;
use warnings;
-use Encode;
use DateTime;
use Clone 'clone';
-use Business::ISBN;
-use Business::ISSN;
use Time::HiRes qw/time/;
+use XML::LibXML;
+use XML::LibXML::XPathContext;
use OpenSRF::Utils::Logger qw/:logger/;
use OpenSRF::Utils::JSON;
+use OpenSRF::Utils::SettingsClient;
use OpenILS::Utils::CStoreEditor qw/:funcs/;
use OpenILS::Utils::DateTime qw/interval_to_seconds/;
use OpenILS::Elastic;
+use OpenILS::Utils::Normalize;
use base qw/OpenILS::Elastic/;
# default number of bibs to index per batch.
my $DEFAULT_BIB_BATCH_SIZE = 500;
+my $INDEX_CLASS = 'bib-search';
-my $INDEX_NAME = 'bib-search';
+# https://www.elastic.co/guide/en/elasticsearch/reference/current/ignore-above.html
+# Useful for ignoring excessively long filters and facets.
+# Only applied to the keyword variation of each index. Does not affect
+# the 'text' varieties. The selected limit is arbitrary.
+my $IGNORE_ABOVE = 256;
+
+# Individual characters of some values like sorters provide less and less
+# value as the length of the text gets longer and longer. Unlike
+# $IGNORE_ABOVE, this only trims the string, it does not prevent it from
+# getting indexed in the first place. The selected limit is arbitrary.
+my $TRIM_ABOVE = 512;
my $BASE_INDEX_SETTINGS = {
analysis => {
analyzer => {
folding => {
- filter => ['lowercase', 'asciifolding'],
+ filter => ['asciifolding', 'lowercase'],
+ tokenizer => 'standard'
+ },
+ icu_folding => {
+ filter => ['icu_folding', 'lowercase'],
+ tokenizer => 'icu_tokenizer'
+ },
+ stripapos => {
+ # "It's A Wonderful Live" => "Its A ..."
+ char_filter => ['stripapos'],
+ filter => ['lowercase'],
tokenizer => 'standard'
},
stripdots => {
}
},
char_filter => {
+ stripapos => {
+ type => 'mapping',
+ mappings => ['\' =>']
+ },
stripdots => {
type => 'mapping',
mappings => ['. =>']
}
};
-# Well-known bib-search index properties
+# Well-known bib-search index properties
my $BASE_PROPERTIES = {
bib_source => {type => 'integer'},
create_date => {type => 'date'},
normalizer => 'custom_lowercase'
},
value => {
- type => 'text',
+ type => 'keyword',
+ ignore_above => $IGNORE_ABOVE,
+ normalizer => 'custom_lowercase',
fields => {
- text_folded => {
- type => 'text',
- analyzer => 'folding'
- }
+ text => {type => 'text'},
+ text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'}
}
}
}
# Values from grouped fields are copied into the group field.
# Here we make some assumptions about the general purpose of
# each group.
+ # The 'keyword' variation of each is used for exact matches,
+ # starts with, and similar searches.
# Note the ignore_above only affects the 'keyword' version of the
# field, the assumption being text that large would solely be
# searched via 'text' indexes.
title => {
type => 'keyword',
- ignore_above => 256,
+ ignore_above => $IGNORE_ABOVE,
normalizer => 'custom_lowercase',
fields => {
text => {type => 'text'},
text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'},
text_spacedots => {type => 'text', analyzer => 'spacedots'},
- text_stripdots => {type => 'text', analyzer => 'stripdots'}
+ text_stripdots => {type => 'text', analyzer => 'stripdots'},
+ text_stripapos => {type => 'text', analyzer => 'stripapos'}
}
},
author => {
type => 'keyword',
- ignore_above => 256,
+ ignore_above => $IGNORE_ABOVE,
normalizer => 'custom_lowercase',
fields => {
text => {type => 'text'},
text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'},
text_spacedots => {type => 'text', analyzer => 'spacedots'},
- text_stripdots => {type => 'text', analyzer => 'stripdots'}
+ text_stripdots => {type => 'text', analyzer => 'stripdots'},
+ text_stripapos => {type => 'text', analyzer => 'stripapos'}
}
},
subject => {
type => 'keyword',
- ignore_above => 256,
+ ignore_above => $IGNORE_ABOVE,
normalizer => 'custom_lowercase',
fields => {
text => {type => 'text'},
text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'},
text_spacedots => {type => 'text', analyzer => 'spacedots'},
text_stripdots => {type => 'text', analyzer => 'stripdots'}
}
},
series => {
type => 'keyword',
- ignore_above => 256,
+ ignore_above => $IGNORE_ABOVE,
normalizer => 'custom_lowercase',
fields => {
text => {type => 'text'},
text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'},
text_spacedots => {type => 'text', analyzer => 'spacedots'},
- text_stripdots => {type => 'text', analyzer => 'stripdots'}
+ text_stripdots => {type => 'text', analyzer => 'stripdots'},
+ text_stripapos => {type => 'text', analyzer => 'stripapos'}
}
},
keyword => {
# term (aka "keyword") searches are not used on the
- # keyword field, but we index it just the same (sans lowercase)
- # for structural consistency with other group fields.
+ # keyword field, but we structure the index just the same
+ # for consistency with other group fields.
type => 'keyword',
- ignore_above => 256,
+ ignore_above => 1, # essentially a no-op.
fields => {
text => {type => 'text'},
text_folded => {type => 'text', analyzer => 'folding'},
+ text_icu_folded => {type => 'text', analyzer => 'icu_folding'},
text_spacedots => {type => 'text', analyzer => 'spacedots'},
- text_stripdots => {type => 'text', analyzer => 'stripdots'}
+ text_stripdots => {type => 'text', analyzer => 'stripdots'},
+ text_stripapos => {type => 'text', analyzer => 'stripapos'}
}
},
+ # Identifier fields only support 'keyword' indexes, no full-text.
identifier => {
- # Avoid full-text indexing on identifier fields.
type => 'keyword',
- ignore_above => 256,
+ ignore_above => $IGNORE_ABOVE,
normalizer => 'custom_lowercase',
- },
-
- # Create some shortcut indexes for streamlining query_string searches.
- ti => {type => 'text'},
- au => {type => 'text'},
- se => {type => 'text'},
- su => {type => 'text'},
- kw => {type => 'text'},
- id => {
- type => 'keyword',
- ignore_above => 256
}
};
-my %SHORT_GROUP_MAP = (
- title => 'ti',
- author => 'au',
- subject => 'su',
- series => 'se',
- keyword => 'kw',
- identifier => 'id'
+# Map 'au' to 'author', etc.
+my %SEARCH_CLASS_ALIAS_MAP = (
+ ti => 'title.text',
+ au => 'author.text',
+ su => 'subject.text',
+ se => 'series.text',
+ kw => 'keyword.text',
+ pb => 'keyword|publisher.text',
+ id => 'identifier'
);
-sub index_name {
- return $INDEX_NAME;
+sub index_class {
+ return $INDEX_CLASS;
}
-# TODO: add index-specific language analyzers to DB config
+# TODO: determine when/how to apply language analyzers.
+# e.g. create lang-specific index fields?
sub language_analyzers {
return ("english");
}
+sub skip_holdings {
+ my $self = shift;
+ return $self->{skip_holdings};
+}
+
+sub bib_fields {
+ my $self = shift;
+ return $self->{bib_fields} if $self->{bib_fields};
+
+ my @bib_fields = $self->xpath_context->findnodes(
+ '//es:fields/es:field', $self->index_config);
+
+ my @fields;
+ for my $field (@bib_fields) {
+
+ my %struct;
+
+ for my $key (qw/search_group name/) {
+ $struct{$key} = $field->getAttribute($key) || '';
+ }
+
+ for my $key (qw/search_field facet_field filter sorter/) {
+ $struct{$key} = ($field->getAttribute($key) || '') eq 'true';
+ }
+
+ push (@fields, OpenILS::Elastic::BibField->new(%struct));
+ }
+
+ return $self->{bib_fields} = \@fields;
+}
+
+sub xsl_file {
+ my ($self) = @_;
+
+ if (!$self->{xsl_file}) {
+ my @nodes = $self->xpath_context->findnodes(
+ '//es:transform/text()', $self->index_config);
+ $self->{xsl_file} = $nodes[0];
+ }
+
+ return $self->{xsl_file};
+}
+
+sub xsl_doc {
+ my ($self) = @_;
+
+ $self->{xsl_doc} = XML::LibXML->load_xml(location => $self->xsl_file)
+ unless $self->{xsl_doc};
+
+ return $self->{xsl_doc};
+}
+
+sub xsl_sheet {
+ my $self = shift;
+
+ $self->{xsl_sheet} = XML::LibXSLT->new->parse_stylesheet($self->xsl_doc)
+ unless $self->{xsl_sheet};
+
+ return $self->{xsl_sheet};
+}
+
+sub get_bib_data {
+ my ($self, $record_ids) = @_;
+
+ my $records = [];
+ my $db_data = $self->get_bib_db_data($record_ids);
+
+ for my $db_rec (@$db_data) {
+
+ my $rec = {fields => []};
+ push(@$records, $rec);
+
+ # Copy DB data into our record object.
+ $rec->{$_} = $db_rec->{$_} for
+ qw/id bib_source metarecord create_date edit_date deleted/;
+
+ # No need to extract index values for delete records;
+ next if $rec->{deleted} == 1;
+
+ my $marc_doc = XML::LibXML->load_xml(string => $db_rec->{marc});
+ my $result = $self->xsl_sheet->transform($marc_doc);
+ my $output = $self->xsl_sheet->output_as_chars($result);
+
+ my @rows = split(/\n/, $output);
+ for my $row (@rows) {
+ my ($purpose, $search_group, $name, @tokens) = split(/ /, $row);
+
+ $search_group = '' if ($search_group || '') eq '_';
+
+ my $value = join(' ', @tokens);
+
+ my $field = {
+ purpose => $purpose,
+ search_group => $search_group,
+ name => $name,
+ value => $value
+ };
+
+ push(@{$rec->{fields}}, $field);
+ }
+ }
+
+ return $records;
+}
+
+sub get_bib_db_data {
+ my ($self, $record_ids) = @_;
+
+ my $ids_str = join(',', @$record_ids);
+
+ my $sql = <<SQL;
+SELECT DISTINCT ON (bre.id)
+ bre.id,
+ bre.create_date,
+ bre.edit_date,
+ bre.source AS bib_source,
+ bre.deleted,
+ bre.marc
+FROM biblio.record_entry bre
+LEFT JOIN metabib.metarecord_source_map mmrsm ON (mmrsm.source = bre.id)
+WHERE bre.id IN ($ids_str)
+SQL
+
+ return $self->get_db_rows($sql);
+}
+
sub create_index_properties {
my ($self) = @_;
} foreach qw/title subject series keyword/;
}
- # elastic.bib_field has no primary key field, so retrieve_all won't work.
- # Note the name value may be repeated across search group depending
- # on local configuration.
- my $fields = new_editor()->search_elastic_bib_field({name => {'!=' => undef}});
+ # field_group will be undef for main/active fields
+ my $fields = $self->bib_fields;
for my $field (@$fields) {
my $def;
if ($search_group) {
+ if ($field->search_field) {
- # Use the same fields and analysis as the 'grouped' field.
- $def = clone($properties->{$search_group});
- $def->{copy_to} = [$search_group, $SHORT_GROUP_MAP{$search_group}];
+ # Use the same fields and analysis as the 'grouped' field.
+ $def = clone($properties->{$search_group});
- # Apply ranking boost to each analysis variation.
- my $flds = $def->{fields};
- if ($flds && (my $boost = ($field->weight || 1)) > 1) {
- $flds->{$_}->{boost} = $boost foreach keys %$flds;
+ # Copy grouped fields into their group parent field.
+ $def->{copy_to} = $search_group;
+
+ # Apply ranking boost to each analysis variation.
+ my $flds = $def->{fields};
+ if ($flds && (my $boost = ($field->weight || 1)) > 1) {
+ $flds->{$_}->{boost} = $boost foreach keys %$flds;
+ }
}
} else {
-
- # Non-grouped fields are used for filtering and sorting, so
- # they don't need as much processing.
+ # Filters and sorters
$def = {
type => 'keyword',
- ignore_above => 256,
normalizer => 'custom_lowercase'
};
+
+ # Long sorter values are not necessarily unexpected,
+ # e.g. long titles.
+ $def->{ignore_above} = $IGNORE_ABOVE unless $field->sorter;
+ }
+
+ if ($def) {
+ $logger->debug("ES adding field $field_name: ".
+ OpenSRF::Utils::JSON->perl2JSON($def));
+
+ $properties->{$field_name} = $def;
}
- if ($field->facet_field eq 't' && $def->{fields}) {
- # Facet fields are used for aggregation which requires
- # an additional unaltered keyword field.
- $def->{fields}->{facet} = {
+ # Search and facet fields can have the same name/group pair,
+ # but are stored as separate fields in ES since the content
+ # may vary between the two.
+ if ($field->facet_field) {
+
+ # Facet fields are stored as separate fields, because their
+ # content may differ from the matching search field.
+ $field_name = "$field_name|facet";
+
+ $def = {
type => 'keyword',
- ignore_above => 256
+ ignore_above => $IGNORE_ABOVE
};
- }
- $logger->debug("ES adding field $field_name: ".
- OpenSRF::Utils::JSON->perl2JSON($def));
+ $logger->debug("ES adding field $field_name: ".
+ OpenSRF::Utils::JSON->perl2JSON($def));
- $properties->{$field_name} = $def;
+ $properties->{$field_name} = $def;
+ }
}
return $properties;
sub create_index {
my ($self) = @_;
+ my $index_name = $self->index_name;
- if ($self->es->indices->exists(index => $INDEX_NAME)) {
- $logger->warn("ES index '$INDEX_NAME' already exists");
+ if ($self->es->indices->exists(index => $index_name)) {
+ $logger->warn("ES index '$index_name' already exists in ES");
return;
}
$logger->info(
- "ES creating index '$INDEX_NAME' on cluster '".$self->cluster."'");
+ "ES creating index '$index_name' on cluster '".$self->cluster."'");
my $properties = $self->create_index_properties;
my $settings = $BASE_INDEX_SETTINGS;
- $settings->{number_of_replicas} = scalar(@{$self->nodes});
- $settings->{number_of_shards} = $self->index->num_shards;
+ $settings->{number_of_shards} = 1; # TODO $index_config->num_shards;
my $conf = {
- index => $INDEX_NAME,
+ index => $index_name,
body => {settings => $settings}
};
- $logger->info("ES creating index '$INDEX_NAME'");
+ $logger->info("ES creating index '$index_name'");
# Create the base index with settings
eval { $self->es->indices->create($conf) };
if ($@) {
- $logger->error("ES failed to create index cluster=".
- $self->cluster. "index=$INDEX_NAME error=$@");
- warn "$@\n\n";
- return 0;
+ my $msg = "ES failed to create index cluster=".
+ $self->cluster. "index=$index_name error=$@";
+
+ $logger->error($msg);
+ die "$msg\n";
}
# Create each mapping one at a time instead of en masse so we
# can more easily report when mapping creation fails.
-
for my $field (keys %$properties) {
- $logger->info("ES Creating index mapping for field $field");
-
- eval {
- $self->es->indices->put_mapping({
- index => $INDEX_NAME,
- type => 'record',
- body => {dynamic => 'strict', properties => {$field => $properties->{$field}}}
- });
- };
+ return 0 unless
+ $self->create_one_field_index($field, $properties->{$field});
+ }
+
+ # Now that we've added the configured fields,
+ # add the shortened search_group aliases.
+ while (my ($alias, $field) = each %SEARCH_CLASS_ALIAS_MAP) {
- if ($@) {
- my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties->{$field});
+ return 0 unless $self->create_one_field_index(
+ $alias, {type => 'alias', path => $field});
+ }
- $logger->error("ES failed to create index mapping: " .
- "index=$INDEX_NAME field=$field error=$@ mapping=$mapjson");
+ return 1;
+}
- warn "$@\n\n";
- return 0;
- }
+sub create_one_field_index {
+ my ($self, $field, $properties) = @_;
+
+ my $index_name = $self->index_name;
+
+ $logger->info("ES Creating index mapping for field $field");
+
+ eval {
+ $self->es->indices->put_mapping({
+ index => $index_name,
+ type => 'record',
+ body => {
+ dynamic => 'strict',
+ properties => {$field => $properties}
+ }
+ });
+ };
+
+ if ($@) {
+ my $mapjson = OpenSRF::Utils::JSON->perl2JSON($properties);
+
+ $logger->error("ES failed to create index mapping: " .
+ "index=$index_name field=$field error=$@ mapping=$mapjson");
+
+ warn "$@\n\n";
+ return 0;
}
return 1;
}
-sub get_bib_data {
- my ($self, $record_ids) = @_;
- my $ids_str = join(',', @$record_ids);
+sub get_bib_field_for_data {
+ my ($self, $field) = @_;
- my $sql = <<SQL;
-SELECT DISTINCT ON (bre.id, search_group, name, value)
- bre.id,
- bre.create_date,
- bre.edit_date,
- bre.source AS bib_source,
- bre.deleted,
- mmrsm.metarecord,
- (elastic.bib_record_properties(bre.id)).*
-FROM biblio.record_entry bre
-LEFT JOIN metabib.metarecord_source_map mmrsm ON (mmrsm.source = bre.id)
-WHERE bre.id IN ($ids_str)
-SQL
+ my @matches = grep {$_->name eq $field->{name}} @{$self->bib_fields};
- return $self->get_db_rows($sql);
+ @matches = grep {
+ (($_->search_group || '') eq ($field->{search_group} || ''))
+ } @matches;
+
+ my ($match) = grep {
+ ($_->search_field && $field->{purpose} eq 'search') ||
+ ($_->facet_field && $field->{purpose} eq 'facet') ||
+ ($_->filter && $field->{purpose} eq 'filter') ||
+ ($_->sorter && $field->{purpose} eq 'sorter')
+ } @matches;
+
+ if (!$match) {
+ # Warning on mismatched fields can lead to a lot of logs
+ # while trying different field configs. Consider a
+ # 'warn-on-field-mismatch' flag.
+ $logger->debug("ES No bib field matches extracted data ".
+ OpenSRF::Utils::JSON->perl2JSON($field));
+ }
+
+ return $match;
}
sub populate_bib_index_batch {
$logger->info("ES indexing ".scalar(@$bib_ids)." records");
- my $bib_data = $self->get_bib_data($bib_ids);
+ my $records = $self->get_bib_data($bib_ids);
# Remove records that are marked deleted.
# This should only happen when running in refresh mode.
for my $bib_id (@$bib_ids) {
# Every row in the result data contains the 'deleted' value.
- my ($field) = grep {$_->{id} == $bib_id} @$bib_data;
+ my ($rec) = grep {$_->{id} == $bib_id} @$records;
- if ($field->{deleted} == 1) { # not 't' / 'f'
+ if ($rec->{deleted} == 1) { # not 't' / 'f'
$self->delete_documents($bib_id);
} else {
push(@active_ids, $bib_id);
$bib_ids = [@active_ids];
- my $holdings = $self->load_holdings($bib_ids);
- my $marc = $self->load_marc($bib_ids);
+ return 0 unless @$bib_ids;
+
+ my $holdings = $self->load_holdings($bib_ids) unless $self->skip_holdings;
for my $bib_id (@$bib_ids) {
+ my ($rec) = grep {$_->{id} == $bib_id} @$records;
my $body = {
- marc => $marc->{$bib_id} || [],
- holdings => $holdings->{$bib_id} || []
+ bib_source => $rec->{bib_source},
+ metarecord => $rec->{metarecord},
+ marc => []
};
- # there are multiple rows per bib in the data list.
- my @fields = grep {$_->{id} == $bib_id} @$bib_data;
+ $body->{holdings} = $holdings->{$bib_id} || [] unless $self->skip_holdings;
- my $first = 1;
- for my $field (@fields) {
-
- if ($first) {
- $first = 0;
- # some values are repeated per field.
- # extract them from the first entry.
- $body->{bib_source} = $field->{bib_source};
- $body->{metarecord} = $field->{metarecord};
-
- # ES ikes the "T" separator for ISO dates
- ($body->{create_date} = $field->{create_date}) =~ s/ /T/g;
- ($body->{edit_date} = $field->{edit_date}) =~ s/ /T/g;
- }
+ # ES likes the "T" separator for ISO dates
+ ($body->{create_date} = $rec->{create_date}) =~ s/ /T/g;
+ ($body->{edit_date} = $rec->{edit_date}) =~ s/ /T/g;
+ for my $field (@{$rec->{fields}}) {
+ my $purpose = $field->{purpose};
my $fclass = $field->{search_group};
my $fname = $field->{name};
my $value = $field->{value};
next unless defined $value && $value ne '';
+ my $trim = $purpose eq 'sorter' ? $TRIM_ABOVE : undef;
+ $value = $self->truncate_value($value, $trim);
+
+ if ($purpose eq 'marc') {
+ # NOTE: we could create/require elastic.bib_field entries for
+ # MARC values as well if we wanted to control the exact
+ # MARC data that's indexed.
+ $self->add_marc_value($body, $fclass, $fname, $value);
+ next;
+ }
+
+ # Ignore any data provided by the transform we have
+ # no configuration for.
+ next unless $self->get_bib_field_for_data($field);
+
$fname = "$fclass|$fname" if $fclass;
- $value = $self->truncate_value($value);
+ $fname = "$fname|facet" if $purpose eq 'facet';
if ($fname eq 'identifier|isbn') {
index_isbns($body, $value);
+
} elsif ($fname eq 'identifier|issn') {
index_issns($body, $value);
+
+ } elsif ($fname eq 'pubdate') {
+ index_pubdate($body, $value);
+
+ } elsif ($fname =~ /sort/) {
+ index_sorter($body, $fname, $value);
+
} else {
append_field_value($body, $fname, $value);
}
}
- return 0 unless $self->index_document($bib_id, $body);
+ if ($self->skip_holdings) {
+ # Skip-Holdings mode performs an update for existing
+ # documents, so the attached holdings will remain, but
+ # performs a create for documents that don't yet exist.
+ if ($self->document_exists($bib_id)) {
+ return 0 unless $self->update_document($bib_id, $body);
+ } else {
+ return 0 unless $self->create_document($bib_id, $body);
+ }
+ } else {
+ return 0 unless $self->index_document($bib_id, $body);
+ }
$state->{start_record} = $bib_id + 1;
$index_count++;
}
- $logger->info("ES indexing completed for records " .
- $bib_ids->[0] . '...' . $bib_ids->[-1]);
+ $self->{total_indexed} += $index_count;
+
+ $logger->info(sprintf(
+ "ES indexed %d records in this batch across records %d ... %d ".
+ "with a session total of %d",
+ $index_count, $bib_ids->[0], $bib_ids->[-1], $self->{total_indexed}));
+
+ my $batch_size = $state->{batch_size} || $DEFAULT_BIB_BATCH_SIZE;
return $index_count;
}
+sub index_sorter {
+ my ($body, $fname, $value) = @_;
+
+ $value = OpenILS::Utils::Normalize::search_normalize($value);
+
+ $value =~ s/^ +//g;
+
+ append_field_value($body, $fname, $value) if $value;
+}
+
+# Normalize the pubdate (used for sorting) to a single 4-digit year.
+# Pad with zeroes where the year fall short of 4 digits.
+sub index_pubdate {
+ my ($body, $value) = @_;
+
+ $value =~ s/\D//g;
+
+ return unless $value; # no numbers
+
+ $value = substr($value . '0' x 4, 0, 4);
+
+ return if $value eq '0000'; # treat as no date.
+
+ append_field_value($body, 'pubdate', $value) if $value;
+}
+
# Indexes ISBN10, ISBN13, and formatted values of both (with hyphens)
sub index_isbns {
return unless $value;
my %seen; # deduplicate values
-
- # Chop up the collected raw values into parts and let
- # Business::* tell us which parts looks like ISBNs.
- for my $token (split(/ /, $value)) {
- if (length($token) > 8) {
- my $isbn = Business::ISBN->new($token);
- if ($isbn && $isbn->is_valid) {
- if ($isbn->as_isbn10) {
- $seen{$isbn->as_isbn10->isbn} = 1;
- $seen{$isbn->as_isbn10->as_string} = 1;
- }
- if ($isbn->as_isbn13) {
- $seen{$isbn->as_isbn13->isbn} = 1;
- $seen{$isbn->as_isbn13->as_string} = 1;
- }
- }
+ my @values = OpenILS::Utils::Normalize::clean_isbns($value);
+ my $isbns = $values[0];
+ my $strings = $values[1];
+
+ for my $isbn (@$isbns) {
+ if ($isbn->as_isbn10) {
+ $seen{$isbn->as_isbn10->isbn} = 1; # compact
+ $seen{$isbn->as_isbn10->as_string} = 1; # with hyphens
+ }
+ if ($isbn->as_isbn13) {
+ $seen{$isbn->as_isbn13->isbn} = 1;
+ $seen{$isbn->as_isbn13->as_string} = 1;
}
}
+ # Add the unvalidated ISBNs
+ $seen{$_} = 1 for @$strings;
+
append_field_value($body, 'identifier|isbn', $_) foreach keys %seen;
}
return unless $value;
my %seen; # deduplicate values
+ my @issns = OpenILS::Utils::Normalize::clean_issns($value);
- # Chop up the collected raw values into parts and let
- # Business::* tell us which parts looks valid.
- for my $token (split(/ /, $value)) {
- my $issn = Business::ISSN->new($token);
- if ($issn && $issn->is_valid) {
- # no option in business::issn to get the unformatted value.
- (my $unformatted = $issn->as_string) =~ s/-//g;
- $seen{$unformatted} = 1;
- $seen{$issn->as_string} = 1;
- }
+ for my $issn (@issns) {
+ # no option in business::issn to get the unformatted value.
+ (my $unformatted = $issn->as_string) =~ s/-//g;
+ $seen{$unformatted} = 1;
+ $seen{$issn->as_string} = 1;
}
append_field_value($body, 'identifier|issn', $_) foreach keys %seen;
if (ref $body->{$fname}) {
# Three or more values encountered for field.
# Add to the list.
+ return if grep {$_ eq $value} @{$body->{$fname}}; # dupe
push(@{$body->{$fname}}, $value);
} else {
# Second value encountered for field.
# Upgrade to array storage.
+ return if $body->{$fname} eq $value; # dupe
$body->{$fname} = [$body->{$fname}, $value];
}
} else {
return $holdings;
}
-sub load_marc {
- my ($self, $bib_ids) = @_;
-
- my $bib_ids_str = join(',', @$bib_ids);
-
- my $marc_data = $self->get_db_rows(<<SQL);
-SELECT record, tag, subfield, value
-FROM metabib.real_full_rec
-WHERE record IN ($bib_ids_str)
-SQL
-
- $logger->info("ES found ".scalar(@$marc_data).
- " MARC rows for current record batch");
-
- my $marc = {};
- for my $row (@$marc_data) {
+sub add_marc_value {
+ my ($self, $rec, $tag, $subfield, $value) = @_;
- my $value = $row->{value};
- next unless defined $value && $value ne '';
+ # XSL uses '_' when no subfield is present (e.g. controlfields)
+ $subfield = undef if $subfield eq '_';
- my $subfield = $row->{subfield};
- my $rec_id = $row->{record};
- delete $row->{record}; # avoid adding this to the index
+ my ($match) = grep {
+ $_->{tag} eq $tag &&
+ ($_->{subfield} || '') eq ($subfield || '')
+ } @{$rec->{marc}};
- $row->{value} = $value = $self->truncate_value($value);
+ if ($match) {
+ if (ref $match->{value}) {
+ # 3rd or more instance of tag/subfield for this record.
- $marc->{$rec_id} = [] unless $marc->{$rec_id};
- delete $row->{subfield} unless defined $subfield;
-
- # Add values to existing record/tag/subfield rows.
-
- my $existing;
- for my $entry (@{$marc->{$rec_id}}) {
- next unless $entry->{tag} eq $row->{tag};
-
- if (defined $subfield) {
- if (defined $entry->{subfield}) {
- if ($subfield eq $entry->{subfield}) {
- $existing = $entry;
- last;
- }
- }
- } elsif (!defined $entry->{subfield}) {
- # Neither has a subfield value / not all tags have subfields
- $existing = $entry;
- last;
- }
- }
+ # avoid dupes
+ return if grep {$_ eq $value} @{$match->{value}};
- if ($existing) {
-
- $existing->{value} = [$existing->{value}] unless ref $existing->{value};
- push(@{$existing->{value}}, $value);
+ push(@{$match->{value}}, $value);
} else {
+ # 2nd instance of tag/subfield for this record.
+
+ # avoid dupes
+ return if $match->{value} eq $value;
- push(@{$marc->{$rec_id}}, $row);
+ $match->{value} = [$match->{value}, $value];
}
- }
-
- return $marc;
-}
+ } else {
+ # first instance of tag/subfield for this record.
+ $match = {tag => $tag, value => $value};
+ $match->{subfield} = $subfield if defined $subfield;
-sub index {
- my $self = shift;
- return $self->{index} if $self->{index};
- ($self->{index}) = grep {$_->code eq $self->index_name} @{$self->indices};
-
- $logger->error("No ndex configured named ".$self->index_name) unless $self->{index};
-
- return $self->{index};
+ push(@{$rec->{marc}}, $match);
+ }
}
-
# Add data to the bib-search index
sub populate_index {
my ($self, $settings) = @_;
$settings ||= {};
my $index_count = 0;
- my $total_indexed = 0;
+ $self->{total_indexed} = 0;
# extract the database settings.
for my $db_key (grep {$_ =~ /^db_/} keys %$settings) {
while (1) {
$index_count = $self->populate_bib_index_batch($settings);
- $total_indexed += $index_count;
-
- $logger->info("ES indexed $total_indexed bib records");
# exit if we're only indexing a single record or if the
# batch indexer says there are no more records to index.
}
}
- $logger->info("ES bib indexing complete with $total_indexed records");
+ $logger->info("ES bib indexing complete with " . $self->{total_indexed} . " records");
}
sub get_bib_ids {
my ($select, $from, $where);
if ($modified_since) {
+ $logger->info("ES bib indexing records modified since $modified_since");
$select = "SELECT id";
- $from = "FROM elastic.bib_last_mod_date";
- $where = "WHERE last_mod_date > '$modified_since'";
+ $from = "FROM elastic.bib_mod_since(QUOTE_LITERAL('$modified_since')::TIMESTAMPTZ)";
+ $where = "WHERE TRUE";
} else {
$select = "SELECT id";
$from = "FROM biblio.record_entry";
BEGIN;
-ALTER TABLE config.record_attr_definition
- ADD COLUMN elastic_field BOOLEAN NOT NULL DEFAULT FALSE;
-
-ALTER TABLE config.metabib_field
- ADD COLUMN elastic_field BOOLEAN NOT NULL DEFAULT FALSE;
-
--- Provide a sweeping set of default elastic fields.
--- Likely this set of fields can be trimmed significantly for most sites,
--- since many of these fields will never be searched from the catalog.
--- Reducing the number of elastic_field's will improve indexing time,
--- search time, and reduce Elastic disk space requirements.
-UPDATE config.record_attr_definition
- SET elastic_field = TRUE WHERE name NOT LIKE 'marc21_%';
-
-UPDATE config.metabib_field
- SET elastic_field = TRUE WHERE search_field OR facet_field;
-
-INSERT INTO config.global_flag (name, enabled, label)
-VALUES (
- 'elastic.bib_search.enabled', FALSE,
- 'Elasticsearch Enable Bib Searching'
-), (
- 'elastic.bib_search.dynamic_properties', FALSE,
- 'Elasticsearch Dynamic Bib Record Properties'
-);
-
CREATE SCHEMA elastic;
CREATE TABLE elastic.cluster (
port INTEGER NOT NULL,
path TEXT NOT NULL DEFAULT '/',
active BOOLEAN NOT NULL DEFAULT FALSE,
- cluster TEXT NOT NULL
+ cluster TEXT NOT NULL
REFERENCES elastic.cluster (code) ON DELETE CASCADE,
CONSTRAINT node_once UNIQUE (host, port, path, cluster)
);
-CREATE TABLE elastic.index (
- id SERIAL PRIMARY KEY,
- code TEXT NOT NULL, -- e.g. 'bib-search'
- cluster TEXT NOT NULL
- REFERENCES elastic.cluster (code) ON DELETE CASCADE,
- active BOOLEAN NOT NULL DEFAULT FALSE,
- num_shards INTEGER NOT NULL DEFAULT 1,
- CONSTRAINT index_type_once_per_cluster UNIQUE (code, cluster)
-);
-
-CREATE OR REPLACE VIEW elastic.bib_field AS
- SELECT fields.* FROM (
- SELECT
- NULL::INT AS metabib_field,
- crad.name,
- crad.label,
- NULL AS search_group,
- crad.sorter,
- FALSE AS search_field,
- FALSE AS facet_field,
- 1 AS weight
- FROM config.record_attr_definition crad
- WHERE crad.elastic_field
- UNION
- SELECT
- cmf.id AS metabib_field,
- cmf.name,
- cmf.label,
- cmf.field_class AS search_group,
- FALSE AS sorter,
- -- always treat identifier fields as non-search fields.
- (cmf.field_class <> 'identifier' AND cmf.search_field) AS search_field,
- cmf.facet_field,
- cmf.weight
- FROM config.metabib_field cmf
- WHERE cmf.elastic_field
- ) fields;
-
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_attrs(bre_id BIGINT)
-RETURNS TABLE (
- search_group TEXT,
- name TEXT,
- source BIGINT,
- value TEXT
-)
-AS $FUNK$
- SELECT DISTINCT record.* FROM (
- SELECT
- NULL::TEXT AS search_group,
- crad.name,
- mrs.source,
- mrs.value
- FROM metabib.record_sorter mrs
- JOIN config.record_attr_definition crad ON (crad.name = mrs.attr)
- WHERE mrs.source = $1 AND crad.elastic_field
- UNION
-
- -- record attributes
- SELECT
- NULL::TEXT AS search_group,
- crad.name,
- mraf.id AS source,
- mraf.value
- FROM metabib.record_attr_flat mraf
- JOIN config.record_attr_definition crad ON (crad.name = mraf.attr)
- WHERE mraf.id = $1 AND crad.elastic_field
- ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_static_props(bre_id BIGINT)
-RETURNS TABLE (
- search_group TEXT,
- name TEXT,
- source BIGINT,
- value TEXT
-)
-AS $FUNK$
- SELECT DISTINCT record.* FROM (
- SELECT
- cmf.field_class AS search_group,
- cmf.name,
- props.source,
- CASE WHEN cmf.joiner IS NOT NULL THEN
- REGEXP_SPLIT_TO_TABLE(props.value, cmf.joiner)
- ELSE
- props.value
- END AS value
- FROM (
- SELECT * FROM metabib.title_field_entry mtfe WHERE mtfe.source = $1
- UNION
- SELECT * FROM metabib.author_field_entry mafe WHERE mafe.source = $1
- UNION
- SELECT * FROM metabib.subject_field_entry msfe WHERE msfe.source = $1
- UNION
- SELECT * FROM metabib.series_field_entry msrfe WHERE msrfe.source = $1
- UNION
- SELECT * FROM metabib.keyword_field_entry mkfe WHERE mkfe.source = $1
- UNION
- SELECT * FROM metabib.identifier_field_entry mife WHERE mife.source = $1
- ) props
- JOIN config.metabib_field cmf ON (cmf.id = props.field)
- WHERE cmf.elastic_field
- ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_dynamic_props(bre_id BIGINT)
-RETURNS TABLE (
- search_group TEXT,
- name TEXT,
- source BIGINT,
- value TEXT
-)
-AS $FUNK$
- SELECT DISTINCT record.* FROM (
- SELECT
- cmf.field_class AS search_group,
- cmf.name,
- props.source,
- CASE WHEN cmf.joiner IS NOT NULL THEN
- REGEXP_SPLIT_TO_TABLE(props.value, cmf.joiner)
- ELSE
- props.value
- END AS value
- FROM biblio.extract_metabib_field_entry(
- $1, ' ', '{facet,search}',
- (SELECT ARRAY_AGG(id) FROM config.metabib_field WHERE elastic_field)
- ) props
- JOIN config.metabib_field cmf ON (cmf.id = props.field)
- ) record
-$FUNK$ LANGUAGE SQL STABLE;
-
-
-CREATE OR REPLACE FUNCTION elastic.bib_record_properties(bre_id BIGINT)
- RETURNS TABLE (
- search_group TEXT,
- name TEXT,
- source BIGINT,
- value TEXT
- )
- AS $FUNK$
-DECLARE
- props_func TEXT;
-BEGIN
-
- PERFORM 1 FROM config.internal_flag cif WHERE
- cif.name = 'elastic.bib_search.dynamic_properties' AND cif.enabled;
-
- IF FOUND THEN
- props_func := 'elastic.bib_record_dynamic_props';
- ELSE
- props_func := 'elastic.bib_record_static_props';
- END IF;
-
- RETURN QUERY EXECUTE $$
- SELECT DISTINCT record.* FROM (
- SELECT * FROM elastic.bib_record_attrs($$ || QUOTE_LITERAL(bre_id) || $$)
- UNION
- SELECT * FROM $$ || props_func || '(' || QUOTE_LITERAL(bre_id) || $$)
- ) record
- $$;
-END $FUNK$ LANGUAGE PLPGSQL;
-
-/* give me bibs I should upate */
-
-CREATE OR REPLACE VIEW elastic.bib_last_mod_date AS
+CREATE OR REPLACE FUNCTION elastic.bib_mod_since(since TIMESTAMPTZ)
+ RETURNS TABLE (id BIGINT) AS $FUNK$
/**
* Last update date for each bib, which is taken from most recent
* edit for either the bib, a linked call number, or a linked copy.
* If no call numbers are linked, uses the bib edit date only.
* Includes deleted data since it can impact indexing.
*/
- WITH mod_dates AS (
- SELECT bre.id,
- bre.edit_date,
- MAX(COALESCE(acn.edit_date, '1901-01-01')) AS max_call_number_edit_date,
- MAX(COALESCE(acp.edit_date, '1901-01-01')) AS max_copy_edit_date
- FROM biblio.record_entry bre
- LEFT JOIN asset.call_number acn ON (acn.record = bre.id)
- LEFT JOIN asset.copy acp ON (acp.call_number = acn.id)
- GROUP BY 1, 2
- ) SELECT dates.id,
- GREATEST(dates.edit_date,
- GREATEST(dates.max_call_number_edit_date, dates.max_copy_edit_date)
- ) AS last_mod_date
- FROM mod_dates dates;
+ SELECT bre.id FROM biblio.record_entry bre WHERE bre.edit_date > since
+ UNION
-/* SEED DATA ------------------------------------------------------------ */
-
-INSERT INTO elastic.cluster (code, label)
- VALUES ('main', 'Main Cluster');
-
-INSERT INTO elastic.node (label, host, proto, port, active, cluster)
- VALUES ('Localhost', 'localhost', 'http', 9200, TRUE, 'main');
+ SELECT bre.id
+ FROM biblio.record_entry bre
+ JOIN asset.call_number acn ON acn.record = bre.id
+ WHERE acn.edit_date > since
+ UNION
-INSERT INTO elastic.index (code, active, cluster)
- VALUES ('bib-search', TRUE, 'main');
+ SELECT bre.id
+ FROM biblio.record_entry bre
+ JOIN asset.call_number acn ON acn.record = bre.id
+ JOIN asset.copy acp ON acp.call_number = acn.id
+ WHERE acp.edit_date > since
-COMMIT;
+$FUNK$ LANGUAGE SQL;
-/* UNDO
-
-DROP SCHEMA IF EXISTS elastic CASCADE;
+/* SEED DATA ------------------------------------------------------------ */
-ALTER TABLE config.record_attr_definition DROP COLUMN elastic_field;
-ALTER TABLE config.metabib_field DROP COLUMN elastic_field;
+INSERT INTO config.global_flag (name, enabled, label, value)
+VALUES (
+ 'elastic.bib_search.enabled', FALSE,
+ 'Elasticsearch Enable Bib Searching', NULL
+);
-DELETE FROM config.global_flag WHERE name ~ 'elastic.*';
+INSERT INTO elastic.cluster (code, label) VALUES ('main', 'Main Cluster');
-*/
+INSERT INTO elastic.node (label, host, proto, port, active, cluster)
+ VALUES ('Localhost', 'localhost', 'http', 9200, TRUE, 'main');
/*
-
--- Sample narrower set of elastic fields to avoid duplication and
--- indexing data that will presumably never be searched in the catalog.
-
-UPDATE config.metabib_field SET elastic_field = FALSE
-WHERE
- (field_class = 'keyword' AND name <> 'keyword') OR
- (field_class = 'subject' AND name = 'complete') OR
- (field_class = 'author' AND name = 'first_author')
-;
-
-UPDATE config.record_attr_definition SET elastic_field = FALSE
-WHERE name NOT IN (
- 'authorsort',
- 'date1',
- 'date2',
- 'bib_level',
- 'icon_format',
- 'item_form',
- 'item_lang',
- 'item_type',
- 'lit_form',
- 'pubdate',
- 'search_format',
- 'titlesort',
- 'sr_format',
- 'vr_format'
-);
+-- turn it on for the UI
+UPDATE config.global_flag SET enabled = TRUE WHERE name = 'elastic.bib_search.enabled';
*/
+
+COMMIT;
use warnings;
use Getopt::Long;
use OpenSRF::Utils::JSON;
+use OpenSRF::Utils::Logger qw/:logger/;
use OpenILS::Utils::Fieldmapper;
use OpenILS::Utils::CStoreEditor;
use OpenILS::Elastic::BibSearch;
-my $help;
+my $lockfile;
+my $batch_size = 500;
my $osrf_config = '/openils/conf/opensrf_core.xml';
my $cluster = 'main';
+my @nodes;
+my $index_class = 'bib-search';
+my $bib_transform;
+my $es_config_file;
my $create_index;
my $delete_index;
-my $index_name = 'bib-search'; # only supported index at time of writing
+my $index_name;
+my $activate_index;
my $populate;
my $index_record;
my $start_record;
my $stop_record;
my $modified_since;
my $max_duration;
-my $batch_size = 500;
+my $skip_holdings;
+my $list_indices;
+my $no_opensrf;
+my $force;
+my $help;
# Database settings read from ENV by default.
my $db_host = $ENV{PGHOST} || 'localhost';
GetOptions(
'help' => \$help,
'osrf-config=s' => \$osrf_config,
+ 'lockfile=s' => \$lockfile,
'cluster=s' => \$cluster,
+ 'node=s' => \@nodes,
'create-index' => \$create_index,
'delete-index' => \$delete_index,
- 'index=s' => \$index_name,
+ 'index-name=s' => \$index_name,
+ 'index-class=s' => \$index_class,
'index-record=s' => \$index_record,
+ 'activate-index' => \$activate_index,
'start-record=s' => \$start_record,
'stop-record=s' => \$stop_record,
'modified-since=s' => \$modified_since,
'max-duration=s' => \$max_duration,
'batch-size=s' => \$batch_size,
+ 'bib-transform=s' => \$bib_transform,
+ 'es-config-file=s' => \$es_config_file,
+ 'skip-holdings' => \$skip_holdings,
+ 'list-indices' => \$list_indices,
+ 'no-opensrf' => \$no_opensrf,
+ 'force' => \$force,
'db-name=s' => \$db_name,
'db-host=s' => \$db_host,
'db-port=s' => \$db_port,
'populate' => \$populate
) || die "\nSee --help for more\n";
+$index_name = "$index_class-$index_name" if $index_name;
+
sub help {
print <<HELP;
Synopsis:
- $0 --delete-index --create-index --index bib-search --populate
+ $0 --index-class bib-search --index-name bib-search-take-2 \
+ --create-index --populate --activate-index
Options:
--osrf-config <file-path>
+ --lockfile <path-to-file>
+ Enables lock file controls over the process. If unset,
+ no lock file is created.
+
--db-name <$db_name>
--db-host <$db_host>
--db-port <$db_port>
Database connection values. This is the Evergreen database
where values should be extracted for elastic search indexing.
+ Beware that data loaded through Evergreen, e.g. elasticsearch
+ configuration data, will be loaded from the DB used by the
+ running Evergreen instance, regardless of these --db-*
+ settings.
+
Values default to their PG* environment variable equivalent.
+ --no-opensrf
+ Avoid connecting to OpenSRF. Requires passing at least
+ one --node.
+
+ --bib-transform <path_to_file>
+ Override the configured global config value for
+ 'elastic.bib_search.transform_file'
+
+ --es-config-file <path_to_file>
+ Override the default ES configuration XML file.
+
--cluster <name>
Specify a cluster name. Defaults to 'main'.
- --index <name>
- Specify an index name. Defaults to 'bib-search'.
+ --node <URL> [repeatable]
+ Override the configured ES nodes.
+
+ --index-class <class>
+ Specifies which data set the current index manages (e.g. bib-search)
+ Must match a well-known index class with backing code.
+
+ --index-name <name>
+ The index name will be automatically prepended with the
+ index class. e.g. "my-index" becomes "bib-search-my-index"
+ on the backend for the "bib-search" index class.
--delete-index
Delete the specified index and all of its data.
--create-index
Create an index whose name equals --index-name.
+ --activate-index
+ Activate the selected index while deactivating all other
+ indexes of the same index_class and cluster.
+
--batch-size <number>
Index at most this many records per batch.
Default is 500.
at regular intervals to keep the ES-indexed data in sync
with the EG data.
+ --skip-holdings
+ Bypass indexing the holdings data. This is useful
+ when reindexing for configuration changes, where the
+ underlying holdings data has not changed.
+
--max-duration <duration>
Stop indexing once the process has been running for this
amount of time.
are provided (e.g. --index-start-record) then all
applicable values will be indexed.
+ --list-indices
+ List all Elasticsearch indices represented in the
+ Evergreen database.
+
+ --force
+ Force various actions.
+
HELP
exit(0);
}
help() if $help;
-# connect to osrf...
-OpenSRF::System->bootstrap_client(config_file => $osrf_config);
-Fieldmapper->import(
- IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
-OpenILS::Utils::CStoreEditor::init();
+if ($lockfile) {
+
+ die "I seem to be running already. If not remove $lockfile, try again\n"
+ if -e $lockfile;
+
+ open(LOCK, ">$lockfile") or die "Cannot open lock file: $lockfile : $@\n";
+ print LOCK $$ or die "Cannot write to lock file: $lockfile : $@\n";
+ close LOCK;
+}
+
+# We only need to connect to opensrf to look up the nodes in the database.
+# If the nodes are provided and --no-opensrf is set, avoid the connection
+# and log to stdout.
+if (@nodes && $no_opensrf) {
+ $logger->set_log_stdout(1);
+ $logger->set_log_level($logger->INFO);
+
+} else {
+ OpenSRF::System->bootstrap_client(config_file => $osrf_config);
+ Fieldmapper->import(
+ IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
+ OpenILS::Utils::CStoreEditor::init();
+}
my $es;
-if ($index_name eq 'bib-search') {
- $es = OpenILS::Elastic::BibSearch->new($cluster);
+if ($index_class eq 'bib-search') {
+ $es = OpenILS::Elastic::BibSearch->new(
+ db_name => $db_name,
+ db_host => $db_host,
+ db_port => $db_port,
+ db_user => $db_user,
+ db_pass => 'REDACTED',
+ db_appn => $db_appn,
+ cluster => $cluster,
+ nodes => \@nodes,
+ xsl_file => $bib_transform,
+ index_name => $index_name,
+ maintenance_mode => 1,
+ es_config_file => $es_config_file,
+ skip_holdings => $skip_holdings
+ );
}
if (!$es) {
- die "Unknown index type: $index_name\n";
+ die "Unknown index class: $index_class\n";
}
$es->connect;
if ($delete_index) {
+
+ if (!$force) {
+ my $active = $es->active_index;
+ if ($active && $active eq $index_name) {
+ die "Index '$index_name' is active! " .
+ "Use --force to delete an active index.\n";
+ }
+ }
+
$es->delete_index or die "Index delete failed.\n";
}
if ($create_index) {
+
+ if ($index_name eq $index_class) {
+ die "An index name cannot match its index_class [$index_class]\n";
+ }
+
$es->create_index or die "Index create failed.\n";
}
if ($populate) {
my $settings = {
- db_name => $db_name,
- db_host => $db_host,
- db_port => $db_port,
- db_user => $db_user,
- db_pass => 'REDACTED',
- db_appn => $db_appn,
index_record => $index_record,
start_record => $start_record,
stop_record => $stop_record,
$es->populate_index($settings) or die "Index populate failed.\n";
}
+if ($activate_index) {
+ $es->activate_index or die "Index activation failed.\n";
+}
+
+if ($list_indices) {
+ my $indices = $es->indices;
+
+ for my $name (keys %{$indices}) {
+ my $index_def = $indices->{$name};
+
+ my @aliases;
+ if ($index_def) {
+ @aliases = keys(%{$index_def->{$name}->{aliases}});
+ } else {
+ warn "ES has no index named $name\n";
+ }
+
+ print sprintf(
+ "index_class=%s index_name=%s active=%s aliases=@aliases\n",
+ $es->index_class, $name,
+ $es->index_is_active($name) ? 'yes' : 'no');
+ }
+}
+
+unlink $lockfile if $lockfile;
+
+
my $help;
my $osrf_config = '/openils/conf/opensrf_core.xml';
my $cluster = 'main';
-my $index = 'bib-search';
+my $index_class = 'bib-search';
+my $index_name;
my $quiet = 0;
my $query_string;
'help' => \$help,
'osrf-config=s' => \$osrf_config,
'cluster=s' => \$cluster,
+ 'index-name=s' => \$index_name,
'quiet' => \$quiet,
) || die "\nSee --help for more\n";
print <<HELP;
Synopsis:
- $0
+ $0 --index-name <name>
Performs a series of canned bib record searches
+ Note if --index-name is omitted, the currently active index on
+ the 'bib-search' index class will be used.
+
HELP
exit(0);
}
IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
OpenILS::Utils::CStoreEditor::init();
-my $es = OpenILS::Elastic::BibSearch->new($cluster);
+my $es = OpenILS::Elastic::BibSearch->new(index_name => $index_name);
$es->connect;
+if ($es->index_name) {
+ print "Using bib-search index '" . $es->index_name . "'\n";
+} else {
+ die "No active 'bib-search' index found. ".
+ "Use --index-name or activate an index in the database.\n";
+}
+
print "Searching...\n";
for my $query_part (@$queries) {
my $help;
my $osrf_config = '/openils/conf/opensrf_core.xml';
my $cluster = 'main';
-my $index = 'bib-search';
+my @nodes;
+my $index_class = 'bib-search';
+my $index_name;
+my $field_group;
my $quiet = 0;
my $query_string;
'help' => \$help,
'osrf-config=s' => \$osrf_config,
'cluster=s' => \$cluster,
+ 'node=s' => \@nodes,
+ 'index-class=s' => \$index_class,
+ 'index-name=s' => \$index_name,
'quiet' => \$quiet,
) || die "\nSee --help for more\n";
print <<HELP;
Synopsis:
- $0
+ $0 --index-name <name>
Performs query string searches.
+ Note if --index-name is omitted, the currently active index on
+ the 'bib-search' index class will be used.
+
HELP
exit(0);
}
IDL => OpenSRF::Utils::SettingsClient->new->config_value("IDL"));
OpenILS::Utils::CStoreEditor::init();
-my $es = OpenILS::Elastic::BibSearch->new($cluster);
+my $es = OpenILS::Elastic::BibSearch->new(
+ maintenance_mode => 1, # allows access to inactive indexes
+ cluster => $cluster,
+ nodes => \@nodes,
+ field_group => $field_group,
+ index_name => $index_name
+);
$es->connect;
+if ($es->index_name) {
+ print "Using bib-search index '" . $es->index_name . "'\n";
+} else {
+ die "No active 'bib-search' index found. ".
+ "Use --index-name or activate an index in the database.\n";
+}
+
print <<MESSAGE;
Enter a query string to perform a search. Ctrl-c to exit.
next unless $query_string;
my $query = {
- _source => ['id', 'title|maintitle'] , # return only a few fields
+ _source => ['id', 'title|maintitle', 'author|personal'] , # return only a few fields
from => 0,
size => 10,
sort => [{'_score' => 'desc'}],
# Search the base keyword text index by default.
default_field => 'keyword.text'
}
+ },
+ # Request highligh data for title/author text fields.
+ # See below for logging highlight response data.
+ highlight => {
+ # Pre/Post tags modified to match stock Evergreen.
+ pre_tags => '<b class="oils_SH">',
+ post_tags => '</b>',
+ fields => {
+ 'title*.text' => {},
+ 'author*.text' => {}
+ }
}
};
$hit->{_id}, $hit->{_score},
($hit->{_source}->{'title|maintitle'} || '')
);
+
+# Uncomment to log highlighted field data.
+# for my $hl (keys %{$hit->{highlight}}) {
+# my @values = @{$hit->{highlight}->{$hl}};
+# print "\tHighlight: $hl => @values\n";
+# }
}
}
--- /dev/null
+<xsl:stylesheet
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:marc="http://www.loc.gov/MARC21/slim"
+ version="1.0">
+ <xsl:output encoding="UTF-8" method="text"/>
+
+ <!--
+ Prints one index value per line for data found by transforming
+ a MARCXML record.
+
+ Output:
+
+ $index_purpose $index_class $index_name $value
+
+ - $value is the only string in the output that may contain spaces.
+
+ e.g.
+
+ search subject topic South America
+ facet author personal Janey Jam "Jojo" Jones
+ -->
+
+ <xsl:template match="@*|node()">
+ <xsl:call-template name="compile_searches" />
+ <xsl:call-template name="compile_facets" />
+ <xsl:call-template name="compile_filters" />
+ <xsl:call-template name="compile_sorters" />
+ <xsl:call-template name="compile_marc" />
+ </xsl:template>
+
+ <xsl:template name="compile_searches">
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">650</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was topic -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">651</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was geographic -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">avxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">655</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was genre -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">630</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was uniftitle -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">adfgklmnoprstvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">600</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was name -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdfgjklmnopqrstuvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">610</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was corpname -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdfgklmnoprstuvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">611</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was meeting -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acdefgjklnpqstuvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">490</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">800</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">tflmnoprs</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">810</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">tflmnoprs</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">830</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">adfgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">100</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <xsl:with-param name="index_name">personal</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">100</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was personal -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">110</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">111</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was meeting -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acdegng</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">700</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was added_personal -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">710</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was coorporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">711</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was meeting -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acde</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">400</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was added_personal -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcd</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcd</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">411</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was meeting -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acdegq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">010</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">lccn</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">010</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">lccn</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">020</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">isbn</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">020</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">isbn</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">022</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">issn</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">022</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">issn</xsl:with-param>
+ <xsl:with-param name="index_subfields">y</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">022</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">issn</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">024</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">upc</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">024</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">upc</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">027</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">tech_number</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">027</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">tech_number</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">028</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">tech_number</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">074</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">sudoc</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">074</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">sudoc</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">086</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">sudoc</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">086</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">sudoc</xsl:with-param>
+ <xsl:with-param name="index_subfields">z</xsl:with-param>
+ </xsl:call-template>
+ <!-- NOTE bibcn depends on local values for asset.call_number_class -->
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">092</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">099</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">086</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">092</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">099</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">086</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">bibcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">901</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">bibid</xsl:with-param>
+ <xsl:with-param name="index_subfields">c</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">901</xsl:with-param>
+ <xsl:with-param name="field_class">identifier</xsl:with-param>
+ <xsl:with-param name="index_name">tcn</xsl:with-param>
+ <xsl:with-param name="index_subfields">c</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">130</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was uniform -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">210</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was abbreviated -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">222</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was magazine -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">240</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was uniform -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">245</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <xsl:with-param name="index_name">maintitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">245</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was maintitle -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">245</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was proper -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">245</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was responsibility -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">c</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">246</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was alternative -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgjklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">247</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was former -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">260</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">publisher</xsl:with-param>
+ <xsl:with-param name="index_subfields">b</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">264</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">publisher</xsl:with-param>
+ <xsl:with-param name="index_subfields">b</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">245</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">title</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">100</xsl:with-param>
+ <xsl:with-param name="field_class">keyword</xsl:with-param>
+ <xsl:with-param name="index_name">author</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">400</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcde</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">411</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was conference -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acdegq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">411</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was seriestitle -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">ptv</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">440</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">490</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">490</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was uniform -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">694</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">700</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was added -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">710</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was added -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">711</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was added -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">fklnpst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">730</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was added -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgjklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">740</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was added -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+
+ <!-- HERE HERE EREHE -->
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">780</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was previous -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">st</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">785</xsl:with-param>
+ <xsl:with-param name="field_class">title</xsl:with-param>
+ <!-- was succeeding -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">st</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">800</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was personal_series -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">800</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">fgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">810</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate_series -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">810</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcdn</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">811</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was conference_series -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="index_subfields">acdegnq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">811</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">fklnpstv</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_search_entry">
+ <xsl:with-param name="tag">830</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="index_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="keyword_full_entry" />
+ </xsl:template>
+
+ <xsl:template name="compile_facets">
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">650</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was topic -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcdvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">651</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was geographic -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">avxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">600</xsl:with-param>
+ <xsl:with-param name="field_class">subject</xsl:with-param>
+ <!-- was name -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcdfgjklmnopqrstuvxyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">490</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">a</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">800</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">tflmnoprs</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">810</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">tflmnoprs</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">830</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">adfgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">100</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was personal -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcdq</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">110</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">710</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">ab</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcd</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">400</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was corporate -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">410</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">411</xsl:with-param>
+ <xsl:with-param name="field_class">author</xsl:with-param>
+ <!-- was conference -->
+ <xsl:with-param name="index_name">combined</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">440</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcefghijklmnopqrstuvwyz</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">490</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">694</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields"></xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">800</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">fgklmnoprst</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">810</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcdn</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">811</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">fklnpstv</xsl:with-param>
+ </xsl:call-template>
+ <xsl:call-template name="add_facet_entry">
+ <xsl:with-param name="tag">830</xsl:with-param>
+ <xsl:with-param name="field_class">series</xsl:with-param>
+ <xsl:with-param name="index_name">seriestitle</xsl:with-param>
+ <xsl:with-param name="facet_subfields">abcefgijklmnopqrstuvwxyz</xsl:with-param>
+ </xsl:call-template>
+ </xsl:template>
+
+ <xsl:template name="compile_sorters">
+
+ <!-- author sort is the first 1XX value -->
+ <xsl:for-each select="marc:datafield[starts-with(@tag, '1')]">
+ <xsl:sort select="@tag"/>
+ <xsl:if test="position() = 1">
+ <xsl:call-template name="add_sorter_entry">
+ <xsl:with-param name="name">authorsort</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="subfieldSelect"></xsl:call-template>
+ </xsl:with-param>
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:for-each>
+
+ <!-- title sort is the 245a non-filing -->
+ <xsl:for-each select="marc:datafield[@tag='245']">
+ <!-- 245 is non-repeating but it happens. just take the first value -->
+ <xsl:if test="position() = 1">
+ <xsl:variable name="full_title">
+ <xsl:call-template name="subfieldSelect">
+ <xsl:with-param name="codes">a</xsl:with-param>
+ </xsl:call-template>
+ </xsl:variable>
+ <xsl:variable name="offset">
+ <xsl:choose>
+ <xsl:when test="number(@ind2) = @ind2">
+ <xsl:value-of select="@ind2" />
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:text>0</xsl:text>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+ <xsl:call-template name="add_sorter_entry">
+ <xsl:with-param name="name">titlesort</xsl:with-param>
+ <xsl:with-param name="value" select="substring($full_title, $offset + 1)" />
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:for-each>
+
+ <!-- pubdate is the same as the date1 filter -->
+ <xsl:call-template name="add_sorter_entry">
+ <xsl:with-param name="name">pubdate</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">7</xsl:with-param>
+ <xsl:with-param name="length">4</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ </xsl:call-template>
+
+ </xsl:template>
+
+ <xsl:template name="compile_filters">
+
+ <!-- start with filters that are not used within composite filters.
+ These can be added to the document inline. -->
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">date1</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">7</xsl:with-param>
+ <xsl:with-param name="length">4</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ <xsl:with-param name="default_value">0000</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">date2</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">11</xsl:with-param>
+ <xsl:with-param name="length">4</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ <xsl:with-param name="default_value">9999</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">lit_form</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">33</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">item_lang</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">35</xsl:with-param>
+ <xsl:with-param name="length">3</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">audience</xsl:with-param>
+ <xsl:with-param name="value">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">22</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:with-param>
+ </xsl:call-template>
+
+ <!-- Filters that may be used within composite filters are
+ stored in a local variable so they can first be added
+ to the document, then used to compile composite filters -->
+
+ <xsl:variable name="item_type">
+ <xsl:call-template name="leader_value">
+ <xsl:with-param name="offset">6</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">item_type</xsl:with-param>
+ <xsl:with-param name="value" select="$item_type" />
+ </xsl:call-template>
+
+ <xsl:variable name="bib_level">
+ <xsl:call-template name="leader_value">
+ <xsl:with-param name="offset">7</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">bib_level</xsl:with-param>
+ <xsl:with-param name="value" select="$bib_level" />
+ </xsl:call-template>
+
+ <xsl:variable name="item_form">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">008</xsl:with-param>
+ <xsl:with-param name="offset">23</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">item_form</xsl:with-param>
+ <xsl:with-param name="value" select="$item_form" />
+ </xsl:call-template>
+
+ <xsl:variable name="category_of_material">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">007</xsl:with-param>
+ <xsl:with-param name="offset">0</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:variable>
+
+ <xsl:variable name="vr_format">
+ <xsl:if test="$category_of_material = 'v'">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">007</xsl:with-param>
+ <xsl:with-param name="offset">4</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:variable>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">vr_format</xsl:with-param>
+ <xsl:with-param name="value" select="$vr_format" />
+ </xsl:call-template>
+
+ <xsl:variable name="sr_format">
+ <xsl:if test="$category_of_material = 's'">
+ <xsl:call-template name="controlfield_value">
+ <xsl:with-param name="tag">007</xsl:with-param>
+ <xsl:with-param name="offset">3</xsl:with-param>
+ <xsl:with-param name="length">1</xsl:with-param>
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:variable>
+
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name">sr_format</xsl:with-param>
+ <xsl:with-param name="value" select="$sr_format" />
+ </xsl:call-template>
+
+ <!-- use the extracted raw filters to create composite filters -->
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">blu-ray</xsl:with-param>
+ <xsl:with-param name="vr_format" select="$vr_format" />
+ <xsl:with-param name="vr_format_codes">s</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">book</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">at</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_type" />
+ <xsl:with-param name="item_form_not_codes">abcfoqrs</xsl:with-param>
+ <xsl:with-param name="bib_level" select="$bib_level" />
+ <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">braille</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">a</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">f</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">casaudiobook</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">i</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">l</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">casmusic</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">j</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">l</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">cdaudiobook</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">i</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">f</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">cdaudiobook</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">j</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">f</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">dvd</xsl:with-param>
+ <xsl:with-param name="vr_format" select="$vr_format" />
+ <xsl:with-param name="vr_format_codes">v</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">eaudio</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">i</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">ebook</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">at</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+ <xsl:with-param name="bib_level" select="$bib_level" />
+ <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">electronic</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">os</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">equip</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">r</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">evideo</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">g</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">oqs</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">kit</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">op</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">lpbook</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">at</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">d</xsl:with-param>
+ <xsl:with-param name="bib_level" select="$bib_level" />
+ <xsl:with-param name="bib_level_codes">acdm</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">map</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">ef</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">microform</xsl:with-param>
+ <xsl:with-param name="item_form" select="$item_form" />
+ <xsl:with-param name="item_form_codes">abc</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">music</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">j</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">phonomusic</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">j</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">abcde</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">phonospoken</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">i</xsl:with-param>
+ <xsl:with-param name="sr_format" select="$sr_format" />
+ <xsl:with-param name="sr_format_codes">abcde</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">picture</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">k</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">serial</xsl:with-param>
+ <xsl:with-param name="bib_level" select="$bib_level" />
+ <xsl:with-param name="bib_level_codes">bs</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">score</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">cd</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">software</xsl:with-param>
+ <xsl:with-param name="item_type" select="$item_type" />
+ <xsl:with-param name="item_type_codes">m</xsl:with-param>
+ </xsl:call-template>
+
+ <xsl:call-template name="add_composite_filter_entry">
+ <xsl:with-param name="name">search_format</xsl:with-param>
+ <xsl:with-param name="value">vhs</xsl:with-param>
+ <xsl:with-param name="vr_format" select="$vr_format" />
+ <xsl:with-param name="vr_format_codes">b</xsl:with-param>
+ </xsl:call-template>
+
+ </xsl:template>
+
+ <xsl:template name="add_sorter_entry">
+ <xsl:param name="name"/>
+ <xsl:param name="value"/>
+ <xsl:text>sorter _ </xsl:text>
+ <xsl:value-of select="$name" />
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="$value" />
+ <xsl:text>
</xsl:text>
+ </xsl:template>
+
+ <xsl:template name="add_filter_entry">
+ <xsl:param name="name"/>
+ <xsl:param name="value"/>
+ <xsl:param name="default_value"/>
+ <xsl:text>filter _ </xsl:text>
+ <xsl:value-of select="$name" />
+ <xsl:text> </xsl:text>
+ <xsl:choose>
+ <xsl:when test="$default_value and translate($value, ' ', '') = ''">
+ <xsl:value-of select="$default_value" />
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="$value" />
+ </xsl:otherwise>
+ </xsl:choose>
+ <xsl:text>
</xsl:text>
+ </xsl:template>
+
+ <xsl:template name="add_composite_filter_entry">
+ <xsl:param name="name"/>
+ <xsl:param name="value"/>
+ <xsl:param name="item_type"/>
+ <xsl:param name="item_type_codes"/>
+ <xsl:param name="item_form"/>
+ <xsl:param name="item_form_codes"/>
+ <xsl:param name="item_form_not_codes"/>
+ <xsl:param name="bib_level"/>
+ <xsl:param name="bib_level_codes"/>
+ <xsl:param name="vr_format"/>
+ <xsl:param name="vr_format_codes"/>
+ <xsl:param name="sr_format"/>
+ <xsl:param name="sr_format_codes"/>
+
+ <xsl:variable name="item_type_matches" select="
+ not($item_type_codes) or (
+ $item_type != '' and
+ contains($item_type_codes, $item_type)
+ )
+ "/>
+
+ <xsl:variable name="item_form_matches" select="
+ (
+ not($item_form_codes) or
+ contains($item_form_codes, $item_form)
+ ) and (
+ not($item_form_not_codes) or
+ not(contains($item_form_not_codes, $item_form))
+ )
+ "/>
+
+ <xsl:variable name="bib_level_matches" select="
+ not($bib_level_codes) or (
+ $bib_level != '' and
+ contains($bib_level_codes, $bib_level)
+ )
+ "/>
+
+ <xsl:variable name="vr_format_matches" select="
+ not($vr_format_codes) or (
+ $vr_format != '' and
+ contains($vr_format_codes, $vr_format)
+ )
+ "/>
+
+ <xsl:variable name="sr_format_matches" select="
+ not($sr_format_codes) or (
+ $sr_format != '' and
+ contains($sr_format_codes, $sr_format)
+ )
+ "/>
+
+ <xsl:if test="
+ $item_type_matches and
+ $item_form_matches and
+ $bib_level_matches and
+ $sr_format_matches and
+ $vr_format_matches">
+ <xsl:call-template name="add_filter_entry">
+ <xsl:with-param name="name" select="$name" />
+ <xsl:with-param name="value" select="$value" />
+ </xsl:call-template>
+ </xsl:if>
+ </xsl:template>
+
+ <xsl:template name="leader_value">
+ <xsl:param name="offset" /> <!-- zero-based -->
+ <xsl:param name="length" />
+ <xsl:for-each select="marc:leader">
+ <xsl:value-of select="substring(text(), $offset + 1, $length)"/>
+ </xsl:for-each>
+ </xsl:template>
+
+ <xsl:template name="controlfield_value">
+ <xsl:param name="tag" />
+ <xsl:param name="offset" /> <!-- zero-based -->
+ <xsl:param name="length" />
+ <xsl:for-each select="marc:controlfield[@tag=$tag]">
+ <xsl:value-of select="substring(text(), $offset + 1, $length)"/>
+ </xsl:for-each>
+ </xsl:template>
+
+ <!-- Produces a single value for the specific tab/subfields.
+ Should only be used in cases where a single value is expected. -->
+ <xsl:template name="datafield_value">
+ <xsl:param name="tag" />
+ <xsl:param name="subfields" />
+ <xsl:for-each select="marc:datafield[@tag=$tag]">
+ <xsl:call-template name="subfieldSelect">
+ <xsl:with-param name="codes">
+ <xsl:value-of select="$subfields" />
+ </xsl:with-param>
+ </xsl:call-template>
+ <xsl:text> </xsl:text>
+ </xsl:for-each>
+ </xsl:template>
+
+ <xsl:template name="subfieldSelect">
+ <xsl:param name="codes">abcdefghijklmnopqrstuvwxyz</xsl:param>
+ <xsl:param name="delimeter">
+ <xsl:text> </xsl:text>
+ </xsl:param>
+ <xsl:variable name="str">
+ <xsl:for-each select="marc:subfield">
+ <xsl:if test="contains($codes, @code)">
+ <xsl:value-of select="text()"/>
+ <xsl:value-of select="$delimeter"/>
+ </xsl:if>
+ </xsl:for-each>
+ </xsl:variable>
+ <xsl:value-of select="substring($str,1,string-length($str)-string-length($delimeter))"/>
+ </xsl:template>
+
+ <xsl:template name="add_search_entry">
+ <xsl:param name="tag" />
+ <xsl:param name="field_class" />
+ <xsl:param name="index_name" />
+ <xsl:param name="index_subfields" />
+ <xsl:for-each select="marc:datafield[@tag=$tag] |
+ marc:datafield[@tag='880']/marc:subfield[@code='6'][starts-with(., $tag)]/..">
+ <xsl:text>search </xsl:text>
+ <xsl:value-of select="$field_class" /><xsl:text> </xsl:text>
+ <xsl:value-of select="$index_name" /><xsl:text> </xsl:text>
+ <xsl:call-template name="subfieldSelect">
+ <xsl:with-param name="codes">
+ <xsl:value-of select="$index_subfields" />
+ </xsl:with-param>
+ </xsl:call-template>
+ <xsl:text>
</xsl:text><!-- newline -->
+ </xsl:for-each>
+ </xsl:template>
+
+ <xsl:template name="add_facet_entry">
+ <xsl:param name="tag" />
+ <xsl:param name="field_class" />
+ <xsl:param name="index_name" />
+ <xsl:param name="facet_subfields" />
+ <xsl:for-each select="marc:datafield[@tag=$tag] |
+ marc:datafield[@tag='880']/marc:subfield[@code='6'][starts-with(., $tag)]/..">
+ <xsl:text>facet </xsl:text>
+ <xsl:value-of select="$field_class"/><xsl:text> </xsl:text>
+ <xsl:value-of select="$index_name"/><xsl:text> </xsl:text>
+ <xsl:call-template name="subfieldSelect">
+ <xsl:with-param name="codes">
+ <xsl:value-of select="$facet_subfields" />
+ </xsl:with-param>
+ </xsl:call-template>
+ <xsl:text>
</xsl:text><!-- newline -->
+ </xsl:for-each>
+ </xsl:template>
+
+ <!-- Dumps practically the entire document into a single
+ keyword|keyword index.
+ -->
+ <xsl:template name="keyword_full_entry">
+ <xsl:text>search keyword keyword </xsl:text>
+ <xsl:for-each select="marc:datafield">
+ <xsl:call-template name="subfieldSelect" />
+ <xsl:text> </xsl:text>
+ </xsl:for-each>
+ <xsl:text>
</xsl:text><!-- newline -->
+ </xsl:template>
+
+ <!-- print: marc $tag $subfield $value -->
+ <xsl:template name="compile_marc">
+ <xsl:for-each select="marc:leader">
+ <xsl:text>marc LDR _ </xsl:text>
+ <xsl:value-of select="text()"/>
+ </xsl:for-each>
+ <xsl:for-each select="marc:controlfield">
+ <xsl:text>marc </xsl:text>
+ <xsl:value-of select="@tag" />
+ <xsl:text> _ </xsl:text>
+ <xsl:value-of select="text()"/>
+ <xsl:text>
</xsl:text><!-- newline -->
+ </xsl:for-each>
+ <xsl:for-each select="marc:datafield">
+ <xsl:variable name="tag" select="@tag" />
+ <xsl:for-each select="marc:subfield">
+ <xsl:text>marc </xsl:text>
+ <xsl:value-of select="$tag" />
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="@code" />
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="text()"/>
+ <xsl:text>
</xsl:text><!-- newline -->
+ </xsl:for-each>
+ </xsl:for-each>
+ </xsl:template>
+
+
+</xsl:stylesheet>
+
+
projects / working / Evergreen.git / commitdiff