Backport r15319 and r15320 from trunk to fix SRU/Z39.50 searches for diacritics

git-svn-id: svn://svn.open-ils.org/ILS/branches/rel_1_6@15388 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/perlmods/OpenILS/Application/AppUtils.pm b/Open-ILS/src/perlmods/OpenILS/Application/AppUtils.pm
index 496222c..eb77f24 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/Application/AppUtils.pm
+++ b/Open-ILS/src/perlmods/OpenILS/Application/AppUtils.pm
@@ -13,6 +13,7 @@ use OpenILS::Utils::CStoreEditor;
 use OpenILS::Const qw/:const/;
 use Unicode::Normalize;
 use OpenSRF::Utils::SettingsClient;
+use Encode;
 
 # ---------------------------------------------------------------------------
 # Pile of utilty methods used accross applications.
@@ -1474,6 +1475,10 @@ sub entityize {
     my($self, $string, $form) = @_;
        $form ||= "";
 
+       # If we're going to convert non-ASCII characters to XML entities,
+       # we had better be dealing with a UTF8 string to begin with
+       $string = decode_utf8($string);
+
        if ($form eq 'D') {
                $string = NFD($string);
        } else {

diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/SuperCat.pm b/Open-ILS/src/perlmods/OpenILS/WWW/SuperCat.pm
index 62a3a82..3a38691 100644 (file)
--- a/Open-ILS/src/perlmods/OpenILS/WWW/SuperCat.pm
+++ b/Open-ILS/src/perlmods/OpenILS/WWW/SuperCat.pm
@@ -1640,8 +1640,13 @@ sub sru_search {
        my ($shortname, $holdings) = $url =~ m#/?([^/]*)(/holdings)?#;
 
        if ( $resp->type eq 'searchRetrieve' ) {
-               my $cql_query = decode_utf8($req->query);
-               my $search_string = decode_utf8($req->cql->toEvergreen);
+
+               # These terms are arriving to us double-encoded, so until we
+               # figure out where in the CGI/SRU chain that's happening, we
+               # have to # forcefully double-decode them a second time with
+               # the outer decode('utf8', $string) call
+               my $cql_query = decode('utf8', decode_utf8($req->query));
+               my $search_string = decode('utf8', decode_utf8($req->cql->toEvergreen));
 
                # Ensure the search string overrides the default site
                if ($shortname and $search_string !~ m#site:#) {