Test failures in Perl Unicode normalization

Newer versions of Perl and Encode exhibit different behaviour from
before. The NFD version of entityize appears not to work, for example,
so this ignores the requested form and always operates on the NFC
version. We also use NFD() instead of decode_utf8() as we are operating
on Unicode data in this environment.

Signed-off-by: Dan Scott <dscott@laurentian.ca>

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Application/AppUtils.pm b/Open-ILS/src/perlmods/lib/OpenILS/Application/AppUtils.pm
index 5a6d385..be6fd2a 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Application/AppUtils.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Application/AppUtils.pm
@@ -1515,13 +1515,7 @@ sub entityize {
 
     # If we're going to convert non-ASCII characters to XML entities,
     # we had better be dealing with a UTF8 string to begin with
-    $string = decode_utf8($string);
-
-    if ($form eq 'D') {
-        $string = NFD($string);
-    } else {
-        $string = NFC($string);
-    }
+       $string = NFC($string);
 
     # Convert raw ampersands to entities
     $string =~ s/&(?!\S+;)/&/gso;

diff --git a/Open-ILS/src/perlmods/lib/OpenILS/Utils/Normalize.pm b/Open-ILS/src/perlmods/lib/OpenILS/Utils/Normalize.pm
index ac82ba4..24c85bf 100644 (file)
--- a/Open-ILS/src/perlmods/lib/OpenILS/Utils/Normalize.pm
+++ b/Open-ILS/src/perlmods/lib/OpenILS/Utils/Normalize.pm
@@ -12,7 +12,7 @@ use Exporter 'import';
 our @EXPORT_OK = qw( clean_marc naco_normalize search_normalize );
 
 sub naco_normalize {
-    my $str = decode_utf8(shift);
+    my $str = NFD(shift);
     my $sf = shift;
 
     # Apply NACO normalization to input string; based on
@@ -28,17 +28,17 @@ sub naco_normalize {
     
     $str = _normalize_codes($str, $sf);
 
-    return $str;
+    return NFC($str);
 }
 
 sub search_normalize {
-    my $str = decode_utf8(shift);
+    my $str = NFD(shift);
     my $sf = shift;
 
     $str = _normalize_substitutions($str, $sf);
     $str = _normalize_codes($str, $sf);
     
-    return $str;
+    return NFC($str);
 }
 
 sub _normalize_substitutions {
@@ -111,14 +111,14 @@ sub _normalize_codes {
 # Assumes input is already in UTF-8.
 sub clean_marc {
     my $input = shift;
-    my $xml = decode_utf8((UNIVERSAL::isa($input, 'MARC::Record')) ? $input->as_xml_record() : $input);
+    my $xml = NFD((UNIVERSAL::isa($input, 'MARC::Record')) ? $input->as_xml_record() : $input);
     $xml =~ s/\n//sog;
     $xml =~ s/^<\?xml.+\?\s*>//go;
     $xml =~ s/>\s+</></go;
     $xml =~ s/\p{Cc}//go;
     $xml = OpenILS::Application::AppUtils->entityize($xml);
     $xml =~ s/[\x00-\x1f]//go;
-    return $xml;
+    return NFC($xml);
 }
 
 1;

diff --git a/Open-ILS/src/perlmods/t/14-OpenILS-Utils.t b/Open-ILS/src/perlmods/t/14-OpenILS-Utils.t
index 621c98f..86faa7a 100644 (file)
--- a/Open-ILS/src/perlmods/t/14-OpenILS-Utils.t
+++ b/Open-ILS/src/perlmods/t/14-OpenILS-Utils.t
@@ -1,7 +1,8 @@
 #!perl -T
 
-use Test::More tests => 30;
+use Test::More tests => 32;
 use Test::Warn;
+use utf8;
 
 use_ok( 'OpenILS::Utils::Configure' );
 use_ok( 'OpenILS::Utils::Cronscript' );
@@ -86,6 +87,8 @@ my $exp_xml = '<record><leader>01614nmm a22003975u 4500</leader><controlfield ta
 my $clean_xml = OpenILS::Utils::Normalize::clean_marc($raw_marcxml);
 is($clean_xml, $exp_xml, "clean_marc: header and space normalization");
 
+is(OpenILS::Application::AppUtils->entityize('èöçÇÈÀ'), '&#xE8;&#xF6;&#xE7;&#xC7;&#xC8;&#xC0;', 'entityize: diacritics NFC');
+is(OpenILS::Application::AppUtils->entityize('èöçÇÈÀ', 'D'), '&#xE8;&#xF6;&#xE7;&#xC7;&#xC8;&#xC0;', 'entityize: diacritics NFD');
 is(OpenILS::Utils::Normalize::clean_marc('èöçÇÈÀ'), '&#xE8;&#xF6;&#xE7;&#xC7;&#xC8;&#xC0;', 'clean_marc: diacritics');
 
 my $edi_invoice = "UNA:+.? 'UNB+UNOC:3+1556150:31B+123EVER:31B+120926:1621+4'UNH+11+INVOIC:D:96A:UN'BGM+380+5TST084026+9'DTM+137:20120924:102'RFF+ON:24'NAD+BY+123EVER 0001::91'NAD+SU+1691503::31B'CUX+2:USD:4'LIN+1++9780446360272'QTY+47:5'MOA+146:4.5:USD:10'MOA+203:14.65'PRI+AAF:2.93:DI:NTP'RFF+LI:24/102'LIN+2++9780446357197'QTY+47:8'MOA+146:6.5:USD:10'MOA+203:33.84'PRI+AAF:4.23:DI:NTP'RFF+LI:24/100'UNS+S'MOA+86:66.18'ALC+C++++DL'MOA+8:2'ALC+C++++CA'MOA+131:12.3'ALC+C++++TX'MOA+8:3.39'UNT+28+11'UNH+12+INVOIC:D:96A:UN'BGM+380+5TST084027+9'DTM+137:20120924:102'RFF+ON:26'NAD+BY+123EVER 0001::91'NAD+SU+1691503::31B'CUX+2:USD:4'LIN+1++9780446360272'QTY+47:1'MOA+146:4.5:USD:10'MOA+203:4.05'PRI+AAF:4.05:DI:NTP'RFF+LI:26/106'LIN+2++9780446350105'QTY+47:3'MOA+146:6.99:USD:10'MOA+203:14.67'PRI+AAF:4.89:DI:NTP'RFF+LI:26/105'UNS+S'MOA+86:25.03'ALC+C++++DL'MOA+8:2'ALC+C++++CA'MOA+131:3'ALC+C++++TX'MOA+8:1.31'UNT+28+12'UNZ+4+4'";