LP 1463943: More Encoding Improvements.

First, we improve the encoding search by looking in the institution
config proper.  The search order for an encoding is now:

1. An encoding attribute specified in the account tag.

2. An encoding tag in the institution tag.

3. An encoding tag in the implementation_config tag of the institution
tag as a last resort.

4. Use ASCII if none is set.  (The SIP2 specification practially
requires this.)

The last is done for backward compatibility with existing Evergreen
oils_sip.xml files.  This should be deprecated in the near future and
only the account tag and institution tag proper should be supported.

Second, we improve the handling of ASCII output by stealing a code
chunk from Evergreen's OILS::SIP->clean_text routine to remove
combining characters on strings.

Third, we remove the encoding attribute from the sample SIPconfig.xml
account tag for lpl-sc, and put an encoding tag in the institution
tags for LPL and UWOLS.

Signed-off-by: Jason Stephenson <jason@sigio.com>

diff --git a/SIPconfig.xml b/SIPconfig.xml
index c2038a7..2e8d240 100644 (file)
--- a/SIPconfig.xml
+++ b/SIPconfig.xml
@@ -63,7 +63,7 @@
       </login>
       <login id="scclient-2" password="clientpwd-2"
              institution="UWOLS" />
-      <login id="lpl-sc" password="1234" institution="LPL" encoding="ascii" />
+      <login id="lpl-sc" password="1234" institution="LPL" />
       <login id="lpl-sc-beacock" password="xyzzy" location_code="WORKSTATION5"
              delimiter="|" error-detect="enabled" institution="LPL" />
   </accounts>
@@ -80,9 +80,11 @@
                  timeout="600" client_location_code="true"
                  retries="3" />
           <relais_extensions_to_msg24 enabled="false" />
+         <encoding>ascii</encoding>
     </institution>
 
     <institution id="LPL" implementation="ILS">
+         <encoding>UTF-8</encoding>
     </institution>
 </institutions>
 </acsconfig>

diff --git a/Sip.pm b/Sip.pm
index d800f57..cf9e575 100644 (file)
--- a/Sip.pm
+++ b/Sip.pm
@@ -26,6 +26,7 @@ use warnings;
 use English;
 use Exporter;
 use Encode;
+use Unicode::Normalize;
 
 use Sys::Syslog qw(syslog);
 use POSIX qw(strftime);
@@ -234,7 +235,26 @@ sub read_SIP_packet {
 sub write_msg {
     my ($self, $msg, $file, $encoding) = @_;
 
-    $msg = encode($encoding, $msg) if ($encoding);
+    if ($encoding) {
+        # Use nomalization form D, because some conversion blow up
+        # without it.
+        $msg = NFD($msg);
+        if ($encoding eq 'ascii') {
+            # Try to maintain a reasonable version of the content by
+            # stripping diacritics from the text, given that the SIP client
+            # wants just plain ASCII. This is the base requirement according
+            # to the SIP2 specification.
+
+            # Stripping the combining characters converts ""béè♁ts"
+            # into "bee?ts" instead of "b???ts" - better, eh?
+            $msg =~ s/\pM+//og;
+        }
+        $msg = encode($encoding, $msg);
+    } else {
+        # Send an UTF-8 string.  This makes the length call, below
+        # succeed and fixes LP 1628995.
+        $msg = encode_utf8($msg);
+    }
 
     if ($error_detection) {
         if (defined($self->{seqno})) {

diff --git a/Sip/MsgType.pm b/Sip/MsgType.pm
index 981b28b..bd7cb52 100644 (file)
--- a/Sip/MsgType.pm
+++ b/Sip/MsgType.pm
@@ -895,7 +895,15 @@ sub _load_ils_handler {
     $server->{policy}      = $server->{institution}->{policy};
     $server->{account}->{location} = $sc_loc if $sc_loc;
     # Set the encoding for responses messages.
-    $server->{encoding} = $server->{account}->{encoding} || $server->{institution}->{implementation_config}->{encoding};
+    $server->{encoding} = $server->{account}->{encoding}
+        || $server->{institution}->{encoding}
+        || $server->{institution}->{implementation_config}->{encoding}
+        || 'ascii'; # Use ascii if not set.  The spec expects this.
+    # We shouldn't be looking at the implementation config here, but
+    # that's where the encoding lived for the longest time.  So, we
+    # look there in the interest of backward compatibility.  That
+    # should be officially deprecated at some point, and that check
+    # removed.
 
     syslog("LOG_INFO", "Successful login for '%s' of '%s'", $server->{account}->{id}, $inst);
     #