removing control characters from XML with reckless abandon

git-svn-id: svn://svn.open-ils.org/ILS/branches/rel_1_2@7699 dcc99617-32d9-48b4-a31d-7c20da2025e4

diff --git a/Open-ILS/src/extras/import/marc2are.pl b/Open-ILS/src/extras/import/marc2are.pl
index 652a6a6..ecf0c42 100755 (executable)
--- a/Open-ILS/src/extras/import/marc2are.pl
+++ b/Open-ILS/src/extras/import/marc2are.pl
@@ -66,6 +66,7 @@ while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
        $xml =~ s/>\s+</></go;
        $xml =~ s/\p{Cc}//go;
        $xml = entityize($xml);
+       $xml =~ s/[\x00-\x1f]//go;
 
        my $bib = new Fieldmapper::authority::record_entry;
        $bib->id($id);

diff --git a/Open-ILS/src/extras/import/marc2bre.pl b/Open-ILS/src/extras/import/marc2bre.pl
index 0dc9929..5aba669 100755 (executable)
--- a/Open-ILS/src/extras/import/marc2bre.pl
+++ b/Open-ILS/src/extras/import/marc2bre.pl
@@ -158,6 +158,7 @@ while ( try { $rec = $batch->next } otherwise { $rec = -1 } ) {
        $xml =~ s/>\s+</></go;
        $xml =~ s/\p{Cc}//go;
        $xml = entityize($xml);
+       $xml =~ s/[\x00-\x1f]//go;
 
        my $bib = new Fieldmapper::biblio::record_entry;
        $bib->id($id);

diff --git a/Open-ILS/src/extras/marc2html b/Open-ILS/src/extras/marc2html
index c076e4c..f367c70 100755 (executable)
--- a/Open-ILS/src/extras/marc2html
+++ b/Open-ILS/src/extras/marc2html
@@ -1,19 +1,21 @@
 #!/usr/bin/perl
 
-use Error;
+use Error qw/:try/;
 use MARC::Batch;
 use MARC::File::XML;
 use XML::LibXSLT;
 use XML::LibXML;
 use Unicode::Normalize;
 use Getopt::Long;
+use FileHandle;
 
-my ($split,$enc,$marc,$out) = (100);
+my ($split,$enc,$marc,$out,$bad) = (100);
 GetOptions(
        'split=i' => \$split,
        'marc=s'  => \$marc,
        'encoding=s'  => \$enc,
        'out_dir=s'  => \$out,
+       'bad=s'  => \$bad,
 );
 
 if ($enc) {
@@ -31,6 +33,7 @@ my $xslt = XML::LibXSLT->new();
 
 $stylesheet = $xslt->parse_stylesheet( $parser->parse_string($xsl) );
 
+$bad = new FileHandle( $bad => '>:raw' ) if ($bad);
 
 my $xml = '';
 my $current = 1;
@@ -42,7 +45,20 @@ $marc->strict_off;
 $marc->warnings_off;
 
 while (my $r = $marc->next) {
-       $xml .= entityize(MARC::File::XML::record($r));
+       my $rxml = entityize(MARC::File::XML::record($r));
+       $rxml =~ s/[\x00-\x1f]//go;
+
+       try { $doc = $parser->parse_string($rxml); }
+       catch Error with {
+               my $e = shift;
+               warn "arg ... bad record $current, skipping: $e\n";
+               $current++;
+               print $bad $r->as_usmarc if ($bad);
+               $r = undef;
+       };
+       next unless ($r);
+
+       $xml .= $rxml;
 
        unless ($current % $split) {
                $xml = <<"              XML";
@@ -51,7 +67,11 @@ while (my $r = $marc->next) {
                        </collection>
                XML
 
-               my $doc = $parser->parse_string($xml);
+               my $doc;
+               try { $doc = $parser->parse_string($xml); }
+               catch Error with { my $e = shift; warn "ARG! Doc failed to parse:\n$e\n-------------------------------------------\n$xml\n"; };
+               die unless $doc;
+
                $xml = '';
 
                my $results = $stylesheet->transform($doc, prev => "'$prev'", next => "'$next'");