From 08bbebd7dc63982befe840de0071fda7a19ff04b Mon Sep 17 00:00:00 2001 From: erickson Date: Wed, 29 Aug 2007 20:16:04 +0000 Subject: [PATCH] moved image fetching into the syndetics added content plugin. added a caching layer to all added content git-svn-id: svn://svn.open-ils.org/ILS/trunk@7740 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm | 208 ++++++-------- .../perlmods/OpenILS/WWW/AddedContent/Syndetic.pm | 314 +++++++++++---------- 2 files changed, 262 insertions(+), 260 deletions(-) diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm index 073f6ec217..9cc6abf962 100644 --- a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm +++ b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm @@ -21,10 +21,11 @@ use OpenSRF::Utils::Logger qw/$logger/; use LWP::UserAgent; use MIME::Base64; +my $AC = __PACKAGE__; + # set the bootstrap config when this module is loaded my $bs_config; -my $handler; sub import { my $self = shift; @@ -32,51 +33,44 @@ sub import { } -my $net_timeout; # max seconds to wait for a response from the added content vendor +my $handler; # added content handler class handle my $cache; # memcache handle +my $net_timeout; # max seconds to wait for a response from the added content vendor my $max_errors; # max consecutive lookup failures before added content is temporarily disabled my $error_countdown; # current consecutive errors countdown -my $jacket_url; # URL for fetching jacket images # number of seconds to wait before next lookup # is attempted after lookups have been disabled my $error_retry_timeout; -my $init = 0; # has the child init been run? sub child_init { OpenSRF::System->bootstrap_client( config_file => $bs_config ); + $cache = OpenSRF::Utils::Cache->new; my $sclient = OpenSRF::Utils::SettingsClient->new(); my $ac_data = $sclient->config_value("added_content"); return unless $ac_data; - - $cache = OpenSRF::Utils::Cache->new; + my $ac_handler = $ac_data->{module}; + return unless $ac_handler; $net_timeout = $ac_data->{timeout} || 1; $error_countdown = $max_errors = $ac_data->{max_errors} || 10; - $jacket_url = $ac_data->{jacket_url}; $error_retry_timeout = $ac_data->{retry_timeout} || 600; - $init = 1; - - my $ac_handler = $ac_data->{module}; + $logger->debug("Attempting to load Added Content handler: $ac_handler"); - if($ac_handler) { - $logger->debug("Attempting to load Added Content handler: $ac_handler"); - - eval "use $ac_handler"; - - if($@) { - $logger->error("Unable to load Added Content handler [$ac_handler]: $@"); - return; - } - - $handler = $ac_handler->new($ac_data); - $logger->debug("added content loaded handler: $handler"); + eval "use $ac_handler"; + + if($@) { + $logger->error("Unable to load Added Content handler [$ac_handler]: $@"); + return; } + + $handler = $ac_handler->new($ac_data); + $logger->debug("added content loaded handler: $handler"); } @@ -85,54 +79,81 @@ sub handler { my $r = shift; my $cgi = CGI->new; my $path = $r->path_info; + my $res; - my( undef, $data, $format, $key ) = split(/\//, $r->path_info); - return Apache2::Const::NOT_FOUND unless $data and $format and $key; + my( undef, $type, $format, $key ) = split(/\//, $r->path_info); - child_init() unless $init; - return Apache2::Const::NOT_FOUND unless $init; + child_init() unless $handler; - return fetch_jacket($format, $key) if $data eq 'jacket'; - return Apache2::Const::NOT_FOUND unless lookups_enabled(); + return Apache2::Const::NOT_FOUND unless $handler and $type and $format and $key; + return $res if defined($res = $AC->serve_from_cache($type, $format, $key)); + return Apache2::Const::NOT_FOUND unless $AC->lookups_enabled; my $err; - my $success; - my $method = "${data}_${format}"; + my $data; + my $method = "${type}_${format}"; + + return Apache2::Const::NOT_FOUND unless $handler->can($method); try { - $success = $handler->$method($key); - } catch Error with { - my $err = shift; - $logger->error("added content handler failed: $method($key) => $err"); + $data = $handler->$method($key); + } catch Error with { + $err = shift; decr_error_countdown(); + $logger->error("added content handler failed: $method($key) => $err"); }; - return Apache2::Const::NOT_FOUND if $err or !$success; + return Apache2::Const::NOT_FOUND if $err; + + if(!$data) { + # if the AC lookup found no corresponding data, cache that information + $logger->debug("added content handler returned no results $method($key)") unless $data; + $AC->cache_result($type, $format, $key, {nocontent=>1}); + return Apache2::Const::NOT_FOUND; + } + + $AC->print_content($data); + $AC->cache_result($type, $format, $key, $data); + + reset_error_countdown(); return Apache2::Const::OK; } -sub decr_error_countdown { - $error_countdown--; - if($error_countdown < 1) { - $logger->warn("added content error count exhausted. Disabling lookups for $error_retry_timeout seconds"); - disable_lookups(); +sub print_content { + my($class, $data, $from_cache) = @_; + return Apache2::Const::NOT_FOUND if $data->{nocontent}; + + my $ct = $data->{content_type}; + my $content = $data->{content}; + print "Content-type: $ct\n\n"; + + if($data->{binary}) { + binmode STDOUT; + # if it hasn't been cached yet, it's still in binary form + print( ($from_cache) ? decode_base64($content) : $content ); + } else { + print $content; } -} -sub reset_error_countdown { - $error_countdown = $max_errors; + + return Apache2::Const::OK; } -# generic GET call + + +# returns an HTPP::Response object sub get_url { my( $self, $url ) = @_; - $logger->info("added content getting [timeout=$net_timeout] URL = $url"); + + $logger->info("added content getting [timeout=$net_timeout, errors_remaining=$error_countdown] URL = $url"); my $agent = LWP::UserAgent->new(timeout => $net_timeout); - my $res = $agent->get($url); + + my $res = $agent->get($url); + $logger->info("added content request returned with code " . $res->code); die "added content request failed: " . $res->status_line ."\n" unless $res->is_success; - reset_error_countdown(); - return $res->content; + + return $res; } sub lookups_enabled { @@ -147,82 +168,33 @@ sub disable_lookups { $cache->put_cache('ac.no_lookup', 1, $error_retry_timeout); } -sub fetch_jacket { - my($size, $isbn) = @_; - return Apache2::Const::NOT_FOUND unless $jacket_url and $size and $isbn; - - if($size eq 'small') { - - # try to serve small images from the cache first - my $img_data = $cache->get_cache("ac.$size.$isbn"); - - if($img_data) { - - $logger->debug("serving jacket $isbn from cache..."); - - my $c_type = $img_data->{content_type}; - my $img = decode_base64($img_data->{img}); - - print "Content-type: $c_type\n\n"; - - binmode STDOUT; - print $img; - return Apache2::Const::OK; - } - } - - if(!lookups_enabled()) { - $error_countdown = $max_errors; # reset the counter - return Apache2::Const::NOT_FOUND; - } - - (my $url = $jacket_url) =~ s/\${isbn}/$isbn/ig; - - $logger->debug("added content getting jacket with timeout=$net_timeout and URL = $url"); - - my $res; - my $err; - - try { - my $agent = LWP::UserAgent->new(timeout => $net_timeout); - $res = $agent->get($url); - } catch Error with { - $err = shift; - $logger->error("added content lookup died with $err"); - }; - - if( $err or $res->code == 500 ) { - $logger->warn("added content jacket fetch failed (retries remaining = $error_countdown) " . - (($res) ? $res->status_line : "$err")); - decr_error_countdown(); - return Apache2::Const::NOT_FOUND; +sub decr_error_countdown { + $error_countdown--; + if($error_countdown < 1) { + $logger->warn("added content error count exhausted. Disabling lookups for $error_retry_timeout seconds"); + $AC->disable_lookups; } +} - return Apache2::Const::NOT_FOUND unless $res->code == 200; - - # ignore old errors after a successful lookup - reset_error_countdown(); - - my $c_type = $res->header('Content-type'); - my $binary_img = $res->content; - print "Content-type: $c_type\n\n"; - - binmode STDOUT; - print $binary_img; - - $cache->put_cache( - "ac.$size.$isbn", { - content_type => $c_type, - img => encode_base64($binary_img,'') - } - ) if $size eq 'small'; - - return Apache2::Const::OK; +sub reset_error_countdown { + $error_countdown = $max_errors; } +sub cache_result { + my($class, $type, $format, $key, $data) = @_; + $logger->debug("caching $type/$format/$key"); + $data->{content} = encode_base64($data->{content}) if $data->{binary}; + return $cache->put_cache("ac.$type.$format.$key", $data); +} +sub serve_from_cache { + my($class, $type, $format, $key) = @_; + my $data = $cache->get_cache("ac.$type.$format.$key"); + return undef unless $data; + $logger->debug("serving $type/$format/$key from cache"); + return $class->print_content($data, 1); +} 1; - diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm index 38f9bfd4df..6ae2a5ab64 100644 --- a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm +++ b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm @@ -1,230 +1,260 @@ package OpenILS::WWW::AddedContent::Syndetic; use strict; use warnings; -use LWP::UserAgent; use OpenSRF::Utils::Logger qw/$logger/; use OpenSRF::Utils::SettingsParser; use OpenSRF::Utils::JSON; use OpenSRF::EX qw/:try/; use OpenILS::WWW::AddedContent; +use XML::LibXML; +use MIME::Base64; +my $AC = 'OpenILS::WWW::AddedContent'; sub new { - my( $class, $args ) = @_; - $class = ref $class || $class; - return bless($args, $class); + my( $class, $args ) = @_; + $class = ref $class || $class; + return bless($args, $class); } sub base_url { - my $self = shift; - return $self->{base_url}; + my $self = shift; + return $self->{base_url}; } sub userid { - my $self = shift; - return $self->{userid}; + my $self = shift; + return $self->{userid}; } # -------------------------------------------------------------------------- +sub jacket_small { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('sc.gif', $key, 1)); +} + +sub jacket_medium { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('mc.gif', $key, 1)); + +} +sub jacket_large { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('lc.gif', $key, 1)); +} + +# -------------------------------------------------------------------------- sub toc_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('toc.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('toc.html', $key)); } sub toc_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('toc.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('toc.xml', $key)); } sub toc_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('toc.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('toc.xml', $key)); } # -------------------------------------------------------------------------- sub anotes_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('anotes.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('anotes.html', $key)); } sub anotes_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('anotes.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('anotes.xml', $key)); } sub anotes_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('anotes.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('anotes.xml', $key)); } # -------------------------------------------------------------------------- sub excerpt_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('dbchapter.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('dbchapter.html', $key)); } sub excerpt_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('dbchapter.xml', $key)); } sub excerpt_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('dbchapter.xml', $key)); } # -------------------------------------------------------------------------- sub reviews_html { - my( $self, $key ) = @_; - - my %reviews; - - $reviews{ljreview} = $self->fetch_content('ljreview.html', $key); - $reviews{pwreview} = $self->fetch_content('pwreview.html', $key); - $reviews{slreview} = $self->fetch_content('slreview.html', $key); - $reviews{chreview} = $self->fetch_content('chreview.html', $key); - $reviews{blreview} = $self->fetch_content('blreview.html', $key); - $reviews{hbreview} = $self->fetch_content('hbreview.html', $key); - $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key); - - for(keys %reviews) { - if( ! $self->data_exists($reviews{$_}) ) { - delete $reviews{$_}; - next; - } - $reviews{$_} =~ s///og; # Strip any doctype declarations - } - - return 0 if scalar(keys %reviews) == 0; - - #my $html = "
"; - my $html; - $html .= $reviews{$_} for keys %reviews; - #$html .= "
"; - - return $self->send_html($html); + my( $self, $key ) = @_; + + my %reviews; + + $reviews{ljreview} = $self->fetch_content('ljreview.html', $key); + $reviews{pwreview} = $self->fetch_content('pwreview.html', $key); + $reviews{slreview} = $self->fetch_content('slreview.html', $key); + $reviews{chreview} = $self->fetch_content('chreview.html', $key); + $reviews{blreview} = $self->fetch_content('blreview.html', $key); + $reviews{hbreview} = $self->fetch_content('hbreview.html', $key); + $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key); + + for(keys %reviews) { + if( ! $self->data_exists($reviews{$_}) ) { + delete $reviews{$_}; + next; + } + $reviews{$_} =~ s///og; # Strip any doctype declarations + } + + return 0 if scalar(keys %reviews) == 0; + + #my $html = "
"; + my $html; + $html .= $reviews{$_} for keys %reviews; + #$html .= "
"; + + return $self->send_html($html); } # we have to aggregate the reviews sub reviews_xml { - my( $self, $key ) = @_; - my %reviews; - - $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key); - $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key); - $reviews{slreview} = $self->fetch_content('slreview.xml', $key); - $reviews{chreview} = $self->fetch_content('chreview.xml', $key); - $reviews{blreview} = $self->fetch_content('blreview.xml', $key); - $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key); - $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key); - - for(keys %reviews) { - if( ! $self->data_exists($reviews{$_}) ) { - delete $reviews{$_}; - next; - } - # Strip the xml and doctype declarations - $reviews{$_} =~ s/<\?xml.*?>//og; - $reviews{$_} =~ s///og; - } - - return 0 if scalar(keys %reviews) == 0; - - my $xml = ""; - $xml .= $reviews{$_} for keys %reviews; - $xml .= ""; - - return $self->send_xml($xml); + my( $self, $key ) = @_; + my %reviews; + + $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key); + $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key); + $reviews{slreview} = $self->fetch_content('slreview.xml', $key); + $reviews{chreview} = $self->fetch_content('chreview.xml', $key); + $reviews{blreview} = $self->fetch_content('blreview.xml', $key); + $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key); + $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key); + + for(keys %reviews) { + if( ! $self->data_exists($reviews{$_}) ) { + delete $reviews{$_}; + next; + } + # Strip the xml and doctype declarations + $reviews{$_} =~ s/<\?xml.*?>//og; + $reviews{$_} =~ s///og; + } + + return 0 if scalar(keys %reviews) == 0; + + my $xml = ""; + $xml .= $reviews{$_} for keys %reviews; + $xml .= ""; + + return $self->send_xml($xml); } sub reviews_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('dbchapter.xml', $key)); } # -------------------------------------------------------------------------- sub data_exists { - my( $self, $data ) = @_; - return 0 if $data =~ m/error<\/title>/iog; - return 1; + my( $self, $data ) = @_; + return 0 if $data =~ m/<title>error<\/title>/iog; + return 1; } sub send_json { - my( $self, $xml ) = @_; - return 0 unless $self->data_exists($xml); - my $doc; - - try { - $doc = XML::LibXML->new->parse_string($xml); - } catch Error with { - my $err = shift; - $logger->error("added content XML parser error: $err\n\n$xml"); - $doc = undef; - }; - - return 0 unless $doc; - my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement); - my $json = OpenSRF::Utils::JSON->perl2JSON($perl); - print "Content-type: text/plain\n\n"; - print $json; - return 1; + my( $self, $xml ) = @_; + return 0 unless $self->data_exists($xml); + my $doc; + + try { + $doc = XML::LibXML->new->parse_string($xml); + } catch Error with { + my $err = shift; + $logger->error("added content XML parser error: $err\n\n$xml"); + $doc = undef; + }; + + return 0 unless $doc; + my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement); + my $json = OpenSRF::Utils::JSON->perl2JSON($perl); + return { content_type => 'text/plain', content => $json }; } sub send_xml { - my( $self, $xml ) = @_; - return 0 unless $self->data_exists($xml); - print "Content-Type: application/xml\n\n"; - print $xml; - return 1; + my( $self, $xml ) = @_; + return 0 unless $self->data_exists($xml); + return { content_type => 'application/xml', content => $xml }; } sub send_html { - my( $self, $content ) = @_; - return 0 unless $self->data_exists($content); - - # Hide anything that might contain a link since it will be broken - my $HTML = <<" HTML"; - <div> - <style type='text/css'> - div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden } - </style> - <div class='ac'> - $content - </div> - </div> - HTML - - print "Content-type: text/html\n\n"; - print $HTML; + my( $self, $content ) = @_; + return 0 unless $self->data_exists($content); + + # Hide anything that might contain a link since it will be broken + my $HTML = <<" HTML"; + <div> + <style type='text/css'> + div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden } + </style> + <div class='ac'> + $content + </div> + </div> + HTML + + return { content_type => 'text/html', content => $HTML }; +} - return 1; +sub send_img { + my($self, $response) = @_; + return { + content_type => $response->header('Content-type'), + content => $response->content, + binary => 1 + }; } +# returns the raw content returned from the URL fetch sub fetch_content { - my( $self, $page, $key ) = @_; - my $uname = $self->userid; - my $url = $self->base_url . "?isbn=$key/$page&client=$uname&type=rw12"; - return OpenILS::WWW::AddedContent->get_url($url); + my( $self, $page, $key ) = @_; + return $self->fetch_response($page, $key)->content; +} + +# returns the HTTP response object from the URL fetch +sub fetch_response { + my( $self, $page, $key, $notype ) = @_; + my $uname = $self->userid; + my $url = $self->base_url . "?isbn=$key/$page&client=$uname" . (($notype) ? '' : "&type=rw12"); + return $AC->get_url($url); } -- 2.11.0