From db9aa2893656e74e91b26afe9c19923c7d7a1486 Mon Sep 17 00:00:00 2001 From: erickson Date: Fri, 16 Nov 2007 21:16:10 +0000 Subject: [PATCH] Back-porting: added content caching layer added content error handling logic (X errors disables added content lookups for Y seconds) added image fetching logic to Syndetic plugin pulling images from added content url updated exmaple configs git-svn-id: svn://svn.open-ils.org/ILS/branches/rel_1_2@8076 dcc99617-32d9-48b4-a31d-7c20da2025e4 --- Open-ILS/examples/apache/eg_vhost.conf | 20 +- Open-ILS/examples/opensrf.xml.example | 35 ++- Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm | 188 ++++++++---- .../perlmods/OpenILS/WWW/AddedContent/Syndetic.pm | 314 +++++++++++---------- Open-ILS/web/opac/common/js/added_content.js | 6 +- 5 files changed, 348 insertions(+), 215 deletions(-) diff --git a/Open-ILS/examples/apache/eg_vhost.conf b/Open-ILS/examples/apache/eg_vhost.conf index 7b21a4a92a..0a428cbf26 100644 --- a/Open-ILS/examples/apache/eg_vhost.conf +++ b/Open-ILS/examples/apache/eg_vhost.conf @@ -15,18 +15,18 @@ RedirectMatch 301 ^/$ /opac/en-US/skin/default/xml/index.xml # ---------------------------------------------------------------------------------- OSRFGatewayConfig /openils/conf/opensrf_core.xml - # ---------------------------------------------------------------------------------- # Set up the book jackets URL -# XXX This pulls images from Amazon, don't use this in a production environment -# ---------------------------------------------------------------------------------- -RewriteEngine on -ProxyTimeout 2 -RewriteRule /opac/extras/jacket/small/(.*) \ - http://images.amazon.com/images/P/$1.01._SCMZZZZZZZ_.jpg [P,L] -RewriteRule /opac/extras/jacket/large/(.*) \ - http://images.amazon.com/images/P/$1.01._SCLZZZZZZZ_.jpg [P,L] - +# This is an example of how you can have bookjacket images via Apache redirect, +# if there is no full-fledged added content plugin for the site you want to +# fetch images from. +# ---------------------------------------------------------------------------------- +#RewriteEngine on +#ProxyTimeout 2 +#RewriteRule /opac/extras/ac/jacket/small/(.*) \ +# http://images.amazon.com/images/P/$1.01._SCMZZZZZZZ_.jpg [P,L] +#RewriteRule /opac/extras/ac/jacket/large/(.*) \ +# http://images.amazon.com/images/P/$1.01._SCLZZZZZZZ_.jpg [P,L] # ---------------------------------------------------------------------------------- diff --git a/Open-ILS/examples/opensrf.xml.example b/Open-ILS/examples/opensrf.xml.example index a916be9b8d..48bdaab846 100644 --- a/Open-ILS/examples/opensrf.xml.example +++ b/Open-ILS/examples/opensrf.xml.example @@ -133,17 +133,38 @@ Example opensrf config file for OpenILS - - - + OpenILS::WWW::AddedContent::Amazon + + http://images.amazon.com/images/P/ + + + 4 + + + 600 + + + 4 + + MY_USER_ID - MY_BASE_URL - 2 + + - + diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm index ad3798796e..c56407947d 100644 --- a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm +++ b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent.pm @@ -16,100 +16,184 @@ use OpenSRF::EX qw(:try); use OpenSRF::Utils::Cache; use OpenSRF::System; use OpenSRF::Utils::Logger qw/$logger/; -use XML::LibXML; + +use LWP::UserAgent; +use MIME::Base64; + +my $AC = __PACKAGE__; # set the bootstrap config when this module is loaded my $bs_config; -my $handler; sub import { - my $self = shift; - $bs_config = shift; + my $self = shift; + $bs_config = shift; } -my $net_timeout; -my $cache; -sub child_init { +my $handler; # added content handler class handle +my $cache; # memcache handle +my $net_timeout; # max seconds to wait for a response from the added content vendor +my $max_errors; # max consecutive lookup failures before added content is temporarily disabled +my $error_countdown; # current consecutive errors countdown - OpenSRF::System->bootstrap_client( config_file => $bs_config ); +# number of seconds to wait before next lookup +# is attempted after lookups have been disabled +my $error_retry_timeout; - my $sclient = OpenSRF::Utils::SettingsClient->new(); - my $ac_data = $sclient->config_value("added_content"); - return unless $ac_data; +sub child_init { + OpenSRF::System->bootstrap_client( config_file => $bs_config ); $cache = OpenSRF::Utils::Cache->new; - my $ac_handler = $ac_data->{module}; - $net_timeout = $ac_data->{timeout} || 3; - - return unless $ac_handler; + my $sclient = OpenSRF::Utils::SettingsClient->new(); + my $ac_data = $sclient->config_value("added_content"); + + return unless $ac_data; + my $ac_handler = $ac_data->{module}; + return unless $ac_handler; - $logger->debug("Attempting to load Added Content handler: $ac_handler"); + $net_timeout = $ac_data->{timeout} || 1; + $error_countdown = $max_errors = $ac_data->{max_errors} || 10; + $error_retry_timeout = $ac_data->{retry_timeout} || 600; - eval "use $ac_handler"; + $logger->debug("Attempting to load Added Content handler: $ac_handler"); - if($@) { - $logger->error("Unable to load Added Content handler [$ac_handler]: $@"); - return; - } + eval "use $ac_handler"; + + if($@) { + $logger->error("Unable to load Added Content handler [$ac_handler]: $@"); + return; + } - $handler = $ac_handler->new($ac_data); - $logger->debug("added content loaded handler: $handler"); + $handler = $ac_handler->new($ac_data); + $logger->debug("added content loaded handler: $handler"); } sub handler { - my $r = shift; - my $cgi = CGI->new; - my $path = $r->path_info; + my $r = shift; + my $cgi = CGI->new; + my $path = $r->path_info; + my $res; - child_init() unless $handler; # why isn't apache doing this for us? - return Apache2::Const::NOT_FOUND unless $handler; + my( undef, $type, $format, $key ) = split(/\//, $r->path_info); - # if this memcache key is set, added content lookups are disabled - if( $cache->get_cache('ac.no_lookup') ) { - $logger->info("added content lookup disabled"); - return Apache2::Const::NOT_FOUND; + child_init() unless $handler; + + return Apache2::Const::NOT_FOUND unless $handler and $type and $format and $key; + return $res if defined($res = $AC->serve_from_cache($type, $format, $key)); + return Apache2::Const::NOT_FOUND unless $AC->lookups_enabled; + + my $err; + my $data; + my $method = "${type}_${format}"; + + return Apache2::Const::NOT_FOUND unless $handler->can($method); + + try { + $data = $handler->$method($key); + } catch Error with { + $err = shift; + decr_error_countdown(); + $logger->error("added content handler failed: $method($key) => $err"); + }; + + return Apache2::Const::NOT_FOUND if $err; + + if(!$data) { + # if the AC lookup found no corresponding data, cache that information + $logger->debug("added content handler returned no results $method($key)") unless $data; + $AC->cache_result($type, $format, $key, {nocontent=>1}); + return Apache2::Const::NOT_FOUND; } + + $AC->print_content($data); + $AC->cache_result($type, $format, $key, $data); + + reset_error_countdown(); + return Apache2::Const::OK; +} +sub print_content { + my($class, $data, $from_cache) = @_; + return Apache2::Const::NOT_FOUND if $data->{nocontent}; - my( undef, $data, $format, $key ) = split(/\//, $r->path_info); + my $ct = $data->{content_type}; + my $content = $data->{content}; + print "Content-type: $ct\n\n"; - my $err; - my $success; - my $method = "${data}_${format}"; + if($data->{binary}) { + binmode STDOUT; + # if it hasn't been cached yet, it's still in binary form + print( ($from_cache) ? decode_base64($content) : $content ); + } else { + print $content; + } - try { - $success = $handler->$method($key); - } catch Error with { - my $err = shift; - $logger->error("added content handler failed: $method($key) => $err"); - }; - return Apache2::Const::NOT_FOUND if $err or !$success; - return Apache2::Const::OK; + return Apache2::Const::OK; } -# generic GET call + +# returns an HTPP::Response object sub get_url { - my( $self, $url ) = @_; - $logger->info("added content getting [timeout=$net_timeout] URL = $url"); - my $agent = LWP::UserAgent->new(timeout => $net_timeout); - my $res = $agent->get($url); - die "added content request failed: " . $res->status_line ."\n" unless $res->is_success; - return $res->content; + my( $self, $url ) = @_; + + $logger->info("added content getting [timeout=$net_timeout, errors_remaining=$error_countdown] URL = $url"); + my $agent = LWP::UserAgent->new(timeout => $net_timeout); + + my $res = $agent->get($url); + $logger->info("added content request returned with code " . $res->code); + die "added content request failed: " . $res->status_line ."\n" unless $res->is_success; + + return $res; } +sub lookups_enabled { + if( $cache->get_cache('ac.no_lookup') ) { + $logger->info("added content lookup disabled"); + return undef; + } + return 1; +} +sub disable_lookups { + $cache->put_cache('ac.no_lookup', 1, $error_retry_timeout); +} +sub decr_error_countdown { + $error_countdown--; + if($error_countdown < 1) { + $logger->warn("added content error count exhausted. Disabling lookups for $error_retry_timeout seconds"); + $AC->disable_lookups; + } +} +sub reset_error_countdown { + $error_countdown = $max_errors; +} +sub cache_result { + my($class, $type, $format, $key, $data) = @_; + $logger->debug("caching $type/$format/$key"); + $data->{content} = encode_base64($data->{content}) if $data->{binary}; + return $cache->put_cache("ac.$type.$format.$key", $data); +} + +sub serve_from_cache { + my($class, $type, $format, $key) = @_; + my $data = $cache->get_cache("ac.$type.$format.$key"); + return undef unless $data; + $logger->debug("serving $type/$format/$key from cache"); + return $class->print_content($data, 1); +} -1; + +1; diff --git a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm index 38f9bfd4df..6ae2a5ab64 100644 --- a/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm +++ b/Open-ILS/src/perlmods/OpenILS/WWW/AddedContent/Syndetic.pm @@ -1,230 +1,260 @@ package OpenILS::WWW::AddedContent::Syndetic; use strict; use warnings; -use LWP::UserAgent; use OpenSRF::Utils::Logger qw/$logger/; use OpenSRF::Utils::SettingsParser; use OpenSRF::Utils::JSON; use OpenSRF::EX qw/:try/; use OpenILS::WWW::AddedContent; +use XML::LibXML; +use MIME::Base64; +my $AC = 'OpenILS::WWW::AddedContent'; sub new { - my( $class, $args ) = @_; - $class = ref $class || $class; - return bless($args, $class); + my( $class, $args ) = @_; + $class = ref $class || $class; + return bless($args, $class); } sub base_url { - my $self = shift; - return $self->{base_url}; + my $self = shift; + return $self->{base_url}; } sub userid { - my $self = shift; - return $self->{userid}; + my $self = shift; + return $self->{userid}; } # -------------------------------------------------------------------------- +sub jacket_small { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('sc.gif', $key, 1)); +} + +sub jacket_medium { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('mc.gif', $key, 1)); + +} +sub jacket_large { + my( $self, $key ) = @_; + return $self->send_img( + $self->fetch_response('lc.gif', $key, 1)); +} + +# -------------------------------------------------------------------------- sub toc_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('toc.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('toc.html', $key)); } sub toc_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('toc.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('toc.xml', $key)); } sub toc_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('toc.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('toc.xml', $key)); } # -------------------------------------------------------------------------- sub anotes_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('anotes.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('anotes.html', $key)); } sub anotes_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('anotes.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('anotes.xml', $key)); } sub anotes_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('anotes.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('anotes.xml', $key)); } # -------------------------------------------------------------------------- sub excerpt_html { - my( $self, $key ) = @_; - return $self->send_html( - $self->fetch_content('dbchapter.html', $key)); + my( $self, $key ) = @_; + return $self->send_html( + $self->fetch_content('dbchapter.html', $key)); } sub excerpt_xml { - my( $self, $key ) = @_; - return $self->send_xml( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_xml( + $self->fetch_content('dbchapter.xml', $key)); } sub excerpt_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('dbchapter.xml', $key)); } # -------------------------------------------------------------------------- sub reviews_html { - my( $self, $key ) = @_; - - my %reviews; - - $reviews{ljreview} = $self->fetch_content('ljreview.html', $key); - $reviews{pwreview} = $self->fetch_content('pwreview.html', $key); - $reviews{slreview} = $self->fetch_content('slreview.html', $key); - $reviews{chreview} = $self->fetch_content('chreview.html', $key); - $reviews{blreview} = $self->fetch_content('blreview.html', $key); - $reviews{hbreview} = $self->fetch_content('hbreview.html', $key); - $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key); - - for(keys %reviews) { - if( ! $self->data_exists($reviews{$_}) ) { - delete $reviews{$_}; - next; - } - $reviews{$_} =~ s///og; # Strip any doctype declarations - } - - return 0 if scalar(keys %reviews) == 0; - - #my $html = "
"; - my $html; - $html .= $reviews{$_} for keys %reviews; - #$html .= "
"; - - return $self->send_html($html); + my( $self, $key ) = @_; + + my %reviews; + + $reviews{ljreview} = $self->fetch_content('ljreview.html', $key); + $reviews{pwreview} = $self->fetch_content('pwreview.html', $key); + $reviews{slreview} = $self->fetch_content('slreview.html', $key); + $reviews{chreview} = $self->fetch_content('chreview.html', $key); + $reviews{blreview} = $self->fetch_content('blreview.html', $key); + $reviews{hbreview} = $self->fetch_content('hbreview.html', $key); + $reviews{kirkreview} = $self->fetch_content('kirkreview.html', $key); + + for(keys %reviews) { + if( ! $self->data_exists($reviews{$_}) ) { + delete $reviews{$_}; + next; + } + $reviews{$_} =~ s///og; # Strip any doctype declarations + } + + return 0 if scalar(keys %reviews) == 0; + + #my $html = "
"; + my $html; + $html .= $reviews{$_} for keys %reviews; + #$html .= "
"; + + return $self->send_html($html); } # we have to aggregate the reviews sub reviews_xml { - my( $self, $key ) = @_; - my %reviews; - - $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key); - $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key); - $reviews{slreview} = $self->fetch_content('slreview.xml', $key); - $reviews{chreview} = $self->fetch_content('chreview.xml', $key); - $reviews{blreview} = $self->fetch_content('blreview.xml', $key); - $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key); - $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key); - - for(keys %reviews) { - if( ! $self->data_exists($reviews{$_}) ) { - delete $reviews{$_}; - next; - } - # Strip the xml and doctype declarations - $reviews{$_} =~ s/<\?xml.*?>//og; - $reviews{$_} =~ s///og; - } - - return 0 if scalar(keys %reviews) == 0; - - my $xml = ""; - $xml .= $reviews{$_} for keys %reviews; - $xml .= ""; - - return $self->send_xml($xml); + my( $self, $key ) = @_; + my %reviews; + + $reviews{ljreview} = $self->fetch_content('ljreview.xml', $key); + $reviews{pwreview} = $self->fetch_content('pwreview.xml', $key); + $reviews{slreview} = $self->fetch_content('slreview.xml', $key); + $reviews{chreview} = $self->fetch_content('chreview.xml', $key); + $reviews{blreview} = $self->fetch_content('blreview.xml', $key); + $reviews{hbreview} = $self->fetch_content('hbreview.xml', $key); + $reviews{kirkreview} = $self->fetch_content('kirkreview.xml', $key); + + for(keys %reviews) { + if( ! $self->data_exists($reviews{$_}) ) { + delete $reviews{$_}; + next; + } + # Strip the xml and doctype declarations + $reviews{$_} =~ s/<\?xml.*?>//og; + $reviews{$_} =~ s///og; + } + + return 0 if scalar(keys %reviews) == 0; + + my $xml = ""; + $xml .= $reviews{$_} for keys %reviews; + $xml .= ""; + + return $self->send_xml($xml); } sub reviews_json { - my( $self, $key ) = @_; - return $self->send_json( - $self->fetch_content('dbchapter.xml', $key)); + my( $self, $key ) = @_; + return $self->send_json( + $self->fetch_content('dbchapter.xml', $key)); } # -------------------------------------------------------------------------- sub data_exists { - my( $self, $data ) = @_; - return 0 if $data =~ m/error<\/title>/iog; - return 1; + my( $self, $data ) = @_; + return 0 if $data =~ m/<title>error<\/title>/iog; + return 1; } sub send_json { - my( $self, $xml ) = @_; - return 0 unless $self->data_exists($xml); - my $doc; - - try { - $doc = XML::LibXML->new->parse_string($xml); - } catch Error with { - my $err = shift; - $logger->error("added content XML parser error: $err\n\n$xml"); - $doc = undef; - }; - - return 0 unless $doc; - my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement); - my $json = OpenSRF::Utils::JSON->perl2JSON($perl); - print "Content-type: text/plain\n\n"; - print $json; - return 1; + my( $self, $xml ) = @_; + return 0 unless $self->data_exists($xml); + my $doc; + + try { + $doc = XML::LibXML->new->parse_string($xml); + } catch Error with { + my $err = shift; + $logger->error("added content XML parser error: $err\n\n$xml"); + $doc = undef; + }; + + return 0 unless $doc; + my $perl = OpenSRF::Utils::SettingsParser::XML2perl($doc->documentElement); + my $json = OpenSRF::Utils::JSON->perl2JSON($perl); + return { content_type => 'text/plain', content => $json }; } sub send_xml { - my( $self, $xml ) = @_; - return 0 unless $self->data_exists($xml); - print "Content-Type: application/xml\n\n"; - print $xml; - return 1; + my( $self, $xml ) = @_; + return 0 unless $self->data_exists($xml); + return { content_type => 'application/xml', content => $xml }; } sub send_html { - my( $self, $content ) = @_; - return 0 unless $self->data_exists($content); - - # Hide anything that might contain a link since it will be broken - my $HTML = <<" HTML"; - <div> - <style type='text/css'> - div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden } - </style> - <div class='ac'> - $content - </div> - </div> - HTML - - print "Content-type: text/html\n\n"; - print $HTML; + my( $self, $content ) = @_; + return 0 unless $self->data_exists($content); + + # Hide anything that might contain a link since it will be broken + my $HTML = <<" HTML"; + <div> + <style type='text/css'> + div.ac input, div.ac a[href],div.ac img, div.ac button { display: none; visibility: hidden } + </style> + <div class='ac'> + $content + </div> + </div> + HTML + + return { content_type => 'text/html', content => $HTML }; +} - return 1; +sub send_img { + my($self, $response) = @_; + return { + content_type => $response->header('Content-type'), + content => $response->content, + binary => 1 + }; } +# returns the raw content returned from the URL fetch sub fetch_content { - my( $self, $page, $key ) = @_; - my $uname = $self->userid; - my $url = $self->base_url . "?isbn=$key/$page&client=$uname&type=rw12"; - return OpenILS::WWW::AddedContent->get_url($url); + my( $self, $page, $key ) = @_; + return $self->fetch_response($page, $key)->content; +} + +# returns the HTTP response object from the URL fetch +sub fetch_response { + my( $self, $page, $key, $notype ) = @_; + my $uname = $self->userid; + my $url = $self->base_url . "?isbn=$key/$page&client=$uname" . (($notype) ? '' : "&type=rw12"); + return $AC->get_url($url); } diff --git a/Open-ILS/web/opac/common/js/added_content.js b/Open-ILS/web/opac/common/js/added_content.js index f7be084239..850858d87b 100644 --- a/Open-ILS/web/opac/common/js/added_content.js +++ b/Open-ILS/web/opac/common/js/added_content.js @@ -1,18 +1,16 @@ /** * This function should return a URL which points to the book cover image based on ISBN. -* Ideally, this should point to some type of added content service. -* The example below uses Amazon... *use at own risk* */ + function buildISBNSrc(isbn, size) { size = (size) ? size : 'small'; if(OILS_OPAC_IMAGES_HOST) return location.protocol + '//' + OILS_OPAC_IMAGES_HOST + size + '/' + isbn; - return '../../../../extras/jacket/'+size+'/'+isbn; + return '../../../../extras/ac/jacket/'+size+'/'+isbn; } - function acMakeURL(type, key) { return '../../../../extras/ac/' + type + '/html/' + key; } -- 2.11.0