From: Josh Stompro Date: Wed, 27 Feb 2019 16:31:54 +0000 (-0600) Subject: LP#1728331 - Action Trigger Aggregator Date Parameter X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=0993383eddd041860bf0b1d1b24d775b1640b17b;p=Evergreen.git LP#1728331 - Action Trigger Aggregator Date Parameter - Added release notes. - Added note about script to support scripts doc. - Added script to makefile for confile processing and installation. - Added processed version of the script to .gitignore I tested out the script with our production data and it worked as described. Signed-off-by: Josh Stompro Signed-off-by: Galen Charlton --- diff --git a/.gitignore b/.gitignore index bbe69e5d31..3e1a995d0b 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ Open-ILS/src/python/build/ Open-ILS/src/python/Evergreen.egg-info/ Open-ILS/src/python/Makefile Open-ILS/src/python/Makefile.in +Open-ILS/src/support-scripts/action_trigger_aggregator.pl Open-ILS/src/support-scripts/action_trigger_runner.pl Open-ILS/src/support-scripts/authority_authority_linker.pl Open-ILS/src/support-scripts/authority_control_fields.pl diff --git a/Open-ILS/src/Makefile.am b/Open-ILS/src/Makefile.am index 7f7954e1ca..9a0d5a7549 100644 --- a/Open-ILS/src/Makefile.am +++ b/Open-ILS/src/Makefile.am @@ -130,6 +130,7 @@ gen_scripts = \ @srcdir@/extras/import/marc2bre.pl \ @srcdir@/extras/import/marc2sre.pl \ @srcdir@/extras/import/parallel_pg_loader.pl \ + $(supportscr)/action_trigger_aggregator.pl \ $(supportscr)/action_trigger_runner.pl \ $(supportscr)/authority_control_fields.pl \ $(supportscr)/authority_authority_linker.pl \ @@ -212,6 +213,10 @@ uninstall-hook: $(do_subst) @srcdir@/extras/import/parallel_pg_loader.pl.in > "$@" chmod 755 "$@" +$(supportscr)/action_trigger_aggregator.pl: Makefile $(supportscr)/action_trigger_aggregator.pl.in + $(do_subst) $(supportscr)/action_trigger_aggregator.pl.in > "$@" + chmod 755 "$@" + $(supportscr)/action_trigger_runner.pl: Makefile $(supportscr)/action_trigger_runner.pl.in $(do_subst) $(supportscr)/action_trigger_runner.pl.in > "$@" chmod 755 "$@" diff --git a/Open-ILS/src/support-scripts/action_trigger_aggregator.pl b/Open-ILS/src/support-scripts/action_trigger_aggregator.pl deleted file mode 100755 index 20e30a8050..0000000000 --- a/Open-ILS/src/support-scripts/action_trigger_aggregator.pl +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/perl -# --------------------------------------------------------------- -# Copyright (C) 2012 Equinox Software, Inc -# Author: Bill Erickson -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU General Public License -# as published by the Free Software Foundation; either version 2 -# of the License, or (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# --------------------------------------------------------------- -use strict; -use warnings; -use DateTime; -use Getopt::Long; -use OpenSRF::System; -use OpenSRF::AppSession; -use OpenILS::Utils::CStoreEditor; -use OpenILS::Utils::RemoteAccount; -use OpenILS::Utils::Fieldmapper; - -my $osrf_config = '/openils/conf/opensrf_core.xml'; -my $default_start_date = DateTime->now->strftime('%F'); -my $date = ''; -my $start_date = ''; -my $end_date = ''; -my $event_defs = ''; -my $granularity = ''; -my $output_file = ''; -my $local_dir = '/tmp'; # where to keep local copies of generated files -my $remote_acct = ''; -my $cleanup = 0; # cleanup generated files -my $verbose = 0; -my $help = 0; - -GetOptions( - 'osrf-config=s' => \$osrf_config, - 'date=s' => \$date, - 'start-date=s' => \$start_date, - 'end-date=s' => \$end_date, - 'event-defs=s' => \$event_defs, - 'granularity=s' => \$granularity, - 'output-file=s' => \$output_file, - 'remote-acct=s' => \$remote_acct, - 'local-dir=s' => \$local_dir, - 'cleanup' => \$cleanup, - 'verbose' => \$verbose, - 'help' => \$help -); - -sub help { - print <$local_file") or - die "unable to open out-file '$local_file' for writing: $!\n"; -binmode(OUTFILE, ":utf8"); - -print "Output will be written to $local_file\n" if $verbose; - -OpenSRF::System->bootstrap_client(config_file => $osrf_config); -Fieldmapper->import(IDL => - OpenSRF::Utils::SettingsClient->new->config_value("IDL")); -OpenILS::Utils::CStoreEditor::init(); -my $editor = OpenILS::Utils::CStoreEditor->new; - -# if granularity is set, append all event defs with the -# selected granularity to the set of event-defs to process. -if ($granularity) { - my $defs = $editor->search_action_trigger_event_definition( - {granularity => $granularity}, - {idlist => 1} - ); - - for my $id (@$defs) { - push(@event_defs, $id) - unless grep { $_ eq $id} @event_defs; - } -} - -print "Processing event-defs @event_defs\n" if $verbose; - -die "Can't use --date with --start-date or --end-date!" if ($date && ($start_date || $end_date) ); - -$start_date = $default_start_date unless $start_date; # This is basically what happened pre- --date - -my %date_filter; -$date_filter{run_time} = {'>=' => $start_date} if $start_date; - -if ($end_date) { - if ($date_filter{run_time}) { - # both filters requested, -and them together - $date_filter{'-and'} = [ - {run_time => delete $date_filter{run_time}}, - {run_time => {'<' => $end_date}} - ]; - } else { - $date_filter{run_time} = {'<' => $end_date}; - } -} - -if ($date) { - delete $date_filter{run_time}; # will always exist because of defaulting $start_date - $date_filter{run_time} = { "=" => {"transform" => "date", "value" => $date}}; -} - -# collect the event tempate output data -# use a real session here so we can stream results directly to the output file -my $ses = OpenSRF::AppSession->create('open-ils.cstore'); -my $req = $ses->request( - 'open-ils.cstore.json_query', { - select => {ateo => [ - # ateo's may be linked to multiple atev's; select distinct. - {column => 'id', transform => 'distinct'}, 'data']}, - from => {ateo => { atev => { - filter => {state => 'complete', %date_filter}, - join => {atevdef => {filter => { - id => \@event_defs, - active => 't' - }}} - }}} - } -); - -# use a large timeout since this is likely to be a hefty query -while (my $resp = $req->recv(timeout => 3600)) { - die $req->failed . "\n" if $req->failed; - my $content = $resp->content or next; - print OUTFILE $content->{data}; -} - -if ($remote_acct) { - # send the file to the remote account - - my $racct = $editor->retrieve_config_remote_account($remote_acct); - die "No such remote account $remote_acct" unless $racct; - - my $type; - my $host = $racct->host; - ($host =~ s/^(S?FTP)://i and $type = uc($1)) or - ($host =~ s/^(SSH|SCP)://i and $type = 'SCP'); - $host =~ s#//##; - - my $acct = OpenILS::Utils::RemoteAccount->new( - type => $type, - remote_host => $host, - account_object => $racct, - local_file => $local_file, - remote_file => $output_file - ); - - my $res = $acct->put; - - die "Unable to push to remote server [$remote_acct] : " . - $acct->error . "\n" unless $res; - - print "Pushed file to $res\n" if $verbose; -} - -if ($cleanup) { - unlink($local_file) or - die "Unable to clean up file '$local_file' : $!\n"; -} - - diff --git a/Open-ILS/src/support-scripts/action_trigger_aggregator.pl.in b/Open-ILS/src/support-scripts/action_trigger_aggregator.pl.in new file mode 100755 index 0000000000..7ec44bf244 --- /dev/null +++ b/Open-ILS/src/support-scripts/action_trigger_aggregator.pl.in @@ -0,0 +1,246 @@ +#!/usr/bin/perl +# --------------------------------------------------------------- +# Copyright (C) 2012 Equinox Software, Inc +# Author: Bill Erickson +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# --------------------------------------------------------------- +use strict; +use warnings; +use DateTime; +use Getopt::Long; +use OpenSRF::System; +use OpenSRF::AppSession; +use OpenILS::Utils::CStoreEditor; +use OpenILS::Utils::RemoteAccount; +use OpenILS::Utils::Fieldmapper; + +my $osrf_config = '@sysconfdir@/opensrf_core.xml'; +my $default_start_date = DateTime->now->strftime('%F'); +my $date = ''; +my $start_date = ''; +my $end_date = ''; +my $event_defs = ''; +my $granularity = ''; +my $output_file = ''; +my $local_dir = '/tmp'; # where to keep local copies of generated files +my $remote_acct = ''; +my $cleanup = 0; # cleanup generated files +my $verbose = 0; +my $help = 0; + +GetOptions( + 'osrf-config=s' => \$osrf_config, + 'date=s' => \$date, + 'start-date=s' => \$start_date, + 'end-date=s' => \$end_date, + 'event-defs=s' => \$event_defs, + 'granularity=s' => \$granularity, + 'output-file=s' => \$output_file, + 'remote-acct=s' => \$remote_acct, + 'local-dir=s' => \$local_dir, + 'cleanup' => \$cleanup, + 'verbose' => \$verbose, + 'help' => \$help +); + +sub help { + print <$local_file") or + die "unable to open out-file '$local_file' for writing: $!\n"; +binmode(OUTFILE, ":utf8"); + +print "Output will be written to $local_file\n" if $verbose; + +OpenSRF::System->bootstrap_client(config_file => $osrf_config); +Fieldmapper->import(IDL => + OpenSRF::Utils::SettingsClient->new->config_value("IDL")); +OpenILS::Utils::CStoreEditor::init(); +my $editor = OpenILS::Utils::CStoreEditor->new; + +# if granularity is set, append all event defs with the +# selected granularity to the set of event-defs to process. +if ($granularity) { + my $defs = $editor->search_action_trigger_event_definition( + {granularity => $granularity}, + {idlist => 1} + ); + + for my $id (@$defs) { + push(@event_defs, $id) + unless grep { $_ eq $id} @event_defs; + } +} + +print "Processing event-defs @event_defs\n" if $verbose; + +die "Can't use --date with --start-date or --end-date!" if ($date && ($start_date || $end_date) ); + +$start_date = $default_start_date unless $start_date; # This is basically what happened pre- --date + +my %date_filter; +$date_filter{run_time} = {'>=' => $start_date} if $start_date; + +if ($end_date) { + if ($date_filter{run_time}) { + # both filters requested, -and them together + $date_filter{'-and'} = [ + {run_time => delete $date_filter{run_time}}, + {run_time => {'<' => $end_date}} + ]; + } else { + $date_filter{run_time} = {'<' => $end_date}; + } +} + +if ($date) { + delete $date_filter{run_time}; # will always exist because of defaulting $start_date + $date_filter{run_time} = { "=" => {"transform" => "date", "value" => $date}}; +} + +# collect the event tempate output data +# use a real session here so we can stream results directly to the output file +my $ses = OpenSRF::AppSession->create('open-ils.cstore'); +my $req = $ses->request( + 'open-ils.cstore.json_query', { + select => {ateo => [ + # ateo's may be linked to multiple atev's; select distinct. + {column => 'id', transform => 'distinct'}, 'data']}, + from => {ateo => { atev => { + filter => {state => 'complete', %date_filter}, + join => {atevdef => {filter => { + id => \@event_defs, + active => 't' + }}} + }}} + } +); + +# use a large timeout since this is likely to be a hefty query +while (my $resp = $req->recv(timeout => 3600)) { + die $req->failed . "\n" if $req->failed; + my $content = $resp->content or next; + print OUTFILE $content->{data}; +} + +if ($remote_acct) { + # send the file to the remote account + + my $racct = $editor->retrieve_config_remote_account($remote_acct); + die "No such remote account $remote_acct" unless $racct; + + my $type; + my $host = $racct->host; + ($host =~ s/^(S?FTP)://i and $type = uc($1)) or + ($host =~ s/^(SSH|SCP)://i and $type = 'SCP'); + $host =~ s#//##; + + my $acct = OpenILS::Utils::RemoteAccount->new( + type => $type, + remote_host => $host, + account_object => $racct, + local_file => $local_file, + remote_file => $output_file + ); + + my $res = $acct->put; + + die "Unable to push to remote server [$remote_acct] : " . + $acct->error . "\n" unless $res; + + print "Pushed file to $res\n" if $verbose; +} + +if ($cleanup) { + unlink($local_file) or + die "Unable to clean up file '$local_file' : $!\n"; +} + + diff --git a/docs/RELEASE_NOTES_NEXT/Administration/action-trigger-aggregator-date.adoc b/docs/RELEASE_NOTES_NEXT/Administration/action-trigger-aggregator-date.adoc new file mode 100644 index 0000000000..439b6364a3 --- /dev/null +++ b/docs/RELEASE_NOTES_NEXT/Administration/action-trigger-aggregator-date.adoc @@ -0,0 +1,8 @@ +Ability to specify specific date in action_trigger_aggregator.pl +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +An option, `--date`, has been added to the `action_trigger_aggregator.pl` +support script that allows the user to specify a specific date to aggregate +event output for. This new argument cannot be used with either `--start-date` +or `--end-date`. This option was added to simplify pulling event output for a +single day. diff --git a/docs/development/support_scripts.adoc b/docs/development/support_scripts.adoc index 11281ac9bd..cb0e10d402 100644 --- a/docs/development/support_scripts.adoc +++ b/docs/development/support_scripts.adoc @@ -15,6 +15,10 @@ http://evergreen-ils.org/communicate/[Evergreen IRC channel or email lists]. Here is a summary of the most commonly used scripts. The script name links to more thorough documentation, if available. + * action_trigger_aggregator.pl + -- Groups together event output for already processed events. Useful for + creating files that contain data from a group of events. Such as a CSV + file with all the overdue data for one day. * <<_processing_action_triggers,action_trigger_runner.pl>> -- Useful for creating events for specified hooks and running pending events * authority_authority_linker.pl