Add pg log analyzer, achive.sh, pgbadger to logger and setup crontab

diff --git a/GenaSYS.sh b/GenaSYS.sh
index a5026fa..687a8dd 100755 (executable)
--- a/GenaSYS.sh
+++ b/GenaSYS.sh
@@ -2632,6 +2632,11 @@ echo "Setting up logger"
 for LOGGER in $MACHINES_LOGGER
 do
        mkdir -p "$OUTDIR/$LOGGER/etc/rsyslog.d"
+       mkdir -p "$OUTDIR/$LOGGER/root/eg
+       cp -f "$TEMPLATEDIR/rsyslog/pg-log-analyzer-cron.sh" "$OUTDIR/$LOGGER/root/eg"
+       cp -f "$TEMPLATEDIR/rsyslog/pg-log-analyzer.sh" "$OUTDIR/$LOGGER/root/eg"
+       cp -f "$TEMPLATEDIR/rsyslog/archive.sh" "$OUTDIR/$LOGGER/root/eg"
+       cp -f "$TEMPLATEDIR/rsyslog/logger-crontab "$OUTDIR/$LOGGER/root/eg"
        cp -f "$TEMPLATEDIR/rsyslog/opensrf-rsyslog.conf" "$OUTDIR/$LOGGER/etc/rsyslog.d"
        cp -f "$TEMPLATEDIR/rsyslog/rsyslog.conf-logger" "$OUTDIR/$LOGGER/etc/rsyslog.conf"
        if [ "$OSDISTRO" = "Debian6" ]

diff --git a/templates/apt/logger01_apt.list b/templates/apt/logger01_apt.list
index 4423ed5..48fdcc8 100644 (file)
--- a/templates/apt/logger01_apt.list
+++ b/templates/apt/logger01_apt.list
@@ -1 +1 @@
-ntp gdebi-core pigz nagios-nrpe-server sysstat munin-node munin-plugins-extra libwww-perl
+ntp gdebi-core pigz nagios-nrpe-server sysstat munin-node munin-plugins-extra libwww-perl pgbadger

diff --git a/templates/rsyslog/logger-crontab b/templates/rsyslog/logger-crontab
new file mode 100644 (file)
index 0000000..e4a8e8f
--- /dev/null
+++ b/templates/rsyslog/logger-crontab
@@ -0,0 +1,6 @@
+# m h  dom mon dow   command
+30 0 * * * /root/eg/archive.sh 1
+# remove logs older than 180 days old
+0 0 * * * find /var/log/evergreen -not -name activity.* -not -name ap_access.* -type f -ctime +180 -exec rm {} \;
+# Postgres log report
+0 * * * * /root/eg/pg-log-analyzer-cron

diff --git a/templates/rsyslog/pg-log-analyzer b/templates/rsyslog/pg-log-analyzer
new file mode 100755 (executable)
index 0000000..fd10fdc
--- /dev/null
+++ b/templates/rsyslog/pg-log-analyzer
@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# This program runs pgfouine or pgbadger to process postresql logs
+# and outputs the reports into a pre-defined directory.
+# It will check to previous our from the time the program
+# is executed in order to get all transactions for the hour.
+
+
+########### Config Section ###########
+
+## The directory that the logs are stored in.
+LOGDIR="/var/log/evergreen"
+
+## The output directory.
+OUTDIR="/var/www/html/pg_reports"
+
+## The output filename.
+OUTFILE="$(date +%F)-pg"
+
+## The logfile for this script
+ANALYZER_LOG="/var/log/pg-log-analyzer.log"
+
+## Number of concurrent processes (pgbadger only)
+CONCURRENT_JOBS="12"
+
+######################################
+
+
+## The date/time this script is run.
+RUN_DATE="$(date)"
+
+## The absolute path to the current day's logs file
+LOGPATH="$LOGDIR/$(date +%Y/%m/%d)"
+
+## The hour of time the program is run  
+CURRENT_HOUR="$(date +%H)"
+
+## This subracts one hour from the current hour
+HOUR="$(( $(date +%k) - 1 ))"
+## Then we pad the $HOUR with a '0'
+HOUR="$(printf "%02d\n" $HOUR)"
+
+## The current day's log files
+LOGDAY="$(date +%Y/%m/%d)"
+
+
+## Since we check 1 hour behind
+## we make sure to set the day yesterday
+## when our current runtime is 12:00 AM
+if [ "$CURRENT_HOUR" = "00" ]
+then
+    HOUR="23"
+    LOGDAY="$(date -d '1 day ago' +%Y/%m/%d)"
+    OUTFILE="$(date -d '1 day ago' +%F)-pg"
+fi
+
+
+
+## This does the actual work.
+case $1 in
+pgfouine)
+    echo -n "$RUN_DATE " >> "$ANALYZER_LOG"
+        pgfouine -file $LOGDIR/$LOGDAY/pg.$HOUR.log -logtype stderr > $OUTDIR/$OUTFILE.$HOUR.log.html 2>> "$ANALYZER_LOG"
+;;
+pgbadger)
+    echo -n "$RUN_DATE " >> "$ANALYZER_LOG"
+        pgbadger --format syslog --jobs $CONCURRENT_JOBS --outfile $OUTDIR/$OUTFILE.$HOUR.log.html $LOGDIR/$LOGDAY/pg.db01.$HOUR.log 2>> "$ANALYZER_LOG"
+;;
+*)
+        echo
+        echo usage:-
+        echo "<$0 pgfouine> to use pgfouine -or-"
+        echo "<$0 pgbadger> to use pgbadger"
+        exit 1
+esac

diff --git a/templates/rsyslog/pg-log-analyzer-cron b/templates/rsyslog/pg-log-analyzer-cron
new file mode 100755 (executable)
index 0000000..284e940
--- /dev/null
+++ b/templates/rsyslog/pg-log-analyzer-cron
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+### This will allow pg-log-analyzer to be run from cron - aw
+
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+/root/eg/pg-log-analyzer pgbadger

diff --git a/templates/setup.sh b/templates/setup.sh
index 2ba4c1a..9b6c3a3 100755 (executable)
--- a/templates/setup.sh
+++ b/templates/setup.sh
@@ -304,6 +304,17 @@ else
        echo "alias osrfwarn=\"findlogs && tail -f osrfwarn.log | grep -v 'File does not exist' | grep -v 'config.hard_due_date'\"" >> /root/.bashrc
        echo "alias osrferror=\"findlogs && tail -f osrferror.log | grep -v 'File does not exist'\"" >> /root/.bashrc
        rm -f /etc/rsyslog.d/60-evergreen-ils.conf
+       if [ -e "/root/eg/logger-crontab" ]
+       then    
+               echo;echo "Installing new crontab"
+               sleep 2
+               crontab -u root /root/eg/logger-crontab
+       else
+               echo
+               echo -e $COL_RED "WARNING! /root/eg/logger-crontab is missing!!"
+               echo -e "NOT installing crontab" $COL_RESET
+               sleep 5
+       fi
 
 fi
 }