--- /dev/null
+#!/usr/bin/perl
+
+# (C) Copyright 2019-2021 Georgia Public Library Service
+# Chris Sharp <csharp@georgialibraries.org>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# This script is for importing student data from a CSV
+# exported from a student information system (SIS).
+#
+use warnings;
+use strict;
+use File::Rsync;
+use Text::CSV qw/ csv /;
+use OpenILS::Utils::Cronscript;
+use OpenILS::Application::Actor;
+
+my $cscript = OpenILS::Utils::Cronscript->new();
+my $editor = $cscript->editor(xact=>1);
+
+# 0 = no stdout messages
+# 1 = basic
+# 2 = detailed
+my $debug = 1;
+# the script can be run manually outside of cron, just
+# append <filename> <school code> to the script invocation
+my $prefix = $ARGV[1] if $ARGV[1];
+my @retrieved_files = $ARGV[0] if $ARGV[0];
+# TODO: develop a reliable query for getting the
+# student card - or use YAOUS or something to set
+# the profile.
+my $profile = 61; # Student Card
+my $district_id = 0;
+my $error_message;
+my $imports;
+my $updates;
+my $deletes;
+my $opt_out;
+my @exceptions;
+
+
+sub calculate_expire_date {
+ my @now = localtime();
+ my ($now_year, $now_month) = ($now[5] + 1900, $now[4] + 1);
+ my $expire_year = $now_year;
+ if ($now_month > 6) { # any student registered before end of June gets expire date the following 9/15
+ $expire_year += 1;
+ }
+ my $expire_date = $expire_year . "-10-15";
+ return $expire_date;
+}
+
+# TODO: we need to alter this to accommodate format-agnostic dates
+sub calculate_password_from_dob {
+ my $dob = shift;
+ my @elements = split('-', $dob);
+ my ($dob_month, $dob_year) = ($elements[1], $elements[0]);
+ my $password = $dob_month . $dob_year;
+ return $password;
+}
+
+
+if (@retrieved_files) {
+ # we're in single-file mode, so get the district ID for later use
+ # TODO: get the district ID via cstore
+} else {
+ # go out and get the files we need
+ my @files;
+ if ($debug) {
+ print "Now processing files for " . $district->{code} . "\n";
+ }
+
+ ## walk through the remote files and filter out any that do
+ ## not meet our naming requirements
+ foreach my $file (@files) {
+ # XXX hard-coding the "OptOut" string here - maybe make it a setting?
+ if ($file =~ /${prefix}(OptOut)?_\d{12}/) {
+ # TODO: add OpenSRF-y way to retrieve previous files
+ my $previous = ;
+ # skip files we have already retreived
+ unless (grep /$file/, @$previous) {
+ # get the file from the local FTP server
+ } else {
+ if ($debug) {
+ print "$file already retreived\n";
+ }
+ }
+ }
+ }
+ }
+}
+
+
+foreach my $csv_file (@retrieved_files) {
+ $imports = 0;
+ $updates = 0;
+ $deletes = 0;
+ $opt_out = 1 if $csv_file =~ /OptOut/;
+ # a couple of options here - we can assume that the headers are correct
+ # and use headers => "auto" here, which will fail to work if there's a
+ # typo in the header names, or we can do headers => "skip" and trust
+ # that the export follows our prescribed order.
+ my $student_entries = csv (in => "$csv_file", headers => "auto", empty_is_undef => 1)
+ or die Text::CSV->error_diag;
+ foreach my $student (@$student_entries) {
+ # set up variables
+ # TODO: make these assignment use the hash we just created with col names
+ my $school_id = @$student[0];
+ my $student_id = @$student[1];
+ my $first_name = @$student[2];
+ my $middle_name = @$student[3];
+ my $last_name = @$student[4];
+ my $dob = @$student[5];
+ my $phone = @$student[6];
+ my $email = @$student[7];
+ my $street1 = @$student[8];
+ my $street2 = @$student[9];
+ my $city = @$student[10];
+ my $county = @$student[11];
+ my $state = @$student[12];
+ my $post_code = @$student[13];
+ my $parent_guardian = @$student[14];
+ my $grade = @$student[15];
+ my $barcode = $prefix . $student_id;
+ my $username = $barcode;
+ my $password = calculate_password_from_dob($dob);
+ my $ident_type = 3; # Other
+ my $ident_value = $student_id;
+ my $expire_date = calculate_expire_date;
+ my $juvenile = "true";
+ my $name_keywords;
+ if ($grade && $grade =~ /^\d+$/) {
+ if ($grade == 0) {
+ $name_keywords = "GradeK";
+ } elsif ($grade < 0) {
+ $name_keywords = "GradePK";
+ } elsif ($grade < 10) {
+ $name_keywords = "Grade0" . $grade;
+ } else {
+ $name_keywords = "Grade" . $grade;
+ }
+ } elsif ($grade) {
+ $name_keywords = $grade;
+ }
+ # figure out how to get the home ou from the DB based
+ # on the school ID
+ # TODO: get these values via OpenSRF
+ my $home_branch = $school_result[0];
+ my $school_name = $school_result[1];
+ my $school_county = $school_result[2];
+ my $ident2_value = $school_name;
+ # make sure we have a usable county entry
+ # if not, add from the school data
+ if (!$county || $county =~ m/\d/) {
+ $county = $school_county;
+ }
+ # default to school for parent/guardian
+ if (!$parent_guardian) {
+ $parent_guardian = $school_name
+ }
+ # check that we have the required data for the student
+ my $no_import = 0;
+ unless (
+ $school_id &&
+ $student_id &&
+ $first_name &&
+ $last_name &&
+ $dob &&
+ $street1 &&
+ $city &&
+ $state &&
+ $post_code &&
+ $home_branch ) {
+ if ($debug == 2) {
+ print "Student $student_id is an exception.\n";
+ }
+ push @exceptions, $student;
+ $no_import = 1;
+ }
+ unless ($no_import) {
+ # check if we already have the student account
+ # TODO: retrieve these with OpenSRF
+ my $sth_check_barcode = $dbh->prepare($check_barcode_sql);
+ $sth_check_barcode->execute($barcode);
+ my $already_imported = $sth_check_barcode->fetchrow_array;
+ $sth_check_barcode->finish;
+ my $sth_user_id = $dbh->prepare($get_user_id_sql);
+ $sth_user_id->execute($barcode);
+ my @user_result = $sth_user_id->fetchrow_array;
+ $sth_user_id->finish;
+ my $user_id = $user_result[0];
+
+ if ($already_imported && !$opt_out) {
+ # assume that we're doing an update
+ my $sth_addr = $dbh->prepare($get_addr_sql);
+ $sth_addr->execute($user_id);
+ my @addr_result = $sth_addr->fetchrow_array;
+ $sth_addr->finish;
+ my $addr_id = $addr_result[0];
+ my @update_user_data = (
+ $first_name,
+ $middle_name,
+ $last_name,
+ $dob,
+ $phone,
+ $parent_guardian,
+ $name_keywords,
+ $ident_type,
+ $ident_value,
+ $ident2_value,
+ $home_branch,
+ $expire_date,
+ $user_id
+ );
+ my @update_addr_data = (
+ $street1,
+ $street2,
+ $city,
+ $county,
+ $state,
+ $post_code,
+ $addr_id
+ );
+ # do the update
+ if ($debug) {
+ print "Updating $barcode\n";
+ }
+ # TODO: see if we still need the try/catch stuff
+ try {
+ # update user
+ if ($debug == 2) {
+ print "Update user data: @update_user_data\n";
+ }
+ my $sth_update_user = $dbh->prepare($update_user_sql);
+ $sth_update_user->execute(@update_user_data);
+ # update address
+ if ($debug == 2) {
+ print "Update address data: @update_addr_data\n";
+ }
+ my $sth_update_addr = $dbh->prepare($update_address_sql);
+ $sth_update_addr->execute(@update_addr_data);
+ $dbh->commit;
+ $sth_update_user->finish;
+ $sth_update_addr->finish;
+ $updates++;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+
+ } elsif ($already_imported && $opt_out ) {
+ if ($debug) {
+ print "Deleting $barcode\n";
+ }
+ try {
+ # delete user
+ my $sth_delete_user = $dbh->prepare($delete_user_sql);
+ $sth_delete_user->execute($user_id);
+ $dbh->commit;
+ $sth_delete_user->finish;
+ $deletes++
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ } elsif ($opt_out) {
+ # do nothing
+ next;
+ } else {
+ # import user
+ my @user_data = (
+ $profile,
+ $username,
+ $email,
+ $password,
+ $ident_type,
+ $ident_value,
+ $ident_type,
+ $ident2_value,
+ $first_name,
+ $middle_name,
+ $last_name,
+ $phone,
+ $home_branch,
+ $dob,
+ $expire_date,
+ $juvenile,
+ $parent_guardian,
+ $name_keywords
+ );
+ my @addr_data = (
+ $username,
+ $street1,
+ $street2,
+ $city,
+ $county,
+ $state,
+ $post_code,
+ );
+ my @card_data = (
+ $username,
+ $barcode
+ );
+ my @link_card_data = (
+ $barcode,
+ $username
+ );
+ my @link_addr_data = (
+ $username,
+ $username
+ );
+
+ # do the import here
+ if ($debug) {
+ print "Importing $barcode\n";
+ }
+ try {
+ # insert user
+ if ($debug == 2) {
+ print "Inserting user data: @user_data\n";
+ }
+ my $sth_insert_user = $dbh->prepare($insert_user_sql);
+ $sth_insert_user->execute(@user_data);
+ # insert address
+ if ($debug == 2) {
+ print "Inserting address data: @addr_data\n";
+ }
+ my $sth_insert_addr = $dbh->prepare($insert_address_sql);
+ $sth_insert_addr->execute(@addr_data);
+ # link address back to the user
+ my $sth_link_addr = $dbh->prepare($link_addr_sql);
+ $sth_link_addr->execute(@link_addr_data);
+ # insert card
+ if ($debug == 2) {
+ print "Inserting card data: @card_data\n";
+ }
+ my $sth_insert_card = $dbh->prepare($insert_card_sql);
+ $sth_insert_card->execute(@card_data);
+ # link card back to the user
+ my $sth_link_card = $dbh->prepare($link_card_sql);
+ $sth_link_card->execute(@link_card_data);
+ $dbh->commit;
+ $sth_insert_user->finish;
+ $sth_insert_addr->finish;
+ $sth_link_addr->finish;
+ $sth_insert_card->finish;
+ $sth_link_card->finish;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ $imports++;
+ }
+
+ }
+ }
+
+ # record the import in the DB
+ my @file_path = split('/', $csv_file);
+ my $filename = $file_path[-1]; # bare filename
+ try {
+ my $sth_insert_import = $dbh->prepare($insert_import_sql);
+ #TODO: error handling that gets pushed to the DB
+ $sth_insert_import->execute($district_id, $filename, $error_message);
+ $dbh->commit;
+ $sth_insert_import->finish;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ # write exceptions to a file
+ my $exceptions_file = "/tmp/$filename.exceptions";
+ my $exceptions_ref = \@exceptions;
+ csv (in => $exceptions_ref, out => "$exceptions_file")
+ or die Text::CSV->error_diag;
+ print "\n\nImport statistics:\n\n\tImports: $imports\n\tUpdates: $updates\n\tOpt-Outs: $deletes\n\tExceptions: " . scalar(@exceptions) . "\n\n\tCreated exceptions file $exceptions_file.\n";
+ shift @retrieved_files;
+}
+
projects / evergreen / pines.git / commitdiff