From: Chris Sharp Date: Tue, 6 Jul 2021 14:43:21 +0000 (-0400) Subject: add support script for importing student cards X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=d9b32844ffb158d8ff1fa4a974591489d58b6ddc;p=evergreen%2Fpines.git add support script for importing student cards Signed-off-by: Chris Sharp --- diff --git a/Open-ILS/src/support-scripts/import_student_data.pl b/Open-ILS/src/support-scripts/import_student_data.pl new file mode 100755 index 0000000000..0d848d7344 --- /dev/null +++ b/Open-ILS/src/support-scripts/import_student_data.pl @@ -0,0 +1,390 @@ +#!/usr/bin/perl + +# (C) Copyright 2019-2021 Georgia Public Library Service +# Chris Sharp +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# This script is for importing student data from a CSV +# exported from a student information system (SIS). +# +use warnings; +use strict; +use File::Rsync; +use Text::CSV qw/ csv /; +use OpenILS::Utils::Cronscript; +use OpenILS::Application::Actor; + +my $cscript = OpenILS::Utils::Cronscript->new(); +my $editor = $cscript->editor(xact=>1); + +# 0 = no stdout messages +# 1 = basic +# 2 = detailed +my $debug = 1; +# the script can be run manually outside of cron, just +# append to the script invocation +my $prefix = $ARGV[1] if $ARGV[1]; +my @retrieved_files = $ARGV[0] if $ARGV[0]; +# TODO: develop a reliable query for getting the +# student card - or use YAOUS or something to set +# the profile. +my $profile = 61; # Student Card +my $district_id = 0; +my $error_message; +my $imports; +my $updates; +my $deletes; +my $opt_out; +my @exceptions; + + +sub calculate_expire_date { + my @now = localtime(); + my ($now_year, $now_month) = ($now[5] + 1900, $now[4] + 1); + my $expire_year = $now_year; + if ($now_month > 6) { # any student registered before end of June gets expire date the following 9/15 + $expire_year += 1; + } + my $expire_date = $expire_year . "-10-15"; + return $expire_date; +} + +# TODO: we need to alter this to accommodate format-agnostic dates +sub calculate_password_from_dob { + my $dob = shift; + my @elements = split('-', $dob); + my ($dob_month, $dob_year) = ($elements[1], $elements[0]); + my $password = $dob_month . $dob_year; + return $password; +} + + +if (@retrieved_files) { + # we're in single-file mode, so get the district ID for later use + # TODO: get the district ID via cstore +} else { + # go out and get the files we need + my @files; + if ($debug) { + print "Now processing files for " . $district->{code} . "\n"; + } + + ## walk through the remote files and filter out any that do + ## not meet our naming requirements + foreach my $file (@files) { + # XXX hard-coding the "OptOut" string here - maybe make it a setting? + if ($file =~ /${prefix}(OptOut)?_\d{12}/) { + # TODO: add OpenSRF-y way to retrieve previous files + my $previous = ; + # skip files we have already retreived + unless (grep /$file/, @$previous) { + # get the file from the local FTP server + } else { + if ($debug) { + print "$file already retreived\n"; + } + } + } + } + } +} + + +foreach my $csv_file (@retrieved_files) { + $imports = 0; + $updates = 0; + $deletes = 0; + $opt_out = 1 if $csv_file =~ /OptOut/; + # a couple of options here - we can assume that the headers are correct + # and use headers => "auto" here, which will fail to work if there's a + # typo in the header names, or we can do headers => "skip" and trust + # that the export follows our prescribed order. + my $student_entries = csv (in => "$csv_file", headers => "auto", empty_is_undef => 1) + or die Text::CSV->error_diag; + foreach my $student (@$student_entries) { + # set up variables + # TODO: make these assignment use the hash we just created with col names + my $school_id = @$student[0]; + my $student_id = @$student[1]; + my $first_name = @$student[2]; + my $middle_name = @$student[3]; + my $last_name = @$student[4]; + my $dob = @$student[5]; + my $phone = @$student[6]; + my $email = @$student[7]; + my $street1 = @$student[8]; + my $street2 = @$student[9]; + my $city = @$student[10]; + my $county = @$student[11]; + my $state = @$student[12]; + my $post_code = @$student[13]; + my $parent_guardian = @$student[14]; + my $grade = @$student[15]; + my $barcode = $prefix . $student_id; + my $username = $barcode; + my $password = calculate_password_from_dob($dob); + my $ident_type = 3; # Other + my $ident_value = $student_id; + my $expire_date = calculate_expire_date; + my $juvenile = "true"; + my $name_keywords; + if ($grade && $grade =~ /^\d+$/) { + if ($grade == 0) { + $name_keywords = "GradeK"; + } elsif ($grade < 0) { + $name_keywords = "GradePK"; + } elsif ($grade < 10) { + $name_keywords = "Grade0" . $grade; + } else { + $name_keywords = "Grade" . $grade; + } + } elsif ($grade) { + $name_keywords = $grade; + } + # figure out how to get the home ou from the DB based + # on the school ID + # TODO: get these values via OpenSRF + my $home_branch = $school_result[0]; + my $school_name = $school_result[1]; + my $school_county = $school_result[2]; + my $ident2_value = $school_name; + # make sure we have a usable county entry + # if not, add from the school data + if (!$county || $county =~ m/\d/) { + $county = $school_county; + } + # default to school for parent/guardian + if (!$parent_guardian) { + $parent_guardian = $school_name + } + # check that we have the required data for the student + my $no_import = 0; + unless ( + $school_id && + $student_id && + $first_name && + $last_name && + $dob && + $street1 && + $city && + $state && + $post_code && + $home_branch ) { + if ($debug == 2) { + print "Student $student_id is an exception.\n"; + } + push @exceptions, $student; + $no_import = 1; + } + unless ($no_import) { + # check if we already have the student account + # TODO: retrieve these with OpenSRF + my $sth_check_barcode = $dbh->prepare($check_barcode_sql); + $sth_check_barcode->execute($barcode); + my $already_imported = $sth_check_barcode->fetchrow_array; + $sth_check_barcode->finish; + my $sth_user_id = $dbh->prepare($get_user_id_sql); + $sth_user_id->execute($barcode); + my @user_result = $sth_user_id->fetchrow_array; + $sth_user_id->finish; + my $user_id = $user_result[0]; + + if ($already_imported && !$opt_out) { + # assume that we're doing an update + my $sth_addr = $dbh->prepare($get_addr_sql); + $sth_addr->execute($user_id); + my @addr_result = $sth_addr->fetchrow_array; + $sth_addr->finish; + my $addr_id = $addr_result[0]; + my @update_user_data = ( + $first_name, + $middle_name, + $last_name, + $dob, + $phone, + $parent_guardian, + $name_keywords, + $ident_type, + $ident_value, + $ident2_value, + $home_branch, + $expire_date, + $user_id + ); + my @update_addr_data = ( + $street1, + $street2, + $city, + $county, + $state, + $post_code, + $addr_id + ); + # do the update + if ($debug) { + print "Updating $barcode\n"; + } + # TODO: see if we still need the try/catch stuff + try { + # update user + if ($debug == 2) { + print "Update user data: @update_user_data\n"; + } + my $sth_update_user = $dbh->prepare($update_user_sql); + $sth_update_user->execute(@update_user_data); + # update address + if ($debug == 2) { + print "Update address data: @update_addr_data\n"; + } + my $sth_update_addr = $dbh->prepare($update_address_sql); + $sth_update_addr->execute(@update_addr_data); + $dbh->commit; + $sth_update_user->finish; + $sth_update_addr->finish; + $updates++; + } catch { + warn "Transaction aborted because $_"; + eval { $dbh->rollback }; + }; + + } elsif ($already_imported && $opt_out ) { + if ($debug) { + print "Deleting $barcode\n"; + } + try { + # delete user + my $sth_delete_user = $dbh->prepare($delete_user_sql); + $sth_delete_user->execute($user_id); + $dbh->commit; + $sth_delete_user->finish; + $deletes++ + } catch { + warn "Transaction aborted because $_"; + eval { $dbh->rollback }; + }; + } elsif ($opt_out) { + # do nothing + next; + } else { + # import user + my @user_data = ( + $profile, + $username, + $email, + $password, + $ident_type, + $ident_value, + $ident_type, + $ident2_value, + $first_name, + $middle_name, + $last_name, + $phone, + $home_branch, + $dob, + $expire_date, + $juvenile, + $parent_guardian, + $name_keywords + ); + my @addr_data = ( + $username, + $street1, + $street2, + $city, + $county, + $state, + $post_code, + ); + my @card_data = ( + $username, + $barcode + ); + my @link_card_data = ( + $barcode, + $username + ); + my @link_addr_data = ( + $username, + $username + ); + + # do the import here + if ($debug) { + print "Importing $barcode\n"; + } + try { + # insert user + if ($debug == 2) { + print "Inserting user data: @user_data\n"; + } + my $sth_insert_user = $dbh->prepare($insert_user_sql); + $sth_insert_user->execute(@user_data); + # insert address + if ($debug == 2) { + print "Inserting address data: @addr_data\n"; + } + my $sth_insert_addr = $dbh->prepare($insert_address_sql); + $sth_insert_addr->execute(@addr_data); + # link address back to the user + my $sth_link_addr = $dbh->prepare($link_addr_sql); + $sth_link_addr->execute(@link_addr_data); + # insert card + if ($debug == 2) { + print "Inserting card data: @card_data\n"; + } + my $sth_insert_card = $dbh->prepare($insert_card_sql); + $sth_insert_card->execute(@card_data); + # link card back to the user + my $sth_link_card = $dbh->prepare($link_card_sql); + $sth_link_card->execute(@link_card_data); + $dbh->commit; + $sth_insert_user->finish; + $sth_insert_addr->finish; + $sth_link_addr->finish; + $sth_insert_card->finish; + $sth_link_card->finish; + } catch { + warn "Transaction aborted because $_"; + eval { $dbh->rollback }; + }; + $imports++; + } + + } + } + + # record the import in the DB + my @file_path = split('/', $csv_file); + my $filename = $file_path[-1]; # bare filename + try { + my $sth_insert_import = $dbh->prepare($insert_import_sql); + #TODO: error handling that gets pushed to the DB + $sth_insert_import->execute($district_id, $filename, $error_message); + $dbh->commit; + $sth_insert_import->finish; + } catch { + warn "Transaction aborted because $_"; + eval { $dbh->rollback }; + }; + # write exceptions to a file + my $exceptions_file = "/tmp/$filename.exceptions"; + my $exceptions_ref = \@exceptions; + csv (in => $exceptions_ref, out => "$exceptions_file") + or die Text::CSV->error_diag; + print "\n\nImport statistics:\n\n\tImports: $imports\n\tUpdates: $updates\n\tOpt-Outs: $deletes\n\tExceptions: " . scalar(@exceptions) . "\n\n\tCreated exceptions file $exceptions_file.\n"; + shift @retrieved_files; +} +