# This script is for importing student data from a CSV
# exported from a student information system (SIS).
#
-
-#TODO: STFP delete perms, create Exceptions dir
-
use warnings;
use strict;
use Getopt::Long;
+#use Net::FTP;
+#use Net::SFTP::Foreign;
+#use IO::Socket::SSL;
use Text::CSV qw/ csv /;
-use OpenILS::Utils::Cronscript;
-use OpenSRF::Utils::Logger qw/ $logger /;
+use DBI;
+use Try::Tiny;
use Date::Parse;
-# something to do rsync/file transfer
use File::Rsync;
use Data::Dumper;
-# TODO: setup command line options to
-# control things like remote server, etc.
-
-
-# initiate Cronscript
-my $cscript = OpenILS::Utils::Cronscript->new();
-# set up a cstore editor
-my $editor = $cscript->editor(xact=>1);
-# set up AppUtils
-my $U = 'OpenILS::Application::AppUtils';
-
-# set up variables
-my $work_dir_prefix = "/tmp";
-my $in_dir_prefix = "/home/opensrf/districts";
+# 0 = no stdout messages
+# 1 = basic
+# 2 = detailed
+my $debug = 1;
my $prefix = $ARGV[1] if $ARGV[1];
my @retrieved_files = $ARGV[0] if $ARGV[0];
-my $is_opt_out;
+my $remote_path_base = "/sftp";
+my $remote_file_dir = "Files";
+# TODO: develop a reliable query for getting the
+# student card - or use YAOUS or something to set
+# the profile.
+my $profile = 61; # Student Card
+my $district_id = 0;
+my $error_message;
my $imports;
my $updates;
my $deletes;
+my $opt_out;
my @exceptions;
-my @required_fields = qw(
- school_id
- student_id
- student_fname
- student_lname
- student_dob
- address_street1
- address_city
- address_state
- address_postal
- grade
+my @bad_student_ids;
+my $sth_district;
+
+# set up the DB connection
+my $dbh = DBI->connect
+(
+ "dbi:Pg:service=evergreen",
+ undef,
+ undef,
+ {
+ AutoCommit => 0,
+ RaiseError => 1,
+ PrintError => 0
+ }
+) or die DBI->errstr;
+
+# set up the SQL queries
+
+# get the districts
+my $district_sql = "select * from student_card.district where active = true";
+
+# get a specific district when in single-file mode
+my $single_district_sql = "select * from student_card.district where code = ?";
+
+# gather previously-processed filenames - populated below
+my $prev_files_sql;
+
+# check that user exists
+my $check_barcode_sql = "select 1 from actor.card where barcode = ?";
+
+# insert user
+my $insert_user_sql = qq(
+insert into actor.usr (
+ profile,
+ usrname,
+ email,
+ passwd,
+ ident_type,
+ ident_value,
+ ident_type2,
+ ident_value2,
+ first_given_name,
+ second_given_name,
+ family_name,
+ day_phone,
+ home_ou,
+ dob,
+ expire_date,
+ juvenile,
+ guardian,
+ name_keywords
+) values (
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ ?
+));
+
+my $school_data_sql = "select home_ou, name, addr_county from student_card.school where district_id = ? and state_id = ?";
+
+my $insert_address_sql = qq(
+insert into actor.usr_address (
+ usr,
+ street1,
+ street2,
+ city,
+ county,
+ state,
+ country,
+ post_code
+) values (
+ (select id from actor.usr where usrname = ?),
+ ?,
+ ?,
+ ?,
+ ?,
+ ?,
+ 'USA',
+ ?
+));
+
+my $link_addr_sql = qq(
+update actor.usr
+set mailing_address = (
+ select id
+ from actor.usr_address
+ where usr in (
+ select id
+ from actor.usr
+ where usrname = ?
+ )
+)
+where usrname = ?
);
-# we start by getting the active school districts
-# via cstore
-sub get_active_districts {
- my $districts = $editor->search_student_card_district(
- {active => 't'});
- return $districts;
-}
-
-
-sub get_files {
- # for each active district, check the FTP server for files
- # and copy them over
- my $district = shift;
- my $code = $district->code;
- my $prev_files = get_prev_files($district);
-
- # TODO: define the directory structure above
- my $work_dir = "$work_dir_prefix/$code/";
- mkdir $work_dir unless -d $work_dir;
- my $in_dir = "$in_dir_prefix/$code/In/";
- $logger->info("StudentCard: rsync-ing from $in_dir to $work_dir");
- my $rsync = File::Rsync->new(
- archive => 1,
- compress => 1);
- my $try = $rsync->exec(
- src => $in_dir,
- dest => $work_dir,
- exclude => $prev_files);
- my $error = $!;
- return $error unless $try;
- my @new_files;
- opendir(DIR, $work_dir) or die $!;
- while (my $file = readdir(DIR)) {
- next if (grep(/$file/, @$prev_files)) || ($file =~ m/^\./);
- push(@new_files, "$work_dir/$file");
- $logger->info("StudentCard: $file is a new file");
- }
- closedir(DIR);
- my $new_files = \@new_files;
- return $new_files;
-}
+my $insert_card_sql = qq(
+insert into actor.card (
+ usr,
+ barcode
+) values(
+ (select id from actor.usr where usrname = ?),
+ ?
+));
+
+my $link_card_sql = qq(
+update actor.usr
+set card = (
+ select id
+ from actor.card
+ where barcode = ?
+)
+where usrname = ?
+and not deleted
+);
-sub get_schools {
- my $district = shift;
- my $district_id = $district->id;
- my $district_name = $district->name;
- $logger->info("StudentCard: Getting schools for $district_name");
- my $schools = $editor->search_student_card_school(
- {district_id => $district_id}
- );
- return $schools;
-}
+my $insert_import_sql = qq(
+insert into student_card.import (
+ district_id,
+ filename,
+ error_message
+ ) values (
+ ?,
+ ?,
+ ?
+));
+
+# update user
+my $get_user_id_sql = qq (
+select usr
+from actor.card
+where barcode = ?
+);
-sub get_prev_files {
- my $district = shift;
- my $district_id = $district->id;
- my $district_name = $district->name;
- my $prev_files = $editor->search_student_card_import(
- {district_id => $district_id}
- );
- my @prev_files;
- for my $file (@$prev_files) {
- my $filename = $file->filename;
- push(@prev_files, $filename);
- }
- my $filelist = join(' ', @prev_files);
- $logger->info("StudentCard: Previous files for $district_name: $filelist");
- return \@prev_files;
-}
+my $get_addr_sql = qq(
+select mailing_address
+from actor.usr
+where id = ?
+);
-sub import_csv {
- my $csv_file = shift;
- $is_opt_out = 1 if $csv_file =~ /OptOut/;
- my $student_entries = csv (
- in => "$csv_file",
- headers => "lc", #lc = "lowercase"
- empty_is_undef => 1
- ) or die Text::CSV->error_diag;
- return $student_entries; # this is an array of hashes
-}
+my $update_user_sql = qq(
+update actor.usr set
+ first_given_name = ?,
+ second_given_name = ?,
+ family_name = ?,
+ dob = ?,
+ day_phone = ?,
+ guardian = ?,
+ name_keywords = ?,
+ ident_type = ?,
+ ident_value = ?,
+ ident_value2 = ?,
+ home_ou = ?,
+ expire_date = ?
+where id = ?
+);
-sub check_required_fields {
- my $student = shift;
- for my $field (@required_fields) {
- unless ( $student->{$field} ) {
- my $id = $student->{student_id};
- $logger->info("StudentCard: no $field for student ID $id");
- add_exception($student, "Missing $field");
- return 0;
- }
- }
- return 1;
-}
+my $update_address_sql = qq(
+update actor.usr_address set
+ street1 = ?,
+ street2 = ?,
+ city = ?,
+ county = ?,
+ state = ?,
+ post_code = ?
+where id = ?
+);
-sub lookup_school {
- my ($schools, $school_id) = @_;
- for my $school (@$schools) {
- my $state_id = $school->state_id;
- if (int($school_id) == $state_id) {
- return $school;
- }
- }
- $logger->warn("StudentCard: No school entered for $school_id");
- return 0;
-}
+# delete user
+my $delete_user_sql = qq(
+select actor.usr_delete(?, 0)
+);
sub calculate_expire_date {
my @now = localtime();
my ($now_year, $now_month) = ($now[5] + 1900, $now[4] + 1);
my $expire_year = $now_year;
- if ($now_month > 6) { # any student registered before end of June gets expire date the following 10/15
+ if ($now_month > 6) { # any student registered before end of June gets expire date the following 9/15
$expire_year += 1;
}
my $expire_date = $expire_year . "-10-15";
sub calculate_password {
my $dob = shift;
- return 0 unless $dob;
- my ($ss,$mm,$hh,$day,$month,$year,$zone) = strptime($dob);
- if ($year =~ /(\d{3})/) {
- $year += 1900;
- } else{
- $year += 2000;
- }
- $month += 1;
- if ($month < 10) {
- $month = "0"."$month";
- }
- my $password = $month . $year;
+ my @elements = split('-', $dob);
+ my ($dob_month, $dob_year) = ($elements[1], $elements[0]);
+ #if ($dob_month < 10) {
+ # $dob_month = "0" . $dob_month;
+ #}
+ my $password = $dob_month . $dob_year;
return $password;
}
-sub get_name_keywords {
- my $grade = shift;
- my $name_keywords;
- if ($grade && $grade =~ /^\d+$/) {
- if ($grade == 0) {
- $name_keywords = "GradeK";
- } elsif ($grade < 0) {
- $name_keywords = "GradePK";
- } elsif ($grade < 10) {
- $name_keywords = "Grade0" . $grade;
- } else {
- $name_keywords = "Grade" . $grade;
- }
- } elsif ($grade) {
- $name_keywords = $grade;
- }
- return $name_keywords;
-}
-
-# copied from Ecard.pm...
-# Create actor.usr perl object and populate column data
-sub make_user {
- my ($student, $school, $district_code) = @_;
-
- my $au = Fieldmapper::actor::user->new;
-
- print Dumper($au);
-
- # set up variables
- my $student_id = $student->{student_id};
- my $grade = $student->{grade};
- my $provided_pass = $student->{student_password};
- my $dob = $student->{student_dob};
-
- # populate fields
- $au->isnew(1);
- $au->first_given_name($student->{student_fname});
- $au->second_given_name($student->{student_mname});
- $au->family_name($student->{student_lname});
- $au->dob($dob);
- $au->day_phone($student->{student_phone});
- $au->email($student->{student_email});
- $au->passwd($provided_pass ? $provided_pass : calculate_password($dob));
- $au->guardian($student->{parent_guardian} ? $student->{parent_guardian} : $school->name);
- $au->ident_type(3); # Other
- $au->ident_value($student_id);
- $au->expire_date(calculate_expire_date);
- $au->juvenile(1);
- $au->name_keywords(get_name_keywords($grade));
- $au->net_access_level(1); # Filtered
- $au->profile($school->eg_perm_group);
- $au->home_ou($school->home_ou);
- $au->usrname($district_code.$student_id);
-
- return $au;
-
-}
-
-sub add_addresses {
- my ($student, $user) = @_;
-
- # address fields
-
- my $physical_addr = Fieldmapper::actor::user_address->new;
- $physical_addr->isnew(1);
- $physical_addr->usr($user->id);
- $physical_addr->address_type('PHYSICAL');
- $physical_addr->within_city_limits('f');
-
- my $mailing_addr = Fieldmapper::actor::user_address->new;
- $mailing_addr->isnew(1);
- $mailing_addr->usr($user->id);
- $mailing_addr->address_type('MAILING');
- $mailing_addr->within_city_limits('f');
-
- for my $type ($mailing_addr, $physical_addr) {
- $type->street1($student->{address_street1});
- $type->street2($student->{address_street2});
- $type->city($student->{address_city});
- $type->county($student->{address_county});
- $type->state($student->{address_state});
- $type->post_code($student->{address_postal});
- $type->country('USA');
- }
-
- # Use as both billing and mailing via virtual ID.
- $physical_addr->id(-1);
- $mailing_addr->id(-1);
- $user->billing_address(-1);
- $user->mailing_address(-1);
-
- $user->billing_address($physical_addr);
- $user->mailing_address($mailing_addr);
- $user->addresses([$physical_addr, $mailing_addr]);
-
- return 1;
-}
-
-
-sub add_card {
- my ($student, $user, $prefix) = @_;
- my $id = $student->{student_id};
- my $barcode = "$prefix"."$id";
-
- my $card = Fieldmapper::actor::card->new;
- $card->id(-1);
- $card->isnew(1);
- $card->usr($user->id);
- $card->barcode($barcode);
-
- # username defaults to barcode
- $user->usrname($barcode);
- $user->card($card);
- $user->cards([$card]);
-
- return 1;
-}
-
-
-sub do_login {
- my $auth = $U->simplereq(
- 'open-ils.auth_internal',
- 'open-ils.auth_internal.session.create',
- {user_id => 1, org_unit => 394, login_type => 'temp'}
- );
-
- return unless $auth && $auth->{textcode} eq 'SUCCESS';
-
- my $authtoken = $auth->{payload}->{authtoken};
+sub get_files {
+ my $district_code = shift;
- return $authtoken;
}
-
-sub user_exists {
- my ($student) = @_;
- my $lname = $student->{lname};
- my $ident = $student->{student_id};
- my $user = $editor->search_actor_user({
- family_name => $lname,
- ident_value => $ident
- });
- return $user if $user;
- return 0;
-
-}
-
-
-sub save_user {
- my ($user, $authtoken) = @_;
-
- print Dumper($user);
- print $authtoken;
+if (@retrieved_files) {
+ # we're in single-file mode, so get the district ID for later use
+ $sth_district = $dbh->prepare($single_district_sql);
+ $sth_district->execute($prefix);
+ while (my $district = $sth_district->fetchrow_hashref) {
+ $district_id = $district->{id};
+ }
+ die "No district with code $prefix." unless $district_id;
+ $sth_district->finish();
+} else {
+ # go out and get the files we need
+ $sth_district = $dbh->prepare($district_sql);
+ $sth_district->execute();
+ while (my $district = $sth_district->fetchrow_hashref) {
+ # check incoming FTP directory for the district
+ $district_id = $district->{id};
+ $prev_files_sql = "select filename from student_card.import where district_id = $district_id";
+ $prefix = $district->{code};
+ my $local_dir = "/tmp";
+ my $remote_dir = "$remote_path_base/$prefix/$remote_file_dir"
+ my @remote_files;
+ my $rsync = File::Rsync->new(
+ archive => 1,
+ compress => 1);
+ if ($debug) {
+ print "Now processing files for " . $district->{code} . "\n";
+ }
+ if ($debug == 2) {
+ print "District Prefix: $prefix\n";
+ print "Local Directory: $local_dir\n";
+ }
+
+
- my $save = $U->simplereq(
- 'open-ils.actor',
- 'open-ils.actor.patron.update',
- $authtoken, $user
- );
-
- if ($U->is_event($save)) {
- my $msg = "Unable to save account: " . $save->{textcode};
- $logger->error("StudentCard: $msg");
- return 0;
+ if ($debug == 2) {
+ print "Remote File Listing: @remote_files\n";
+ }
+ ## walk through the remote files and filter out any that do
+ ## not meet our naming requirements
+ foreach my $file (@remote_files) {
+ # XXX hard-coding the "OptOut" string here - maybe make it a setting?
+ if ($file =~ /${prefix}(OptOut)?_\d{12}/) {
+ my $previous = $dbh->selectcol_arrayref($prev_files_sql);
+ # skip files we have already retreived
+ unless (grep /$file/, @$previous) {
+ if ($debug) {
+ print "Getting $file\n";
+ }
+ if ($ftp_protocol =~ '^ftp(es)?') {
+ $ftp->get($file, "$local_dir/$file")
+ or die "Could not retrieve $file: ", $ftp->message;
+ push (@retrieved_files, "$local_dir/$file");
+ } elsif ($ftp_protocol = 'sftp') {
+ $sftp->get($file, "$local_dir/$file")
+ or die "Could not retrieve $file: " . $sftp->error;
+ push (@retrieved_files, "$local_dir/$file");
+ }
+ } else {
+ if ($debug) {
+ print "$file already retreived\n";
+ }
+ }
+ }
+ }
+ if ($ftp_protocol =~ '^ftp(es)?') {
+ $ftp->quit;
+ } elsif ($ftp_protocol = 'sftp') {
+ $sftp->disconnect;
+ }
}
- return 1;
-}
-
-
-sub add_exception {
- my ($student, $error) = @_;
- $student->{error} = $error;
- push (@exceptions, $student);
+ $sth_district->finish;
}
-# We've built everything, now get to work!
-my $districts = get_active_districts;
-my $authtoken = do_login;
-die "No authtoken available, cannot continue\n" unless $authtoken;
-
-# for each active district,
-for my $district(@$districts) {
- my $schools = get_schools($district);
- my $district_name = $district->name;
- my $district_code = $district->code;
-
- my $retrieved_files = get_files($district);
-
- if (scalar(@$retrieved_files) > 0) {
- $logger->info("StudentCard: Copying files from $district_name");
- } else {
- $logger->info("StudentCard: No new files for $district_name");
- next;
- }
+foreach my $csv_file (@retrieved_files) {
+ $imports = 0;
+ $updates = 0;
+ $deletes = 0;
+ $opt_out = 1 if $csv_file =~ /OptOut/;
+ # a couple of options here - we can assume that the headers are correct
+ # and use headers => "auto" here, which will fail to work if there's a
+ # typo in the header names, or we can do headers => "skip" and trust
+ # that the export follows our prescribed order.
+ # For now, we'll trust correct order over correct header names.
+ my $student_entries = csv (in => "$csv_file", headers => "skip", empty_is_undef => 1)
+ or die Text::CSV->error_diag;
+ foreach my $student (@$student_entries) {
+ # set up variables
+ my $school_id = @$student[0];
+ my $student_id = @$student[1];
+ my $first_name = @$student[2];
+ my $middle_name = @$student[3];
+ my $last_name = @$student[4];
+ my $dob = @$student[5];
+ my $phone = @$student[6];
+ my $email = @$student[7];
+ my $street1 = @$student[8];
+ my $street2 = @$student[9];
+ my $city = @$student[10];
+ my $county = @$student[11];
+ my $state = @$student[12];
+ my $post_code = @$student[13];
+ my $parent_guardian = @$student[14];
+ my $grade = @$student[15];
+ my $barcode = $prefix . $student_id;
+ my $username = $barcode;
+ my $password = calculate_password($dob);
+ my $ident_type = 3; # Other
+ my $ident_value = $student_id;
+ my $expire_date = calculate_expire_date;
+ my $juvenile = "true";
+ my $name_keywords;
+ if ($grade && $grade =~ /^\d+$/) {
+ if ($grade == 0) {
+ $name_keywords = "GradeK";
+ } elsif ($grade < 0) {
+ $name_keywords = "GradePK";
+ } elsif ($grade < 10) {
+ $name_keywords = "Grade0" . $grade;
+ } else {
+ $name_keywords = "Grade" . $grade;
+ }
+ } elsif ($grade) {
+ $name_keywords = $grade;
+ }
+ # figure out how to get the home ou from the DB based
+ # on the school ID
+ my $sth_school_data = $dbh->prepare($school_data_sql);
+ $sth_school_data->execute($district_id, $school_id);
+ my @school_result = $sth_school_data->fetchrow_array;
+ $sth_school_data->finish;
+ my $home_branch = $school_result[0];
+ my $school_name = $school_result[1];
+ my $school_county = $school_result[2];
+ my $ident2_value = $school_name;
+ # make sure we have a usable county entry
+ # if not, add from the school data
+ if (!$county || $county =~ m/\d/) {
+ $county = $school_county;
+ }
+ # check that we have the required data for the student
+ if (!$parent_guardian) {
+ $parent_guardian = $school_name
+ }
+ #TODO: write out an exceptions file
+ my $no_import = 0;
+ unless (
+ $school_id &&
+ $student_id &&
+ $first_name &&
+ $last_name &&
+ $dob &&
+ $street1 &&
+ $city &&
+ $state &&
+ $post_code &&
+ $home_branch ) {
+ if ($debug == 2) {
+ print "Student $student_id is an exception.\n";
+ }
+ push @exceptions, $student;
+ # push IDs into array for email
+ push @bad_student_ids, $student_id;
+ $no_import = 1;
+ }
+ unless ($no_import) {
+ # check if we already have the student account
+ my $sth_check_barcode = $dbh->prepare($check_barcode_sql);
+ $sth_check_barcode->execute($barcode);
+ my $already_imported = $sth_check_barcode->fetchrow_array;
+ $sth_check_barcode->finish;
+ my $sth_user_id = $dbh->prepare($get_user_id_sql);
+ $sth_user_id->execute($barcode);
+ my @user_result = $sth_user_id->fetchrow_array;
+ $sth_user_id->finish;
+ my $user_id = $user_result[0];
+ if ($already_imported && !$opt_out) {
+ my $sth_addr = $dbh->prepare($get_addr_sql);
+ $sth_addr->execute($user_id);
+ my @addr_result = $sth_addr->fetchrow_array;
+ $sth_addr->finish;
+ my $addr_id = $addr_result[0];
+ my @update_user_data = (
+ $first_name,
+ $middle_name,
+ $last_name,
+ $dob,
+ $phone,
+ $parent_guardian,
+ $name_keywords,
+ $ident_type,
+ $ident_value,
+ $ident2_value,
+ $home_branch,
+ $expire_date,
+ $user_id
+ );
+ my @update_addr_data = (
+ $street1,
+ $street2,
+ $city,
+ $county,
+ $state,
+ $post_code,
+ $addr_id
+ );
+ # do the update
+ if ($debug) {
+ print "Updating $barcode\n";
+ }
+ try {
+ # update user
+ if ($debug == 2) {
+ print "Update user data: @update_user_data\n";
+ }
+ my $sth_update_user = $dbh->prepare($update_user_sql);
+ $sth_update_user->execute(@update_user_data);
+ # update address
+ if ($debug == 2) {
+ print "Update address data: @update_addr_data\n";
+ }
+ my $sth_update_addr = $dbh->prepare($update_address_sql);
+ $sth_update_addr->execute(@update_addr_data);
+ $dbh->commit;
+ $sth_update_user->finish;
+ $sth_update_addr->finish;
+ $updates++;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+
+ } elsif ($already_imported && $opt_out ) {
+ if ($debug) {
+ print "Deleting $barcode\n";
+ }
+ try {
+ # delete user
+ my $sth_delete_user = $dbh->prepare($delete_user_sql);
+ $sth_delete_user->execute($user_id);
+ $dbh->commit;
+ $sth_delete_user->finish;
+ $deletes++
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ } else {
+ # import user
+ my @user_data = (
+ $profile,
+ $username,
+ $email,
+ $password,
+ $ident_type,
+ $ident_value,
+ $ident_type,
+ $ident2_value,
+ $first_name,
+ $middle_name,
+ $last_name,
+ $phone,
+ $home_branch,
+ $dob,
+ $expire_date,
+ $juvenile,
+ $parent_guardian,
+ $name_keywords
+ );
+ my @addr_data = (
+ $username,
+ $street1,
+ $street2,
+ $city,
+ $county,
+ $state,
+ $post_code,
+ );
+ my @card_data = (
+ $username,
+ $barcode
+ );
+ my @link_card_data = (
+ $barcode,
+ $username
+ );
+ my @link_addr_data = (
+ $username,
+ $username
+ );
+
+ # do the import here
+ if ($debug) {
+ print "Importing $barcode\n";
+ }
+ try {
+ # insert user
+ if ($debug == 2) {
+ print "Inserting user data: @user_data\n";
+ }
+ my $sth_insert_user = $dbh->prepare($insert_user_sql);
+ $sth_insert_user->execute(@user_data);
+ # insert address
+ if ($debug == 2) {
+ print "Inserting address data: @addr_data\n";
+ }
+ my $sth_insert_addr = $dbh->prepare($insert_address_sql);
+ $sth_insert_addr->execute(@addr_data);
+ # link address back to the user
+ my $sth_link_addr = $dbh->prepare($link_addr_sql);
+ $sth_link_addr->execute(@link_addr_data);
+ # insert card
+ if ($debug == 2) {
+ print "Inserting card data: @card_data\n";
+ }
+ my $sth_insert_card = $dbh->prepare($insert_card_sql);
+ $sth_insert_card->execute(@card_data);
+ # link card back to the user
+ my $sth_link_card = $dbh->prepare($link_card_sql);
+ $sth_link_card->execute(@link_card_data);
+ $dbh->commit;
+ $sth_insert_user->finish;
+ $sth_insert_addr->finish;
+ $sth_link_addr->finish;
+ $sth_insert_card->finish;
+ $sth_link_card->finish;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ $imports++;
+ }
+
+ }
+ }
- # for each retrieved file for the current district,
- for my $csv_file (@$retrieved_files) {
+ # record the import in the DB
+ my @file_path = split('/', $csv_file);
+ my $filename = $file_path[-1]; # bare filename
+ try {
+ my $sth_insert_import = $dbh->prepare($insert_import_sql);
+ #TODO: error handling that gets pushed to the DB
+ $sth_insert_import->execute($district_id, $filename, $error_message);
+ $dbh->commit;
+ $sth_insert_import->finish;
+ } catch {
+ warn "Transaction aborted because $_";
+ eval { $dbh->rollback };
+ };
+ # write exceptions to a file
+ my $exceptions_file = "/tmp/$filename.exceptions";
+ my $exceptions_ref = \@exceptions;
+ csv (in => $exceptions_ref, out => "$exceptions_file")
+ or die Text::CSV->error_diag;
+ print "\n\nImport statistics:\n\n\tImports: $imports\n\tUpdates: $updates\n\tOpt-Outs: $deletes\n\tExceptions: " . scalar(@exceptions) . "\n\n\tCreated exceptions file $exceptions_file.\n";
+ shift @retrieved_files;
+}
+# clean up
+$dbh->disconnect();
- #import the file
- $logger->info("Importing $csv_file for $district_name");
- my $student_entries = import_csv($csv_file);
-
- #for each student row in the current file,
- for my $student (@$student_entries) {
- next unless check_required_fields($student);
- #my $existing_user = user_exists($student, $district_code);
- #$logger->info("CSHARP: \$existing_user = " . Dumper($existing_user));
- my $school = lookup_school($schools, $student->{school_id});
- add_exception($student, "No school found with State ID $student->{school_id}") && next unless $school;
- my $user = make_user($student, $school, $district_code);
- add_addresses($student, $user);
- save_user($user, $authtoken);
- add_card($student, $user, $district_code);
- save_user($user, $authtoken); # not sure this is right...
- } # student
- } # csv_file
-} # district
projects / working / Evergreen.git / commitdiff