# This script is for importing student data from a CSV
# exported from a student information system (SIS).
#
+
+#TODO: STFP delete perms, create Exceptions dir
+
use warnings;
use strict;
-use File::Rsync;
+use Getopt::Long;
use Text::CSV qw/ csv /;
use OpenILS::Utils::Cronscript;
-use OpenILS::Application::Actor;
+use OpenSRF::Utils::Logger qw/ $logger /;
+use Date::Parse;
+# something to do rsync/file transfer
+use File::Rsync;
+use Data::Dumper;
+
+# TODO: setup command line options to
+# control things like remote server, etc.
+
+# initiate Cronscript
my $cscript = OpenILS::Utils::Cronscript->new();
+# set up a cstore editor
my $editor = $cscript->editor(xact=>1);
+# set up AppUtils
+my $U = 'OpenILS::Application::AppUtils';
-# 0 = no stdout messages
-# 1 = basic
-# 2 = detailed
-my $debug = 1;
-# the script can be run manually outside of cron, just
-# append <filename> <school code> to the script invocation
+# set up variables
+my $work_dir_prefix = "/tmp";
+my $in_dir_prefix = "/home/opensrf/districts";
my $prefix = $ARGV[1] if $ARGV[1];
my @retrieved_files = $ARGV[0] if $ARGV[0];
-my $district_id = 0;
-my $error_message;
+my $is_opt_out;
my $imports;
my $updates;
my $deletes;
-my $opt_out;
my @exceptions;
+my @required_fields = qw(
+ school_id
+ student_id
+ student_fname
+ student_lname
+ student_dob
+ address_street1
+ address_city
+ address_state
+ address_postal
+ grade
+);
+
+# we start by getting the active school districts
+# via cstore
+sub get_active_districts {
+ my $districts = $editor->search_student_card_district(
+ {active => 't'});
+ return $districts;
+}
+
+
+sub get_files {
+ # for each active district, check the FTP server for files
+ # and copy them over
+ my $district = shift;
+ my $code = $district->code;
+ my $prev_files = get_prev_files($district);
+
+ # TODO: define the directory structure above
+ my $work_dir = "$work_dir_prefix/$code/";
+ mkdir $work_dir unless -d $work_dir;
+ my $in_dir = "$in_dir_prefix/$code/In/";
+ $logger->info("StudentCard: rsync-ing from $in_dir to $work_dir");
+ my $rsync = File::Rsync->new(
+ archive => 1,
+ compress => 1);
+ my $try = $rsync->exec(
+ src => $in_dir,
+ dest => $work_dir,
+ exclude => $prev_files);
+ my $error = $!;
+ return $error unless $try;
+ my @new_files;
+ opendir(DIR, $work_dir) or die $!;
+ while (my $file = readdir(DIR)) {
+ next if (grep(/$file/, @$prev_files)) || ($file =~ m/^\./);
+ push(@new_files, "$work_dir/$file");
+ $logger->info("StudentCard: $file is a new file");
+ }
+ closedir(DIR);
+ my $new_files = \@new_files;
+ return $new_files;
+}
+
+sub get_schools {
+ my $district = shift;
+ my $district_id = $district->id;
+ my $district_name = $district->name;
+ $logger->info("StudentCard: Getting schools for $district_name");
+ my $schools = $editor->search_student_card_school(
+ {district_id => $district_id}
+ );
+ return $schools;
+}
+sub get_prev_files {
+ my $district = shift;
+ my $district_id = $district->id;
+ my $district_name = $district->name;
+ my $prev_files = $editor->search_student_card_import(
+ {district_id => $district_id}
+ );
+ my @prev_files;
+ for my $file (@$prev_files) {
+ my $filename = $file->filename;
+ push(@prev_files, $filename);
+ }
+ my $filelist = join(' ', @prev_files);
+ $logger->info("StudentCard: Previous files for $district_name: $filelist");
+ return \@prev_files;
+}
+
+sub import_csv {
+ my $csv_file = shift;
+ $is_opt_out = 1 if $csv_file =~ /OptOut/;
+ my $student_entries = csv (
+ in => "$csv_file",
+ headers => "lc", #lc = "lowercase"
+ empty_is_undef => 1
+ ) or die Text::CSV->error_diag;
+ return $student_entries; # this is an array of hashes
+}
+
+sub check_required_fields {
+ my $student = shift;
+ for my $field (@required_fields) {
+ unless ( $student->{$field} ) {
+ my $id = $student->{student_id};
+ $logger->info("StudentCard: no $field for student ID $id");
+ add_exception($student, "Missing $field");
+ return 0;
+ }
+ }
+ return 1;
+}
+
+sub lookup_school {
+ my ($schools, $school_id) = @_;
+ for my $school (@$schools) {
+ my $state_id = $school->state_id;
+ if (int($school_id) == $state_id) {
+ return $school;
+ }
+ }
+ $logger->warn("StudentCard: No school entered for $school_id");
+ return 0;
+}
sub calculate_expire_date {
my @now = localtime();
my ($now_year, $now_month) = ($now[5] + 1900, $now[4] + 1);
my $expire_year = $now_year;
- if ($now_month > 6) { # any student registered before end of June gets expire date the following 9/15
+ if ($now_month > 6) { # any student registered before end of June gets expire date the following 10/15
$expire_year += 1;
}
my $expire_date = $expire_year . "-10-15";
return $expire_date;
}
-# TODO: we need to alter this to accommodate format-agnostic dates
-sub calculate_password_from_dob {
+sub calculate_password {
my $dob = shift;
- my @elements = split('-', $dob);
- my ($dob_month, $dob_year) = ($elements[1], $elements[0]);
- my $password = $dob_month . $dob_year;
+ return 0 unless $dob;
+ my ($ss,$mm,$hh,$day,$month,$year,$zone) = strptime($dob);
+ if ($year =~ /(\d{3})/) {
+ $year += 1900;
+ } else{
+ $year += 2000;
+ }
+ $month += 1;
+ if ($month < 10) {
+ $month = "0"."$month";
+ }
+ my $password = $month . $year;
return $password;
}
-
-if (@retrieved_files) {
- # we're in single-file mode, so get the district ID for later use
- # TODO: get the district ID via cstore
-} else {
- # go out and get the files we need
- my @files;
- if ($debug) {
- print "Now processing files for " . $district->{code} . "\n";
- }
-
- ## walk through the remote files and filter out any that do
- ## not meet our naming requirements
- foreach my $file (@files) {
- # XXX hard-coding the "OptOut" string here - maybe make it a setting?
- if ($file =~ /${prefix}(OptOut)?_\d{12}/) {
- # TODO: add OpenSRF-y way to retrieve previous files
- my $previous = ;
- # skip files we have already retreived
- unless (grep /$file/, @$previous) {
- # get the file from the local FTP server
- } else {
- if ($debug) {
- print "$file already retreived\n";
- }
- }
- }
+sub get_name_keywords {
+ my $grade = shift;
+ my $name_keywords;
+ if ($grade && $grade =~ /^\d+$/) {
+ if ($grade == 0) {
+ $name_keywords = "GradeK";
+ } elsif ($grade < 0) {
+ $name_keywords = "GradePK";
+ } elsif ($grade < 10) {
+ $name_keywords = "Grade0" . $grade;
+ } else {
+ $name_keywords = "Grade" . $grade;
}
+ } elsif ($grade) {
+ $name_keywords = $grade;
}
+ return $name_keywords;
}
+# copied from Ecard.pm...
+# Create actor.usr perl object and populate column data
+sub make_user {
+ my ($student, $school, $district_code) = @_;
-foreach my $csv_file (@retrieved_files) {
- $imports = 0;
- $updates = 0;
- $deletes = 0;
- $opt_out = 1 if $csv_file =~ /OptOut/;
- # a couple of options here - we can assume that the headers are correct
- # and use headers => "auto" here, which will fail to work if there's a
- # typo in the header names, or we can do headers => "skip" and trust
- # that the export follows our prescribed order.
- my $student_entries = csv (in => "$csv_file", headers => "auto", empty_is_undef => 1)
- or die Text::CSV->error_diag;
- foreach my $student (@$student_entries) {
- # set up variables
- # TODO: make these assignment use the hash we just created with col names
- my $school_id = @$student[0];
- my $student_id = @$student[1];
- my $first_name = @$student[2];
- my $middle_name = @$student[3];
- my $last_name = @$student[4];
- my $dob = @$student[5];
- my $phone = @$student[6];
- my $email = @$student[7];
- my $street1 = @$student[8];
- my $street2 = @$student[9];
- my $city = @$student[10];
- my $county = @$student[11];
- my $state = @$student[12];
- my $post_code = @$student[13];
- my $parent_guardian = @$student[14];
- my $grade = @$student[15];
- my $barcode = $prefix . $student_id;
- my $username = $barcode;
- my $password = calculate_password_from_dob($dob);
- my $ident_type = 3; # Other
- my $ident_value = $student_id;
- my $expire_date = calculate_expire_date;
- my $juvenile = "true";
- my $name_keywords;
- if ($grade && $grade =~ /^\d+$/) {
- if ($grade == 0) {
- $name_keywords = "GradeK";
- } elsif ($grade < 0) {
- $name_keywords = "GradePK";
- } elsif ($grade < 10) {
- $name_keywords = "Grade0" . $grade;
- } else {
- $name_keywords = "Grade" . $grade;
- }
- } elsif ($grade) {
- $name_keywords = $grade;
- }
- # figure out how to get the home ou from the DB based
- # on the school ID
- # TODO: get these values via OpenSRF
- my $home_branch = $school_result[0];
- my $school_name = $school_result[1];
- my $school_county = $school_result[2];
- my $ident2_value = $school_name;
- # make sure we have a usable county entry
- # if not, add from the school data
- if (!$county || $county =~ m/\d/) {
- $county = $school_county;
- }
- # default to school for parent/guardian
- if (!$parent_guardian) {
- $parent_guardian = $school_name
- }
- # check that we have the required data for the student
- my $no_import = 0;
- unless (
- $school_id &&
- $student_id &&
- $first_name &&
- $last_name &&
- $dob &&
- $street1 &&
- $city &&
- $state &&
- $post_code &&
- $home_branch ) {
- if ($debug == 2) {
- print "Student $student_id is an exception.\n";
- }
- push @exceptions, $student;
- $no_import = 1;
- }
- unless ($no_import) {
- # check if we already have the student account
- # TODO: retrieve these with OpenSRF
- my $sth_check_barcode = $dbh->prepare($check_barcode_sql);
- $sth_check_barcode->execute($barcode);
- my $already_imported = $sth_check_barcode->fetchrow_array;
- $sth_check_barcode->finish;
- my $sth_user_id = $dbh->prepare($get_user_id_sql);
- $sth_user_id->execute($barcode);
- my @user_result = $sth_user_id->fetchrow_array;
- $sth_user_id->finish;
- my $user_id = $user_result[0];
-
- if ($already_imported && !$opt_out) {
- # assume that we're doing an update
- my $sth_addr = $dbh->prepare($get_addr_sql);
- $sth_addr->execute($user_id);
- my @addr_result = $sth_addr->fetchrow_array;
- $sth_addr->finish;
- my $addr_id = $addr_result[0];
- my @update_user_data = (
- $first_name,
- $middle_name,
- $last_name,
- $dob,
- $phone,
- $parent_guardian,
- $name_keywords,
- $ident_type,
- $ident_value,
- $ident2_value,
- $home_branch,
- $expire_date,
- $user_id
- );
- my @update_addr_data = (
- $street1,
- $street2,
- $city,
- $county,
- $state,
- $post_code,
- $addr_id
- );
- # do the update
- if ($debug) {
- print "Updating $barcode\n";
- }
- # TODO: see if we still need the try/catch stuff
- try {
- # update user
- if ($debug == 2) {
- print "Update user data: @update_user_data\n";
- }
- my $sth_update_user = $dbh->prepare($update_user_sql);
- $sth_update_user->execute(@update_user_data);
- # update address
- if ($debug == 2) {
- print "Update address data: @update_addr_data\n";
- }
- my $sth_update_addr = $dbh->prepare($update_address_sql);
- $sth_update_addr->execute(@update_addr_data);
- $dbh->commit;
- $sth_update_user->finish;
- $sth_update_addr->finish;
- $updates++;
- } catch {
- warn "Transaction aborted because $_";
- eval { $dbh->rollback };
- };
-
- } elsif ($already_imported && $opt_out ) {
- if ($debug) {
- print "Deleting $barcode\n";
- }
- try {
- # delete user
- my $sth_delete_user = $dbh->prepare($delete_user_sql);
- $sth_delete_user->execute($user_id);
- $dbh->commit;
- $sth_delete_user->finish;
- $deletes++
- } catch {
- warn "Transaction aborted because $_";
- eval { $dbh->rollback };
- };
- } elsif ($opt_out) {
- # do nothing
- next;
- } else {
- # import user
- my @user_data = (
- $profile,
- $username,
- $email,
- $password,
- $ident_type,
- $ident_value,
- $ident_type,
- $ident2_value,
- $first_name,
- $middle_name,
- $last_name,
- $phone,
- $home_branch,
- $dob,
- $expire_date,
- $juvenile,
- $parent_guardian,
- $name_keywords
- );
- my @addr_data = (
- $username,
- $street1,
- $street2,
- $city,
- $county,
- $state,
- $post_code,
- );
- my @card_data = (
- $username,
- $barcode
- );
- my @link_card_data = (
- $barcode,
- $username
- );
- my @link_addr_data = (
- $username,
- $username
- );
-
- # do the import here
- if ($debug) {
- print "Importing $barcode\n";
- }
- try {
- # insert user
- if ($debug == 2) {
- print "Inserting user data: @user_data\n";
- }
- my $sth_insert_user = $dbh->prepare($insert_user_sql);
- $sth_insert_user->execute(@user_data);
- # insert address
- if ($debug == 2) {
- print "Inserting address data: @addr_data\n";
- }
- my $sth_insert_addr = $dbh->prepare($insert_address_sql);
- $sth_insert_addr->execute(@addr_data);
- # link address back to the user
- my $sth_link_addr = $dbh->prepare($link_addr_sql);
- $sth_link_addr->execute(@link_addr_data);
- # insert card
- if ($debug == 2) {
- print "Inserting card data: @card_data\n";
- }
- my $sth_insert_card = $dbh->prepare($insert_card_sql);
- $sth_insert_card->execute(@card_data);
- # link card back to the user
- my $sth_link_card = $dbh->prepare($link_card_sql);
- $sth_link_card->execute(@link_card_data);
- $dbh->commit;
- $sth_insert_user->finish;
- $sth_insert_addr->finish;
- $sth_link_addr->finish;
- $sth_insert_card->finish;
- $sth_link_card->finish;
- } catch {
- warn "Transaction aborted because $_";
- eval { $dbh->rollback };
- };
- $imports++;
- }
-
- }
- }
+ my $au = Fieldmapper::actor::user->new;
+
+ print Dumper($au);
+
+ # set up variables
+ my $student_id = $student->{student_id};
+ my $grade = $student->{grade};
+ my $provided_pass = $student->{student_password};
+ my $dob = $student->{student_dob};
+
+ # populate fields
+ $au->isnew(1);
+ $au->first_given_name($student->{student_fname});
+ $au->second_given_name($student->{student_mname});
+ $au->family_name($student->{student_lname});
+ $au->dob($dob);
+ $au->day_phone($student->{student_phone});
+ $au->email($student->{student_email});
+ $au->passwd($provided_pass ? $provided_pass : calculate_password($dob));
+ $au->guardian($student->{parent_guardian} ? $student->{parent_guardian} : $school->name);
+ $au->ident_type(3); # Other
+ $au->ident_value($student_id);
+ $au->expire_date(calculate_expire_date);
+ $au->juvenile(1);
+ $au->name_keywords(get_name_keywords($grade));
+ $au->net_access_level(1); # Filtered
+ $au->profile($school->eg_perm_group);
+ $au->home_ou($school->home_ou);
+ $au->usrname($district_code.$student_id);
+
+ return $au;
- # record the import in the DB
- my @file_path = split('/', $csv_file);
- my $filename = $file_path[-1]; # bare filename
- try {
- my $sth_insert_import = $dbh->prepare($insert_import_sql);
- #TODO: error handling that gets pushed to the DB
- $sth_insert_import->execute($district_id, $filename, $error_message);
- $dbh->commit;
- $sth_insert_import->finish;
- } catch {
- warn "Transaction aborted because $_";
- eval { $dbh->rollback };
- };
- # write exceptions to a file
- my $exceptions_file = "/tmp/$filename.exceptions";
- my $exceptions_ref = \@exceptions;
- csv (in => $exceptions_ref, out => "$exceptions_file")
- or die Text::CSV->error_diag;
- print "\n\nImport statistics:\n\n\tImports: $imports\n\tUpdates: $updates\n\tOpt-Outs: $deletes\n\tExceptions: " . scalar(@exceptions) . "\n\n\tCreated exceptions file $exceptions_file.\n";
- shift @retrieved_files;
}
+sub add_addresses {
+ my ($student, $user) = @_;
+
+ # address fields
+
+ my $physical_addr = Fieldmapper::actor::user_address->new;
+ $physical_addr->isnew(1);
+ $physical_addr->usr($user->id);
+ $physical_addr->address_type('PHYSICAL');
+ $physical_addr->within_city_limits('f');
+
+ my $mailing_addr = Fieldmapper::actor::user_address->new;
+ $mailing_addr->isnew(1);
+ $mailing_addr->usr($user->id);
+ $mailing_addr->address_type('MAILING');
+ $mailing_addr->within_city_limits('f');
+
+ for my $type ($mailing_addr, $physical_addr) {
+ $type->street1($student->{address_street1});
+ $type->street2($student->{address_street2});
+ $type->city($student->{address_city});
+ $type->county($student->{address_county});
+ $type->state($student->{address_state});
+ $type->post_code($student->{address_postal});
+ $type->country('USA');
+ }
+
+ # Use as both billing and mailing via virtual ID.
+ $physical_addr->id(-1);
+ $mailing_addr->id(-1);
+ $user->billing_address(-1);
+ $user->mailing_address(-1);
+
+ $user->billing_address($physical_addr);
+ $user->mailing_address($mailing_addr);
+ $user->addresses([$physical_addr, $mailing_addr]);
+
+ return 1;
+}
+
+
+sub add_card {
+ my ($student, $user, $prefix) = @_;
+ my $id = $student->{student_id};
+ my $barcode = "$prefix"."$id";
+
+ my $card = Fieldmapper::actor::card->new;
+ $card->id(-1);
+ $card->isnew(1);
+ $card->usr($user->id);
+ $card->barcode($barcode);
+
+ # username defaults to barcode
+ $user->usrname($barcode);
+ $user->card($card);
+ $user->cards([$card]);
+
+ return 1;
+}
+
+
+sub do_login {
+ my $auth = $U->simplereq(
+ 'open-ils.auth_internal',
+ 'open-ils.auth_internal.session.create',
+ {user_id => 1, org_unit => 394, login_type => 'temp'}
+ );
+
+ return unless $auth && $auth->{textcode} eq 'SUCCESS';
+
+ my $authtoken = $auth->{payload}->{authtoken};
+
+ return $authtoken;
+}
+
+
+sub user_exists {
+ my ($student) = @_;
+ my $lname = $student->{lname};
+ my $ident = $student->{student_id};
+ my $user = $editor->search_actor_user({
+ family_name => $lname,
+ ident_value => $ident
+ });
+ return $user if $user;
+ return 0;
+
+}
+
+
+sub save_user {
+ my ($user, $authtoken) = @_;
+
+ print Dumper($user);
+ print $authtoken;
+
+ my $save = $U->simplereq(
+ 'open-ils.actor',
+ 'open-ils.actor.patron.update',
+ $authtoken, $user
+ );
+
+ if ($U->is_event($save)) {
+ my $msg = "Unable to save account: " . $save->{textcode};
+ $logger->error("StudentCard: $msg");
+ return 0;
+ }
+ return 1;
+}
+
+
+sub add_exception {
+ my ($student, $error) = @_;
+ $student->{error} = $error;
+ push (@exceptions, $student);
+}
+
+# We've built everything, now get to work!
+
+my $districts = get_active_districts;
+my $authtoken = do_login;
+die "No authtoken available, cannot continue\n" unless $authtoken;
+
+# for each active district,
+for my $district(@$districts) {
+ my $schools = get_schools($district);
+ my $district_name = $district->name;
+ my $district_code = $district->code;
+
+ my $retrieved_files = get_files($district);
+
+ if (scalar(@$retrieved_files) > 0) {
+ $logger->info("StudentCard: Copying files from $district_name");
+ } else {
+ $logger->info("StudentCard: No new files for $district_name");
+ next;
+ }
+
+ # for each retrieved file for the current district,
+ for my $csv_file (@$retrieved_files) {
+
+ #import the file
+ $logger->info("Importing $csv_file for $district_name");
+ my $student_entries = import_csv($csv_file);
+
+ #for each student row in the current file,
+ for my $student (@$student_entries) {
+ next unless check_required_fields($student);
+ #my $existing_user = user_exists($student, $district_code);
+ #$logger->info("CSHARP: \$existing_user = " . Dumper($existing_user));
+ my $school = lookup_school($schools, $student->{school_id});
+ add_exception($student, "No school found with State ID $student->{school_id}") && next unless $school;
+ my $user = make_user($student, $school, $district_code);
+ add_addresses($student, $user);
+ save_user($user, $authtoken);
+ add_card($student, $user, $district_code);
+ save_user($user, $authtoken); # not sure this is right...
+ } # student
+ } # csv_file
+} # district
projects / working / Evergreen.git / commitdiff