From: Chris Sharp <csharp@georgialibraries.org>
Date: Mon, 25 Oct 2021 17:26:29 +0000 (-0400)
Subject: latest updates
X-Git-Url: https://old-git.evergreen-ils.org/?a=commitdiff_plain;h=b9af45fd3ba19ceee59648015817d8cffd89702d;p=working%2FEvergreen.git

latest updates
---

diff --git a/Open-ILS/src/support-scripts/import_student_data.pl b/Open-ILS/src/support-scripts/import_student_data.pl
index 42cd8dbf8b..0b9b27d380 100755
--- a/Open-ILS/src/support-scripts/import_student_data.pl
+++ b/Open-ILS/src/support-scripts/import_student_data.pl
@@ -19,368 +19,403 @@
 # This script is for importing student data from a CSV
 # exported from a student information system (SIS).
 #
+
+#TODO: STFP delete perms, create Exceptions dir
+
 use warnings;
 use strict;
-use File::Rsync;
+use Getopt::Long;
 use Text::CSV qw/ csv /;
 use OpenILS::Utils::Cronscript;
-use OpenILS::Application::Actor;
+use OpenSRF::Utils::Logger qw/ $logger /;
+use Date::Parse;
+# something to do rsync/file transfer
+use File::Rsync;
+use Data::Dumper;
+
+# TODO: setup command line options to 
+# control things like remote server, etc.
+
 
+# initiate Cronscript
 my $cscript = OpenILS::Utils::Cronscript->new();
+# set up a cstore editor
 my $editor = $cscript->editor(xact=>1);
+# set up AppUtils
+my $U = 'OpenILS::Application::AppUtils';
 
-# 0 = no stdout messages
-# 1 = basic
-# 2 = detailed
-my $debug = 1;
-# the script can be run manually outside of cron, just
-# append <filename> <school code> to the script invocation
+# set up variables
+my $work_dir_prefix = "/tmp";
+my $in_dir_prefix = "/home/opensrf/districts";
 my $prefix = $ARGV[1] if $ARGV[1];
 my @retrieved_files = $ARGV[0] if $ARGV[0];
-my $district_id = 0;
-my $error_message;
+my $is_opt_out;
 my $imports;
 my $updates;
 my $deletes;
-my $opt_out;
 my @exceptions;
+my @required_fields = qw(
+    school_id
+    student_id
+    student_fname
+    student_lname
+    student_dob 
+    address_street1
+    address_city
+    address_state
+    address_postal
+    grade
+);
+
+# we start by getting the active school districts
+# via cstore
+sub get_active_districts {
+    my $districts = $editor->search_student_card_district(
+        {active => 't'});
+    return $districts;
+}
+
+
+sub get_files {
+    # for each active district, check the FTP server for files
+    # and copy them over 
+    my $district = shift;
+    my $code = $district->code;
+    my $prev_files = get_prev_files($district);
+    
+    # TODO: define the directory structure above
+    my $work_dir = "$work_dir_prefix/$code/";
+    mkdir $work_dir unless -d $work_dir;
+    my $in_dir = "$in_dir_prefix/$code/In/";
+    $logger->info("StudentCard: rsync-ing from $in_dir to $work_dir");
+    my $rsync = File::Rsync->new(
+        archive      => 1,
+        compress     => 1);
+    my $try = $rsync->exec(
+        src => $in_dir, 
+        dest => $work_dir,
+        exclude => $prev_files);
+    my $error = $!;
+    return $error unless $try;
+    my @new_files;
+    opendir(DIR, $work_dir) or die $!;
+    while (my $file = readdir(DIR)) {
+        next if (grep(/$file/, @$prev_files)) || ($file =~ m/^\./);
+        push(@new_files, "$work_dir/$file");
+        $logger->info("StudentCard: $file is a new file");
+    }
+    closedir(DIR);
+    my $new_files = \@new_files;
+    return $new_files;
+}
+
+sub get_schools {
+    my $district = shift;
+    my $district_id = $district->id;
+    my $district_name = $district->name;
+    $logger->info("StudentCard: Getting schools for $district_name");
+    my $schools = $editor->search_student_card_school(
+       {district_id => $district_id}
+    );
+    return $schools;
+}
 
+sub get_prev_files {
+    my $district = shift;
+    my $district_id = $district->id;
+    my $district_name = $district->name;
+    my $prev_files = $editor->search_student_card_import(
+        {district_id => $district_id}
+    );
+    my @prev_files;
+    for my $file (@$prev_files) {
+        my $filename = $file->filename;
+        push(@prev_files, $filename);
+    }
+    my $filelist = join(' ', @prev_files);
+    $logger->info("StudentCard: Previous files for $district_name: $filelist");
+    return \@prev_files;
+}
+
+sub import_csv {
+    my $csv_file = shift;
+    $is_opt_out = 1 if $csv_file =~ /OptOut/;
+    my $student_entries = csv (
+        in => "$csv_file",
+        headers => "lc", #lc = "lowercase"
+        empty_is_undef => 1
+    ) or die Text::CSV->error_diag;
+    return $student_entries; # this is an array of hashes
+}
+
+sub check_required_fields {
+    my $student = shift;
+    for my $field (@required_fields) {
+        unless ( $student->{$field} ) {
+            my $id = $student->{student_id};
+            $logger->info("StudentCard: no $field for student ID $id");
+            add_exception($student, "Missing $field");
+            return 0;
+        }
+    }
+    return 1;   
+}
+
+sub lookup_school {
+    my ($schools, $school_id) = @_;
+    for my $school (@$schools) {
+        my $state_id = $school->state_id;
+        if (int($school_id) == $state_id) {
+            return $school;
+        }
+    }
+    $logger->warn("StudentCard: No school entered for $school_id");
+    return 0;
+}
 
 sub calculate_expire_date {
     my @now = localtime();
     my ($now_year, $now_month) = ($now[5] + 1900, $now[4] + 1);
     my $expire_year = $now_year;
-    if ($now_month > 6) { # any student registered before end of June gets expire date the following 9/15
+    if ($now_month > 6) { # any student registered before end of June gets expire date the following 10/15
         $expire_year += 1;
     }
     my $expire_date = $expire_year . "-10-15";
     return $expire_date;
 }
 
-# TODO: we need to alter this to accommodate format-agnostic dates
-sub calculate_password_from_dob {
+sub calculate_password {
     my $dob = shift;
-    my @elements = split('-', $dob);
-    my ($dob_month, $dob_year) = ($elements[1], $elements[0]);
-    my $password = $dob_month . $dob_year;
+    return 0 unless $dob;
+    my ($ss,$mm,$hh,$day,$month,$year,$zone) = strptime($dob);
+    if ($year =~ /(\d{3})/) {
+        $year += 1900;
+    } else{
+        $year += 2000;
+    }
+    $month += 1;
+    if ($month < 10) {
+            $month = "0"."$month";
+    }
+    my $password = $month . $year;
     return $password;
 }
 
-
-if (@retrieved_files) {
-    # we're in single-file mode, so get the district ID for later use
-    # TODO: get the district ID via cstore
-} else {
-    # go out and get the files we need
-        my @files;
-        if ($debug) {
-            print "Now processing files for " . $district->{code} . "\n";
-        }
-    
-        ## walk through the remote files and filter out any that do
-        ## not meet our naming requirements
-        foreach my $file (@files) {
-            # XXX hard-coding the "OptOut" string here - maybe make it a setting? 
-            if ($file =~ /${prefix}(OptOut)?_\d{12}/) { 
-                # TODO: add OpenSRF-y way to retrieve previous files
-                my $previous =  ;
-                # skip files we have already retreived
-                unless (grep /$file/, @$previous) {
-                    # get the file from the local FTP server
-                } else {
-                    if ($debug) {
-                        print "$file already retreived\n";
-                    }
-                }
-            }
+sub get_name_keywords {
+    my $grade = shift;
+    my $name_keywords;
+    if ($grade && $grade =~ /^\d+$/) {
+        if ($grade == 0) {
+            $name_keywords = "GradeK";
+        } elsif ($grade < 0) {
+            $name_keywords = "GradePK";
+        } elsif ($grade < 10) {
+            $name_keywords = "Grade0" . $grade;
+        } else {
+            $name_keywords = "Grade" . $grade;
         }
+    } elsif ($grade) {
+        $name_keywords = $grade;
     }
+    return $name_keywords;
 }
 
+# copied from Ecard.pm...
+# Create actor.usr perl object and populate column data
+sub make_user {
+    my ($student, $school, $district_code) = @_;
 
-foreach my $csv_file (@retrieved_files) {
-    $imports = 0;
-    $updates = 0;
-    $deletes = 0;
-    $opt_out = 1 if $csv_file =~ /OptOut/;
-    # a couple of options here - we can assume that the headers are correct
-    # and use headers => "auto" here, which will fail to work if there's a
-    # typo in the header names, or we can do headers => "skip" and trust
-    # that the export follows our prescribed order.
-    my $student_entries = csv (in => "$csv_file", headers => "auto", empty_is_undef => 1)
-        or die Text::CSV->error_diag;
-    foreach my $student (@$student_entries) {
-        # set up variables
-        # TODO: make these assignment use the hash we just created with col names
-        my $school_id       = @$student[0]; 
-        my $student_id      = @$student[1]; 
-        my $first_name      = @$student[2]; 
-        my $middle_name     = @$student[3]; 
-        my $last_name       = @$student[4]; 
-        my $dob             = @$student[5]; 
-        my $phone           = @$student[6]; 
-        my $email           = @$student[7]; 
-        my $street1         = @$student[8]; 
-        my $street2         = @$student[9]; 
-        my $city            = @$student[10]; 
-        my $county          = @$student[11];
-        my $state           = @$student[12]; 
-        my $post_code       = @$student[13]; 
-        my $parent_guardian = @$student[14]; 
-        my $grade           = @$student[15];
-        my $barcode         = $prefix . $student_id;
-        my $username        = $barcode;
-        my $password        = calculate_password_from_dob($dob);         
-        my $ident_type      = 3; # Other
-        my $ident_value     = $student_id;
-        my $expire_date     = calculate_expire_date;
-        my $juvenile        = "true";
-        my $name_keywords;
-        if ($grade && $grade =~ /^\d+$/) {
-            if ($grade == 0) {
-                $name_keywords = "GradeK";
-            } elsif ($grade < 0) {
-                $name_keywords = "GradePK";
-            } elsif ($grade < 10) {
-                $name_keywords = "Grade0" . $grade;
-            } else {
-                $name_keywords = "Grade" . $grade;
-            }
-        } elsif ($grade) {
-            $name_keywords = $grade;
-        }
-        # figure out how to get the home ou from the DB based
-        # on the school ID
-        # TODO: get these values via OpenSRF
-        my $home_branch = $school_result[0];
-        my $school_name = $school_result[1];
-        my $school_county = $school_result[2];
-        my $ident2_value = $school_name;
-        # make sure we have a usable county entry
-        # if not, add from the school data
-        if (!$county || $county =~ m/\d/) {
-            $county = $school_county;
-        }
-        # default to school for parent/guardian
-        if (!$parent_guardian) {
-            $parent_guardian = $school_name
-        }
-        # check that we have the required data for the student
-        my $no_import = 0;
-        unless (
-            $school_id && 
-            $student_id &&
-            $first_name &&
-            $last_name &&
-            $dob &&
-            $street1 &&
-            $city &&
-            $state &&
-            $post_code &&
-            $home_branch ) {
-            if ($debug == 2) {
-                print "Student $student_id is an exception.\n";
-            }
-            push @exceptions, $student;
-            $no_import = 1;
-        }
-        unless ($no_import) {
-            # check if we already have the student account
-            # TODO: retrieve these with OpenSRF
-            my $sth_check_barcode = $dbh->prepare($check_barcode_sql);
-            $sth_check_barcode->execute($barcode);
-            my $already_imported = $sth_check_barcode->fetchrow_array;
-            $sth_check_barcode->finish;
-            my $sth_user_id = $dbh->prepare($get_user_id_sql);
-            $sth_user_id->execute($barcode);
-            my @user_result = $sth_user_id->fetchrow_array;
-            $sth_user_id->finish;
-            my $user_id = $user_result[0];
-
-            if ($already_imported && !$opt_out) {
-                # assume that we're doing an update
-                my $sth_addr = $dbh->prepare($get_addr_sql);
-                $sth_addr->execute($user_id);
-                my @addr_result = $sth_addr->fetchrow_array;
-                $sth_addr->finish;
-                my $addr_id = $addr_result[0];
-                my @update_user_data = (
-                    $first_name,
-                    $middle_name,
-                    $last_name,
-                    $dob,
-                    $phone,
-                    $parent_guardian,
-                    $name_keywords,
-                    $ident_type,
-                    $ident_value,
-                    $ident2_value,
-                    $home_branch,
-                    $expire_date,
-                    $user_id
-                );
-                my @update_addr_data = (
-                    $street1,
-                    $street2,
-                    $city,
-                    $county,
-                    $state,
-                    $post_code,
-                    $addr_id
-                );
-                # do the update
-                if ($debug) {
-                    print "Updating $barcode\n";
-                }
-                # TODO: see if we still need the try/catch stuff
-                try {
-                    # update user
-                    if ($debug == 2) {
-                        print "Update user data: @update_user_data\n";
-                    }
-                    my $sth_update_user = $dbh->prepare($update_user_sql);
-                    $sth_update_user->execute(@update_user_data);
-                    # update address
-                    if ($debug == 2) {
-                        print "Update address data: @update_addr_data\n";
-                    }
-                    my $sth_update_addr = $dbh->prepare($update_address_sql);
-                    $sth_update_addr->execute(@update_addr_data);
-                    $dbh->commit;
-                    $sth_update_user->finish;
-                    $sth_update_addr->finish;
-                    $updates++;
-                } catch {
-                    warn "Transaction aborted because $_";
-                    eval { $dbh->rollback };
-                };
- 
-            } elsif ($already_imported && $opt_out ) {
-                if ($debug) {
-                    print "Deleting $barcode\n";
-                }
-                try {
-                    # delete user
-                    my $sth_delete_user = $dbh->prepare($delete_user_sql);
-                    $sth_delete_user->execute($user_id);
-                    $dbh->commit;
-                    $sth_delete_user->finish;
-                    $deletes++
-                } catch {
-                    warn "Transaction aborted because $_";
-                    eval { $dbh->rollback };
-                };
-            } elsif ($opt_out) {
-                # do nothing
-                next;
-            } else {
-                # import user
-                my @user_data = (
-                    $profile, 
-                    $username, 
-                    $email,
-                    $password,
-                    $ident_type,
-                    $ident_value,
-                    $ident_type,
-                    $ident2_value,
-                    $first_name,
-                    $middle_name,
-                    $last_name,
-                    $phone,
-                    $home_branch,
-                    $dob,
-                    $expire_date,   
-                    $juvenile,
-                    $parent_guardian,
-                    $name_keywords
-                );
-                my @addr_data = (
-                    $username,
-                    $street1,
-                    $street2,
-                    $city,
-                    $county,
-                    $state,
-                    $post_code,
-                );
-                my @card_data = (
-                    $username,
-                    $barcode
-                );
-                my @link_card_data = (
-                    $barcode,
-                    $username
-                );
-                my @link_addr_data = (
-                    $username,
-                    $username
-                );
-                    
-               # do the import here
-                if ($debug) {
-                    print "Importing $barcode\n";
-                    }
-                    try {
-                        # insert user
-                        if ($debug == 2) {
-                            print "Inserting user data: @user_data\n";
-                        }
-                        my $sth_insert_user = $dbh->prepare($insert_user_sql);
-                        $sth_insert_user->execute(@user_data);
-                        # insert address
-                        if ($debug == 2) {
-                            print "Inserting address data: @addr_data\n";
-                        }
-                        my $sth_insert_addr = $dbh->prepare($insert_address_sql);
-                        $sth_insert_addr->execute(@addr_data);
-                        # link address back to the user
-                        my $sth_link_addr = $dbh->prepare($link_addr_sql);
-                        $sth_link_addr->execute(@link_addr_data);
-                        # insert card
-                        if ($debug == 2) {
-                            print "Inserting card data: @card_data\n";
-                        }
-                        my $sth_insert_card = $dbh->prepare($insert_card_sql);
-                        $sth_insert_card->execute(@card_data);
-                        # link card back to the user
-                        my $sth_link_card = $dbh->prepare($link_card_sql);
-                        $sth_link_card->execute(@link_card_data);
-                        $dbh->commit;
-                        $sth_insert_user->finish;
-                        $sth_insert_addr->finish;
-                        $sth_link_addr->finish;
-                        $sth_insert_card->finish;
-                        $sth_link_card->finish;
-                    } catch {
-                        warn "Transaction aborted because $_";
-                        eval { $dbh->rollback };
-                    };
-                    $imports++;
-                }
-            
-            }
-        }
+    my $au = Fieldmapper::actor::user->new;
+
+    print Dumper($au);
+
+    # set up variables
+    my $student_id      = $student->{student_id};
+    my $grade           = $student->{grade};
+    my $provided_pass   = $student->{student_password};
+    my $dob             = $student->{student_dob};
+
+    # populate fields 
+    $au->isnew(1);
+    $au->first_given_name($student->{student_fname});
+    $au->second_given_name($student->{student_mname});
+    $au->family_name($student->{student_lname});
+    $au->dob($dob);
+    $au->day_phone($student->{student_phone});
+    $au->email($student->{student_email});
+    $au->passwd($provided_pass ? $provided_pass : calculate_password($dob));
+    $au->guardian($student->{parent_guardian} ? $student->{parent_guardian} : $school->name);
+    $au->ident_type(3); # Other
+    $au->ident_value($student_id);
+    $au->expire_date(calculate_expire_date);
+    $au->juvenile(1);
+    $au->name_keywords(get_name_keywords($grade));
+    $au->net_access_level(1); # Filtered
+    $au->profile($school->eg_perm_group);
+    $au->home_ou($school->home_ou);
+    $au->usrname($district_code.$student_id);
+
+    return $au;
 
-    # record the import in the DB
-    my @file_path = split('/', $csv_file);
-    my $filename = $file_path[-1]; # bare filename
-    try {     
-        my $sth_insert_import = $dbh->prepare($insert_import_sql);
-        #TODO: error handling that gets pushed to the DB
-        $sth_insert_import->execute($district_id, $filename, $error_message);
-        $dbh->commit;
-        $sth_insert_import->finish;
-    } catch {
-        warn "Transaction aborted because $_";
-        eval { $dbh->rollback };
-    };
-    # write exceptions to a file
-    my $exceptions_file = "/tmp/$filename.exceptions";
-    my $exceptions_ref = \@exceptions;
-    csv (in => $exceptions_ref, out => "$exceptions_file")
-        or die Text::CSV->error_diag;
-    print "\n\nImport statistics:\n\n\tImports: $imports\n\tUpdates: $updates\n\tOpt-Outs: $deletes\n\tExceptions: " . scalar(@exceptions) . "\n\n\tCreated exceptions file $exceptions_file.\n";
-    shift @retrieved_files;
 }
 
+sub add_addresses {
+    my ($student, $user) = @_;
+
+    # address fields
+
+    my $physical_addr = Fieldmapper::actor::user_address->new;
+    $physical_addr->isnew(1);
+    $physical_addr->usr($user->id);
+    $physical_addr->address_type('PHYSICAL');
+    $physical_addr->within_city_limits('f');
+
+    my $mailing_addr = Fieldmapper::actor::user_address->new;
+    $mailing_addr->isnew(1);
+    $mailing_addr->usr($user->id);
+    $mailing_addr->address_type('MAILING');
+    $mailing_addr->within_city_limits('f');
+
+    for my $type ($mailing_addr, $physical_addr) {
+        $type->street1($student->{address_street1});
+        $type->street2($student->{address_street2});
+        $type->city($student->{address_city});
+        $type->county($student->{address_county});
+        $type->state($student->{address_state});
+        $type->post_code($student->{address_postal});
+        $type->country('USA');
+    }
+
+    # Use as both billing and mailing via virtual ID.
+    $physical_addr->id(-1);
+    $mailing_addr->id(-1);
+    $user->billing_address(-1);
+    $user->mailing_address(-1);
+
+    $user->billing_address($physical_addr);
+    $user->mailing_address($mailing_addr);
+    $user->addresses([$physical_addr, $mailing_addr]);
+
+    return 1;
+}
+
+
+sub add_card {
+    my ($student, $user, $prefix) = @_;
+    my $id = $student->{student_id};
+    my $barcode = "$prefix"."$id";
+
+    my $card = Fieldmapper::actor::card->new;
+    $card->id(-1);
+    $card->isnew(1);
+    $card->usr($user->id);
+    $card->barcode($barcode);
+
+    # username defaults to barcode
+    $user->usrname($barcode);
+    $user->card($card);
+    $user->cards([$card]);
+
+    return 1;
+}
+
+
+sub do_login {
+    my $auth = $U->simplereq(
+    'open-ils.auth_internal',
+    'open-ils.auth_internal.session.create',
+    {user_id => 1, org_unit => 394, login_type => 'temp'}
+    );
+    
+    return unless $auth && $auth->{textcode} eq 'SUCCESS';
+
+    my $authtoken = $auth->{payload}->{authtoken};
+    
+    return $authtoken;
+}
+
+
+sub user_exists {
+    my ($student) = @_;
+    my $lname = $student->{lname};
+    my $ident = $student->{student_id};
+    my $user = $editor->search_actor_user({
+        family_name => $lname,
+        ident_value => $ident
+    });
+    return $user if $user;
+    return 0;
+
+}
+
+
+sub save_user {
+    my ($user, $authtoken) = @_;
+
+    print Dumper($user);
+    print $authtoken;
+    
+    my $save = $U->simplereq(
+        'open-ils.actor',
+        'open-ils.actor.patron.update',
+        $authtoken, $user
+    );
+
+    if ($U->is_event($save)) {
+        my $msg = "Unable to save account: " . $save->{textcode};
+        $logger->error("StudentCard: $msg");
+        return 0;
+    }
+    return 1;
+}
+
+
+sub add_exception {
+    my ($student, $error) = @_;
+    $student->{error} = $error;
+    push (@exceptions, $student);
+}
+
+# We've built everything, now get to work!
+
+my $districts = get_active_districts;
+my $authtoken = do_login;
+die "No authtoken available, cannot continue\n" unless $authtoken;
+
+# for each active district,
+for my $district(@$districts) {
+    my $schools = get_schools($district);
+    my $district_name = $district->name;
+    my $district_code = $district->code;
+    
+    my $retrieved_files = get_files($district);
+
+    if (scalar(@$retrieved_files) > 0) {
+        $logger->info("StudentCard: Copying files from $district_name");
+    } else {
+        $logger->info("StudentCard: No new files for $district_name");
+        next;
+    }
+
+    # for each retrieved file for the current district,
+    for my $csv_file (@$retrieved_files) {
+
+        #import the file
+        $logger->info("Importing $csv_file for $district_name");
+        my $student_entries = import_csv($csv_file);
+    
+        #for each student row in the current file,
+        for my $student (@$student_entries) {
+            next unless check_required_fields($student);
+            #my $existing_user = user_exists($student, $district_code); 
+            #$logger->info("CSHARP: \$existing_user = " . Dumper($existing_user));
+            my $school = lookup_school($schools, $student->{school_id});
+            add_exception($student, "No school found with State ID $student->{school_id}") && next unless $school;
+            my $user = make_user($student, $school, $district_code);
+            add_addresses($student, $user);
+            save_user($user, $authtoken);
+            add_card($student, $user, $district_code);
+            save_user($user, $authtoken); # not sure this is right...
+        } # student
+    } # csv_file
+} # district