Progress towards reality - generating staging table inserts

Signed-off-by: Dan Scott <dscott@laurentian.ca>

diff --git a/tools/patron-load/ldap_sync b/tools/patron-load/ldap_sync
index aca97de..aa23dd3 100644 (file)
--- a/tools/patron-load/ldap_sync
+++ b/tools/patron-load/ldap_sync
@@ -9,7 +9,7 @@ imported to avoid storing credentials in the VCS.
 Rough plan:
 
 1. Create new accounts
-  a. Pull new LDAP records since a given time from the LDAP directory
+  a. Pull new LDAP records since a 2011-07--1 from the LDAP directory
      using the filter (createTimestamp>=$time) and insert into a
      staging table with the following columns; included is a sample
      mapping to the LU LDAP attributes:
@@ -20,6 +20,7 @@ Rough plan:
      * language (preferredLanguage)
      * profile (lulPrimaryAffiliation)
      * datatel_barcode (datatel_to_barcode(lulColleagueId)
+     * student_type (???)
   b. For each LDAP record, create a new library system account if it
      does not already exist (check for matches based on usrname, email
      address, datatel_barcode). Map LDAP attributes to account profile,
@@ -61,6 +62,21 @@ def datatel_to_barcode(datatel):
 
     return barcode
 
+def barcode_to_datatel(barcode):
+    """
+    Converts a barcode into a Datatel Colleague ID
+
+    Used to generate the ident_value for legacy users.
+
+    >>> barcode_to_datatel('00007001049233')
+    '0104923'
+    """
+
+    if len(barcode) != 14:
+        return false
+
+    return barcode[6:13]
+
 def mod10_checksum(barcode):
     """
     Calculates the mod10 checksum for a given string of digits
@@ -86,6 +102,26 @@ def mod10_checksum(barcode):
         return 10 - rem
     return rem
 
+def database_barcode_to_datatel():
+    """
+    Define a PostgreSQL function for generating Colleague ID from barcode
+    """
+
+    print """CREATE OR REPLACE FUNCTION evergreen.barcode_to_datatel(TEXT) RETURNS TEXT AS $$
+    use strict;
+    use warnings;
+
+    my $barcode = shift;
+
+    if (length($barcode) != 14) {
+        return undef;
+    }
+
+    return substr($barcode, 6, 7);
+
+$$ LANGUAGE PLPERLU STRICT IMMUTABLE;
+"""
+
 def database_mod10():
     """
     Define a PostgreSQL function for generating mod10 check digits
@@ -129,14 +165,14 @@ def create_staging_table():
     print """
 DROP TABLE IF EXISTS scratchpad.usr_staging;
 CREATE TABLE scratchpad.usr_staging (usrname TEXT, family_name TEXT, first_given_name TEXT, ident_value TEXT, lang TEXT);
+COPY scratchpad.usr_staging (usrname, family_name, first_given_name, ident_value, lang) FROM STDIN;
 """
 
-def search_for_students(con):
+def search_for_students(con, attributes, create_date):
     base_dn = 'o=lul'
     search_scope = ldap.SCOPE_SUBTREE
-    attributes = ['lulPrimaryAffiliation', 'cn', 'mail', 'givenName', 'sn', 'lulColleagueId', 'preferredLanguage']
     filter = '(&(objectclass=lulEduPerson))'
-    filter = '(&(objectclass=lulEduPerson)(lulPrimaryAffiliation=*))'
+    filter = '(&(objectclass=lulEduPerson)(lulPrimaryAffiliation=*)(createTimestamp>=%s000000Z)' % create_date
 
     try:
         result_id = con.search(base_dn, search_scope, filter, attributes)
@@ -146,22 +182,49 @@ def search_for_students(con):
             if result_data == []:
                 break
             else:
-                print result_data[0][0]
-                for key in result_data[0][1]:
-                    print key, result_data[0][1][key]
+                # dump_data(result_data)
+                insert_into_staging(result_data)
     except ldap.LDAPError, e:
         print e
 
+def insert_into_staging(result_data):
+    """
+    Generate statements to push data into the staging table
+    """
+
+    usrname = result_data[0][1]['cn']
+    family_name = result_data[0][1]['sn']
+    given_name = result_data[0][1]['givenName']
+    datatel = result_data[0][1]['lulColleagueId']
+    lang = result_data[0][1]['preferredLanguage'] or r'\N'
+
+    print "%s\t%s\t%s\t%s\t%s" % (
+        usrname, family_name, given_name, datatel, lang
+    )
+
+def dump_data(result_data):
+    """
+    Simple dump of all data received
+    """
+
+    print()
+    print(result_data[0][0])
+    for key in result_data[0][1]:
+        print(key, result_data[0][1][key])
+
 if __name__ == '__main__':
     import doctest
     doctest.testmod()
+    exit()
 
+    create_date = '20110701'
     con = ldap.initialize(luauth.hostname)
     con.set_option(ldap.OPT_REFERRALS, 0)
 
     try:
+        attributes = ['lulPrimaryAffiliation', 'cn', 'mail', 'givenName', 'sn', 'lulColleagueId', 'preferredLanguage']
         con.simple_bind_s(luauth.dn, luauth.pw)
-        search_for_students(con)
+        search_for_students(con, attributes, create_date)
     except ldap.LDAPError, e:
         print "Could not connect: " + e.message['info']
         if type(e.message) == dict and e.message.has_key('desc'):