From 30c3f0afab3239b102a8c28d78489ac1f91d9912 Mon Sep 17 00:00:00 2001 From: gfawcett Date: Thu, 15 Jul 2010 00:54:53 +0000 Subject: [PATCH] changing 'published' to a char field; storing marc record in item. Breaking schema change. Unicode in Z39.50 results is currently broken. git-svn-id: svn://svn.open-ils.org/ILS-Contrib/servres/trunk@914 6d9bc8c9-1ec2-4278-b937-99fde70a366f --- conifer/custom/lib_integration.py | 4 +- conifer/libsystems/z3950/marcxml.py | 48 +++-- conifer/libsystems/z3950/pyz3950_search.py | 5 +- conifer/syrup/migrations/0001_initial.py | 128 ++++++------- ..._add_field_site_course__add_unique_site_owne.py | 205 --------------------- conifer/syrup/models.py | 16 +- conifer/syrup/views/items.py | 17 +- conifer/templates/item/item_add_cat_search.xhtml | 15 +- conifer/templates/item/item_add_url.xhtml | 18 ++ conifer/templates/item/item_metadata.xhtml | 25 +-- 10 files changed, 159 insertions(+), 322 deletions(-) delete mode 100644 conifer/syrup/migrations/0002_auto__add_field_site_term__add_field_site_course__add_unique_site_owne.py diff --git a/conifer/custom/lib_integration.py b/conifer/custom/lib_integration.py index ebe08bf..50686ee 100644 --- a/conifer/custom/lib_integration.py +++ b/conifer/custom/lib_integration.py @@ -50,7 +50,7 @@ from conifer.libsystems.evergreen import item_status as I from conifer.libsystems.sip.sipclient import SIP #from conifer.libsystems.z3950 import yaz_search from conifer.libsystems.z3950 import pyz3950_search -from conifer.libsystems.z3950.marcxml import marcxml_to_dictionary +from conifer.libsystems.z3950.marcxml import marcxml_to_records @caching('patroninfo', timeout=300) @@ -89,7 +89,7 @@ def cat_search(query, start=1, limit=10): # this is a total hack for conifer. If the query is a Conifer # title-detail URL, then return just that one item. if query.startswith(EG_BASE): - results = marcxml_to_dictionary(I.url_to_marcxml(query), multiples=True) + results = marcxml_to_records(I.url_to_marcxml(query)) numhits = len(results) else: cat_host, cat_port, cat_db = settings.Z3950_CONFIG diff --git a/conifer/libsystems/z3950/marcxml.py b/conifer/libsystems/z3950/marcxml.py index 6e9930d..aaf466c 100644 --- a/conifer/libsystems/z3950/marcxml.py +++ b/conifer/libsystems/z3950/marcxml.py @@ -1,5 +1,28 @@ from xml.etree import ElementTree +def marcxml_to_records(rec): + tree = ElementTree.fromstring(rec) + if tree.tag == '{http://www.loc.gov/MARC21/slim}collection': + # then we may have multiple records + records = tree.findall('{http://www.loc.gov/MARC21/slim}record') + elif tree.tag == '{http://www.loc.gov/MARC21/slim}record': + records = [tree] + else: + return [] + return records + +def record_to_dictionary(record, multiples=True): + tree = ElementTree.fromstring(record) + dct = {} + for df in tree.findall('{http://www.loc.gov/MARC21/slim}datafield'): + t = df.attrib['tag'] + for sf in df.findall('{http://www.loc.gov/MARC21/slim}subfield'): + c = sf.attrib['code'] + v = sf.text or '' + dct.setdefault(t+c, []).append(v) + dct = dict((k,'\n'.join(v or [])) for k,v in dct.items()) + return dct + def marcxml_to_dictionary(rec, multiples=False): tree = ElementTree.fromstring(rec) if tree.tag == '{http://www.loc.gov/MARC21/slim}collection': @@ -30,23 +53,26 @@ def marcxml_dictionary_to_dc(dct): extract some Dublin Core elements from it. Fixme, I'm sure this could be way improved.""" out = {} - meta = [('245a', 'dc:title'), ('100a', 'dc:creator'), ('260b', 'dc:publisher'), + meta = [('245a', 'dc:title'), ('100a', 'dc:creator'), ('260c', 'dc:date'), ('700a', 'dc:contributor')] for marc, dc in meta: value = dct.get(marc) if value: out[dc] = value - title = out.get('dc:title') + + pub = [v.strip() for k,v in sorted(dct.items()) if k.startswith('260')] + if pub: + out['dc:publisher'] = strip_punct(' '.join(pub)) + + title = [v.strip() for k,v in sorted(dct.items()) if k in ('245a', '245b')] if title: - if '245b' in dct: - title += (' %s' % dct['245b']) - # if title ends with a single character, strip it. usually a - # spurious punctuation character. - if ' ' in title: - init, last = title.rsplit(' ',1) - if len(last) == 1: - title = init - out['dc:title'] = title + out['dc:title'] = strip_punct(' '.join(title)) return out +def strip_punct(s): + # strip whitespace and trailing single punctuation characters + s = s.strip() + if s and s[-1] in ',.;:/': + s = s[:-1] + return s.strip() diff --git a/conifer/libsystems/z3950/pyz3950_search.py b/conifer/libsystems/z3950/pyz3950_search.py index 09a951f..b111f46 100644 --- a/conifer/libsystems/z3950/pyz3950_search.py +++ b/conifer/libsystems/z3950/pyz3950_search.py @@ -74,16 +74,15 @@ def search(host, port, database, query, start=1, limit=10): for rec in raw_records: try: rec = _marc_utf8_pattern.sub(_decode_marc_utf8, rec) - dct = marcxml_to_dictionary(rec) except 'x': raise rec - parsed.append(dct) + parsed.append(rec) return parsed, len(res) # decoding MARC \X.. UTF-8 patterns. -_marc_utf8_pattern = re.compile(r'\\X([0-9A-F]{2})') +_marc_utf8_pattern = re.compile(r'\\X([0-9A-F]{2})', re.I) def _decode_marc_utf8(regex_match): return chr(int(regex_match.group(1), 16)) diff --git a/conifer/syrup/migrations/0001_initial.py b/conifer/syrup/migrations/0001_initial.py index 2c46d4b..bf807c7 100644 --- a/conifer/syrup/migrations/0001_initial.py +++ b/conifer/syrup/migrations/0001_initial.py @@ -5,132 +5,121 @@ from south.v2 import SchemaMigration from django.db import models class Migration(SchemaMigration): - + def forwards(self, orm): # Adding model 'UserProfile' db.create_table('syrup_userprofile', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('last_email_notice', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, null=True, blank=True)), ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('wants_email_notices', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True)), ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), ('ils_userid', self.gf('django.db.models.fields.CharField')(max_length=50, null=True, blank=True)), - ('wants_email_notices', self.gf('django.db.models.fields.BooleanField')(default=False, blank=True)), - ('last_email_notice', self.gf('django.db.models.fields.DateTimeField')(default=datetime.datetime.now, null=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['UserProfile']) # Adding model 'ServiceDesk' db.create_table('syrup_servicedesk', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=100)), ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), - ('name', self.gf('django.db.models.fields.CharField')(max_length=100)), ('active', self.gf('django.db.models.fields.BooleanField')(default=True, blank=True)), ('external_id', self.gf('django.db.models.fields.CharField')(max_length=256, null=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['ServiceDesk']) # Adding model 'Term' db.create_table('syrup_term', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('code', self.gf('django.db.models.fields.CharField')(max_length=64)), ('name', self.gf('django.db.models.fields.CharField')(max_length=256)), - ('start', self.gf('django.db.models.fields.DateField')()), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), ('finish', self.gf('django.db.models.fields.DateField')()), + ('start', self.gf('django.db.models.fields.DateField')()), + ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['Term']) # Adding model 'Department' db.create_table('syrup_department', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('service_desk', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.ServiceDesk'])), + ('name', self.gf('django.db.models.fields.CharField')(max_length=256)), ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), - ('name', self.gf('django.db.models.fields.CharField')(max_length=256)), ('active', self.gf('django.db.models.fields.BooleanField')(default=True, blank=True)), - ('service_desk', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.ServiceDesk'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['Department']) # Adding model 'Course' db.create_table('syrup_course', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('code', self.gf('django.db.models.fields.CharField')(max_length=64)), ('name', self.gf('django.db.models.fields.CharField')(max_length=1024)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('department', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Department'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['Course']) # Adding model 'Z3950Target' db.create_table('syrup_z3950target', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), ('name', self.gf('django.db.models.fields.CharField')(max_length=100)), - ('host', self.gf('django.db.models.fields.CharField')(max_length=50)), ('database', self.gf('django.db.models.fields.CharField')(max_length=50)), - ('port', self.gf('django.db.models.fields.IntegerField')(default=210)), ('syntax', self.gf('django.db.models.fields.CharField')(default='USMARC', max_length=10)), ('active', self.gf('django.db.models.fields.BooleanField')(default=True, blank=True)), + ('host', self.gf('django.db.models.fields.CharField')(max_length=50)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('port', self.gf('django.db.models.fields.IntegerField')(default=210)), )) db.send_create_signal('syrup', ['Z3950Target']) # Adding model 'Config' db.create_table('syrup_config', ( ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('name', self.gf('django.db.models.fields.CharField')(max_length=256)), ('value', self.gf('django.db.models.fields.CharField')(max_length=8192)), + ('name', self.gf('django.db.models.fields.CharField')(max_length=256)), )) db.send_create_signal('syrup', ['Config']) # Adding model 'Site' db.create_table('syrup_site', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('term', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Term'])), + ('service_desk', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.ServiceDesk'])), ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), + ('passkey', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=256, null=True, blank=True)), + ('access', self.gf('django.db.models.fields.CharField')(default='CLOSE', max_length=5)), + ('course', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Course'])), ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('owner', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), - ('service_desk', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.ServiceDesk'])), - ('access', self.gf('django.db.models.fields.CharField')(default='CLOSE', max_length=5)), - ('passkey', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=256, null=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['Site']) - # Adding M2M table for field courses on 'Site' - db.create_table('syrup_site_courses', ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('site', models.ForeignKey(orm['syrup.site'], null=False)), - ('course', models.ForeignKey(orm['syrup.course'], null=False)) - )) - db.create_unique('syrup_site_courses', ['site_id', 'course_id']) - - # Adding M2M table for field terms on 'Site' - db.create_table('syrup_site_terms', ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('site', models.ForeignKey(orm['syrup.site'], null=False)), - ('term', models.ForeignKey(orm['syrup.term'], null=False)) - )) - db.create_unique('syrup_site_terms', ['site_id', 'term_id']) + # Adding unique constraint on 'Site', fields ['course', 'term', 'owner'] + db.create_unique('syrup_site', ['course_id', 'term_id', 'owner_id']) # Adding model 'Group' db.create_table('syrup_group', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('site', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Site'])), + ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), ('external_id', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=2048, null=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), )) db.send_create_signal('syrup', ['Group']) # Adding model 'Membership' db.create_table('syrup_membership', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('group', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Group'])), ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), - ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), - ('group', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Group'])), ('role', self.gf('django.db.models.fields.CharField')(default='STUDT', max_length=6)), + ('user', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['auth.User'])), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), )) db.send_create_signal('syrup', ['Membership']) @@ -139,26 +128,26 @@ class Migration(SchemaMigration): # Adding model 'Item' db.create_table('syrup_item', ( - ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), - ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), - ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), - ('site', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Site'])), - ('item_type', self.gf('django.db.models.fields.CharField')(max_length=7)), - ('bib_id', self.gf('django.db.models.fields.CharField')(max_length=256, null=True, blank=True)), - ('marcxml', self.gf('django.db.models.fields.TextField')(null=True, blank=True)), - ('title', self.gf('django.db.models.fields.CharField')(max_length=8192, db_index=True)), - ('author', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=8192, null=True, blank=True)), + ('parent_heading', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Item'], null=True, blank=True)), ('publisher', self.gf('django.db.models.fields.CharField')(max_length=8192, null=True, blank=True)), - ('published', self.gf('django.db.models.fields.DateField')(null=True, blank=True)), ('itemtype', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=1, null=True, blank=True)), - ('parent_heading', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Item'], null=True, blank=True)), + ('marcxml', self.gf('django.db.models.fields.TextField')(null=True, blank=True)), + ('author', self.gf('django.db.models.fields.CharField')(db_index=True, max_length=8192, null=True, blank=True)), + ('created', self.gf('django.db.models.fields.DateTimeField')(auto_now_add=True, blank=True)), ('url', self.gf('django.db.models.fields.URLField')(max_length=200, null=True, blank=True)), - ('fileobj', self.gf('django.db.models.fields.files.FileField')(default=None, max_length=255, null=True, blank=True)), + ('title', self.gf('django.db.models.fields.CharField')(max_length=8192, db_index=True)), ('fileobj_mimetype', self.gf('django.db.models.fields.CharField')(max_length=128, null=True, blank=True)), + ('site', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['syrup.Site'])), + ('item_type', self.gf('django.db.models.fields.CharField')(max_length=7)), + ('last_modified', self.gf('django.db.models.fields.DateTimeField')(auto_now=True, blank=True)), + ('fileobj', self.gf('django.db.models.fields.files.FileField')(default=None, max_length=255, null=True, blank=True)), + ('published', self.gf('django.db.models.fields.CharField')(max_length=64, null=True, blank=True)), + ('id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('bib_id', self.gf('django.db.models.fields.CharField')(max_length=256, null=True, blank=True)), )) db.send_create_signal('syrup', ['Item']) - - + + def backwards(self, orm): # Deleting model 'UserProfile' @@ -185,11 +174,8 @@ class Migration(SchemaMigration): # Deleting model 'Site' db.delete_table('syrup_site') - # Removing M2M table for field courses on 'Site' - db.delete_table('syrup_site_courses') - - # Removing M2M table for field terms on 'Site' - db.delete_table('syrup_site_terms') + # Removing unique constraint on 'Site', fields ['course', 'term', 'owner'] + db.delete_unique('syrup_site', ['course_id', 'term_id', 'owner_id']) # Deleting model 'Group' db.delete_table('syrup_group') @@ -202,8 +188,8 @@ class Migration(SchemaMigration): # Deleting model 'Item' db.delete_table('syrup_item') - - + + models = { 'auth.group': { 'Meta': {'object_name': 'Group'}, @@ -286,7 +272,7 @@ class Migration(SchemaMigration): 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), 'marcxml': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), 'parent_heading': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Item']", 'null': 'True', 'blank': 'True'}), - 'published': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), + 'published': ('django.db.models.fields.CharField', [], {'max_length': '64', 'null': 'True', 'blank': 'True'}), 'publisher': ('django.db.models.fields.CharField', [], {'max_length': '8192', 'null': 'True', 'blank': 'True'}), 'site': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Site']"}), 'title': ('django.db.models.fields.CharField', [], {'max_length': '8192', 'db_index': 'True'}), @@ -311,16 +297,16 @@ class Migration(SchemaMigration): 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) }, 'syrup.site': { - 'Meta': {'object_name': 'Site'}, + 'Meta': {'unique_together': "(('course', 'term', 'owner'),)", 'object_name': 'Site'}, 'access': ('django.db.models.fields.CharField', [], {'default': "'CLOSE'", 'max_length': '5'}), - 'courses': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['syrup.Course']"}), + 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Course']"}), 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), 'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), 'passkey': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '256', 'null': 'True', 'blank': 'True'}), 'service_desk': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.ServiceDesk']"}), - 'terms': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['syrup.Term']"}) + 'term': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Term']"}) }, 'syrup.term': { 'Meta': {'object_name': 'Term'}, @@ -353,5 +339,5 @@ class Migration(SchemaMigration): 'syntax': ('django.db.models.fields.CharField', [], {'default': "'USMARC'", 'max_length': '10'}) } } - + complete_apps = ['syrup'] diff --git a/conifer/syrup/migrations/0002_auto__add_field_site_term__add_field_site_course__add_unique_site_owne.py b/conifer/syrup/migrations/0002_auto__add_field_site_term__add_field_site_course__add_unique_site_owne.py deleted file mode 100644 index edebbd6..0000000 --- a/conifer/syrup/migrations/0002_auto__add_field_site_term__add_field_site_course__add_unique_site_owne.py +++ /dev/null @@ -1,205 +0,0 @@ -# encoding: utf-8 -import datetime -from south.db import db -from south.v2 import SchemaMigration -from django.db import models - -class Migration(SchemaMigration): - - def forwards(self, orm): - - # Adding field 'Site.term' - db.add_column('syrup_site', 'term', self.gf('django.db.models.fields.related.ForeignKey')(default=1, to=orm['syrup.Term']), keep_default=False) - - # Adding field 'Site.course' - db.add_column('syrup_site', 'course', self.gf('django.db.models.fields.related.ForeignKey')(default=1, to=orm['syrup.Course']), keep_default=False) - - # Removing M2M table for field courses on 'Site' - db.delete_table('syrup_site_courses') - - # Removing M2M table for field terms on 'Site' - db.delete_table('syrup_site_terms') - - # Adding unique constraint on 'Site', fields ['owner', 'course', 'term'] - db.create_unique('syrup_site', ['owner_id', 'course_id', 'term_id']) - - - def backwards(self, orm): - - # Deleting field 'Site.term' - db.delete_column('syrup_site', 'term_id') - - # Deleting field 'Site.course' - db.delete_column('syrup_site', 'course_id') - - # Adding M2M table for field courses on 'Site' - db.create_table('syrup_site_courses', ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('site', models.ForeignKey(orm['syrup.site'], null=False)), - ('course', models.ForeignKey(orm['syrup.course'], null=False)) - )) - db.create_unique('syrup_site_courses', ['site_id', 'course_id']) - - # Adding M2M table for field terms on 'Site' - db.create_table('syrup_site_terms', ( - ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)), - ('site', models.ForeignKey(orm['syrup.site'], null=False)), - ('term', models.ForeignKey(orm['syrup.term'], null=False)) - )) - db.create_unique('syrup_site_terms', ['site_id', 'term_id']) - - # Removing unique constraint on 'Site', fields ['owner', 'course', 'term'] - db.delete_unique('syrup_site', ['owner_id', 'course_id', 'term_id']) - - - models = { - 'auth.group': { - 'Meta': {'object_name': 'Group'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '80'}), - 'permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}) - }, - 'auth.permission': { - 'Meta': {'unique_together': "(('content_type', 'codename'),)", 'object_name': 'Permission'}, - 'codename': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'content_type': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['contenttypes.ContentType']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '50'}) - }, - 'auth.user': { - 'Meta': {'object_name': 'User'}, - 'date_joined': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'email': ('django.db.models.fields.EmailField', [], {'max_length': '75', 'blank': 'True'}), - 'first_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'groups': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Group']", 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'is_active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'is_staff': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'is_superuser': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}), - 'last_login': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now'}), - 'last_name': ('django.db.models.fields.CharField', [], {'max_length': '30', 'blank': 'True'}), - 'password': ('django.db.models.fields.CharField', [], {'max_length': '128'}), - 'user_permissions': ('django.db.models.fields.related.ManyToManyField', [], {'to': "orm['auth.Permission']", 'blank': 'True'}), - 'username': ('django.db.models.fields.CharField', [], {'unique': 'True', 'max_length': '30'}) - }, - 'contenttypes.contenttype': { - 'Meta': {'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, - 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - 'syrup.config': { - 'Meta': {'object_name': 'Config'}, - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), - 'value': ('django.db.models.fields.CharField', [], {'max_length': '8192'}) - }, - 'syrup.course': { - 'Meta': {'object_name': 'Course'}, - 'code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'department': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Department']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '1024'}) - }, - 'syrup.department': { - 'Meta': {'object_name': 'Department'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), - 'service_desk': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.ServiceDesk']"}) - }, - 'syrup.group': { - 'Meta': {'object_name': 'Group'}, - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'external_id': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '2048', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'site': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Site']"}) - }, - 'syrup.item': { - 'Meta': {'object_name': 'Item'}, - 'author': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '8192', 'null': 'True', 'blank': 'True'}), - 'bib_id': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'fileobj': ('django.db.models.fields.files.FileField', [], {'default': 'None', 'max_length': '255', 'null': 'True', 'blank': 'True'}), - 'fileobj_mimetype': ('django.db.models.fields.CharField', [], {'max_length': '128', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'item_type': ('django.db.models.fields.CharField', [], {'max_length': '7'}), - 'itemtype': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '1', 'null': 'True', 'blank': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'marcxml': ('django.db.models.fields.TextField', [], {'null': 'True', 'blank': 'True'}), - 'parent_heading': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Item']", 'null': 'True', 'blank': 'True'}), - 'published': ('django.db.models.fields.DateField', [], {'null': 'True', 'blank': 'True'}), - 'publisher': ('django.db.models.fields.CharField', [], {'max_length': '8192', 'null': 'True', 'blank': 'True'}), - 'site': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Site']"}), - 'title': ('django.db.models.fields.CharField', [], {'max_length': '8192', 'db_index': 'True'}), - 'url': ('django.db.models.fields.URLField', [], {'max_length': '200', 'null': 'True', 'blank': 'True'}) - }, - 'syrup.membership': { - 'Meta': {'unique_together': "(('group', 'user'),)", 'object_name': 'Membership'}, - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'group': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Group']"}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'role': ('django.db.models.fields.CharField', [], {'default': "'STUDT'", 'max_length': '6'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}) - }, - 'syrup.servicedesk': { - 'Meta': {'object_name': 'ServiceDesk'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'external_id': ('django.db.models.fields.CharField', [], {'max_length': '256', 'null': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) - }, - 'syrup.site': { - 'Meta': {'unique_together': "(('course', 'term', 'owner'),)", 'object_name': 'Site'}, - 'access': ('django.db.models.fields.CharField', [], {'default': "'CLOSE'", 'max_length': '5'}), - 'course': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Course']"}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'owner': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), - 'passkey': ('django.db.models.fields.CharField', [], {'db_index': 'True', 'max_length': '256', 'null': 'True', 'blank': 'True'}), - 'service_desk': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.ServiceDesk']"}), - 'term': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['syrup.Term']"}) - }, - 'syrup.term': { - 'Meta': {'object_name': 'Term'}, - 'code': ('django.db.models.fields.CharField', [], {'max_length': '64'}), - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'finish': ('django.db.models.fields.DateField', [], {}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '256'}), - 'start': ('django.db.models.fields.DateField', [], {}) - }, - 'syrup.userprofile': { - 'Meta': {'object_name': 'UserProfile'}, - 'created': ('django.db.models.fields.DateTimeField', [], {'auto_now_add': 'True', 'blank': 'True'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'ils_userid': ('django.db.models.fields.CharField', [], {'max_length': '50', 'null': 'True', 'blank': 'True'}), - 'last_email_notice': ('django.db.models.fields.DateTimeField', [], {'default': 'datetime.datetime.now', 'null': 'True', 'blank': 'True'}), - 'last_modified': ('django.db.models.fields.DateTimeField', [], {'auto_now': 'True', 'blank': 'True'}), - 'user': ('django.db.models.fields.related.ForeignKey', [], {'to': "orm['auth.User']"}), - 'wants_email_notices': ('django.db.models.fields.BooleanField', [], {'default': 'False', 'blank': 'True'}) - }, - 'syrup.z3950target': { - 'Meta': {'object_name': 'Z3950Target'}, - 'active': ('django.db.models.fields.BooleanField', [], {'default': 'True', 'blank': 'True'}), - 'database': ('django.db.models.fields.CharField', [], {'max_length': '50'}), - 'host': ('django.db.models.fields.CharField', [], {'max_length': '50'}), - 'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), - 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}), - 'port': ('django.db.models.fields.IntegerField', [], {'default': '210'}), - 'syntax': ('django.db.models.fields.CharField', [], {'default': "'USMARC'", 'max_length': '10'}) - } - } - - complete_apps = ['syrup'] diff --git a/conifer/syrup/models.py b/conifer/syrup/models.py index 7473633..abed3a8 100644 --- a/conifer/syrup/models.py +++ b/conifer/syrup/models.py @@ -14,6 +14,9 @@ from django.conf import settings campus = settings.CAMPUS_INTEGRATION # TODO: fixme, not sure if conifer.custom is a good parent. from conifer.custom import lib_integration +from conifer.libsystems.z3950.marcxml import record_to_dictionary +from conifer.libsystems.z3950.marcxml import marcxml_dictionary_to_dc +from django.utils import simplejson as json #---------------------------------------------------------------------- @@ -405,7 +408,7 @@ class Item(BaseModel): # publisher: "Place: Publisher", as in a bibliography, for display. publisher = m.CharField(max_length=8192, null=True, blank=True) - published = m.DateField(null=True, blank=True) + published = m.CharField(max_length=64, null=True, blank=True) ITEMTYPE_CHOICES = [ # From http://www.oclc.org/bibformats/en/fixedfield/type.shtm. @@ -447,6 +450,15 @@ class Item(BaseModel): fileobj_mimetype = m.CharField(max_length=128, blank=True, null=True) + #-------------------------------------------------- + # MARC + def marc_as_dict(self): + return record_to_dictionary(self.marcxml) + + def marc_dc_subset(self): + return json.dumps(self.marc_as_dict()) + + #-------------------------------------------------- def __unicode__(self): return self.title @@ -495,7 +507,7 @@ class Item(BaseModel): and a friendly description of the physical item's status""" # TODO: this needs to be reimplemented, based on copy detail # lookup in the ILS. It also may not belong here! - raise NotImplementedError + return (True, 'NOT-IMPLEMENTED') # TODO: stuff I'm not sure about yet. I don't think it belongs here. diff --git a/conifer/syrup/views/items.py b/conifer/syrup/views/items.py index e7b2d2e..148f390 100644 --- a/conifer/syrup/views/items.py +++ b/conifer/syrup/views/items.py @@ -1,5 +1,6 @@ from _common import * from django.utils.translation import ugettext as _ +from xml.etree import ElementTree as E @members_only def item_detail(request, site_id, item_id): @@ -170,9 +171,13 @@ def item_add_cat_search(request, site_id, item_id): if not site.can_edit(request.user): return _access_denied(_('You are not an editor.')) - pickitem = simplejson.loads(raw_pickitem) + pickitem = marcxml_to_dictionary(raw_pickitem) dublin = marcxml_dictionary_to_dc(pickitem) + assert dublin + + #TODO: this data munging does not belong here. + # one last thing. If this picked item has an 856$9 field, then # it's an electronic resource, not a physical item. In that # case, we add it as a URL, not a PHYS. @@ -180,19 +185,21 @@ def item_add_cat_search(request, site_id, item_id): dct = dict(item_type='URL', url=pickitem.get('856u')) else: dct = dict(item_type='PHYS') - + try: pubdate = dublin.get('dc:date') - pubdate = re.search('^([0-9]+)', pubdate).group(1) - pubdate = '%d-01-01' % int(pubdate) + m = re.search('([0-9]+)', pubdate) + if m: + pubdate = pubdate.group(1) except: - pubdate = None + pubdate = '' item = site.item_set.create(parent_heading=parent_item, title=dublin.get('dc:title','Untitled'), author=dublin.get('dc:creator'), publisher=dublin.get('dc:publisher',''), published=pubdate, + marcxml=raw_pickitem, **dct) item.save() return HttpResponseRedirect('../../../%d/meta' % item.id) diff --git a/conifer/templates/item/item_add_cat_search.xhtml b/conifer/templates/item/item_add_cat_search.xhtml index a6d502a..fa8ace3 100644 --- a/conifer/templates/item/item_add_cat_search.xhtml +++ b/conifer/templates/item/item_add_cat_search.xhtml @@ -1,5 +1,7 @@ #TitleAuthorPublisherPubDate + py:with="dct=record_to_dictionary(res); dc=to_dublin(dct)"> ${resultnum+start}. ${dc.get('dc:title', '???')} details -

- Electronic resource. view +

+ Electronic resource. view

${dc.get(k) or '—'}
- + +
@@ -75,9 +78,9 @@ dc_keys = ['dc:title', 'dc:creator', 'dc:publisher', 'dc:date'] - + - +
${k}${res[k]}${k}${dct[k]}
diff --git a/conifer/templates/item/item_add_url.xhtml b/conifer/templates/item/item_add_url.xhtml index cbb8ec7..9ff5023 100644 --- a/conifer/templates/item/item_add_url.xhtml +++ b/conifer/templates/item/item_add_url.xhtml @@ -23,7 +23,25 @@ site.title = '%s: %s (%s)' % (site.course.code, site.course.name, site.term) + + +
Title
URL
Author
Publisher
Published
+
+

Display MARC Record

+ +

diff --git a/conifer/templates/item/item_metadata.xhtml b/conifer/templates/item/item_metadata.xhtml index 8e2f00a..a90986e 100644 --- a/conifer/templates/item/item_metadata.xhtml +++ b/conifer/templates/item/item_metadata.xhtml @@ -33,6 +33,9 @@ is_editor = site.can_edit(request.user) + + + @@ -48,29 +51,17 @@ is_editor = site.can_edit(request.user)
Title${item.title}
Author${item.author}
Publisher${item.publisher}
Published${item.published and item.published.year}
Type${item.get_item_type_display()}
Download
-

TODO: get metadata back...

-
-

Additional metadata

- - - - - - -
${attr.get_name_display()} - ${attr.value} - - +
+

Display MARC Record

-
-
+ -- 2.11.0