From 213496fa3ece2e05409da13531db00e43ec5114d Mon Sep 17 00:00:00 2001 From: michigg Date: Sun, 1 Apr 2018 17:35:55 +0200 Subject: [PATCH] Add logging, Refractor importer --- .../management/commands/delete_lectures.py | 13 ++++ .../donar/management/commands/delete_rooms.py | 13 ++++ .../migrations/0005_auto_20180401_1136.py | 18 +++++ .../migrations/0006_auto_20180401_1139.py | 28 ++++++++ .../migrations/0007_auto_20180401_1140.py | 18 +++++ .../migrations/0008_auto_20180401_1142.py | 28 ++++++++ .../migrations/0009_auto_20180401_1143.py | 33 +++++++++ ofu_app/apps/donar/models.py | 4 +- .../apps/donar/utils/migrate_data_lectures.py | 67 +++++++++++-------- .../apps/donar/utils/migrate_data_rooms.py | 27 ++++++-- ofu_app/apps/events/utils/migrate_data.py | 67 ++++++++----------- ofu_app/core/settings.py | 4 ++ 12 files changed, 247 insertions(+), 73 deletions(-) create mode 100644 ofu_app/apps/donar/management/commands/delete_lectures.py create mode 100644 ofu_app/apps/donar/management/commands/delete_rooms.py create mode 100644 ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py create mode 100644 ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py create mode 100644 ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py create mode 100644 ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py create mode 100644 ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py diff --git a/ofu_app/apps/donar/management/commands/delete_lectures.py b/ofu_app/apps/donar/management/commands/delete_lectures.py new file mode 100644 index 0000000..444ff9a --- /dev/null +++ b/ofu_app/apps/donar/management/commands/delete_lectures.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand, CommandError +from apps.donar.models import Room +from apps.donar.utils import migrate_data_lectures + + +class Command(BaseCommand): + help = "Imports Lectures from UnivIS PRG. Requires Room import" + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + migrate_data_lectures.delete() diff --git a/ofu_app/apps/donar/management/commands/delete_rooms.py b/ofu_app/apps/donar/management/commands/delete_rooms.py new file mode 100644 index 0000000..b26763a --- /dev/null +++ b/ofu_app/apps/donar/management/commands/delete_rooms.py @@ -0,0 +1,13 @@ +from django.core.management.base import BaseCommand, CommandError +from apps.donar.models import Room +from apps.donar.utils import migrate_data_rooms + + +class Command(BaseCommand): + help = "Imports Rooms from Univis PRG" + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + migrate_data_rooms.delete() diff --git a/ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py b/ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py new file mode 100644 index 0000000..5d72ed0 --- /dev/null +++ b/ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.1 on 2018-04-01 11:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('donar', '0004_auto_20180117_0137'), + ] + + operations = [ + migrations.AlterField( + model_name='lecture', + name='short', + field=models.CharField(max_length=256), + ), + ] diff --git a/ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py b/ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py new file mode 100644 index 0000000..38385a7 --- /dev/null +++ b/ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py @@ -0,0 +1,28 @@ +# Generated by Django 2.0.1 on 2018-04-01 11:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('donar', '0005_auto_20180401_1136'), + ] + + operations = [ + migrations.AlterField( + model_name='lecture', + name='lecturer_id', + field=models.CharField(max_length=512), + ), + migrations.AlterField( + model_name='lecture', + name='univis_id', + field=models.CharField(max_length=512, unique=True), + ), + migrations.AlterField( + model_name='lecture', + name='univis_ref', + field=models.CharField(max_length=512, unique=True), + ), + ] diff --git a/ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py b/ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py new file mode 100644 index 0000000..f8f72c7 --- /dev/null +++ b/ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py @@ -0,0 +1,18 @@ +# Generated by Django 2.0.1 on 2018-04-01 11:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('donar', '0006_auto_20180401_1139'), + ] + + operations = [ + migrations.AlterField( + model_name='lecture', + name='name', + field=models.CharField(max_length=512), + ), + ] diff --git a/ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py b/ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py new file mode 100644 index 0000000..f19cd75 --- /dev/null +++ b/ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py @@ -0,0 +1,28 @@ +# Generated by Django 2.0.1 on 2018-04-01 11:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('donar', '0007_auto_20180401_1140'), + ] + + operations = [ + migrations.AlterField( + model_name='lecture', + name='lecturer_id', + field=models.CharField(max_length=1024), + ), + migrations.AlterField( + model_name='lecture', + name='univis_id', + field=models.CharField(max_length=1024, unique=True), + ), + migrations.AlterField( + model_name='lecture', + name='univis_ref', + field=models.CharField(max_length=1024, unique=True), + ), + ] diff --git a/ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py b/ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py new file mode 100644 index 0000000..b65e3cd --- /dev/null +++ b/ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py @@ -0,0 +1,33 @@ +# Generated by Django 2.0.1 on 2018-04-01 11:43 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('donar', '0008_auto_20180401_1142'), + ] + + operations = [ + migrations.AlterField( + model_name='lecture', + name='lecturer_id', + field=models.CharField(max_length=256), + ), + migrations.AlterField( + model_name='lecture', + name='short', + field=models.CharField(max_length=512), + ), + migrations.AlterField( + model_name='lecture', + name='univis_id', + field=models.CharField(max_length=256, unique=True), + ), + migrations.AlterField( + model_name='lecture', + name='univis_ref', + field=models.CharField(max_length=256, unique=True), + ), + ] diff --git a/ofu_app/apps/donar/models.py b/ofu_app/apps/donar/models.py index 6450cc5..484e5da 100644 --- a/ofu_app/apps/donar/models.py +++ b/ofu_app/apps/donar/models.py @@ -11,8 +11,8 @@ MAX_COORDS_NAME_LENGTH = 256 MAX_COORDS_LENGTH = 256 MAX_LECTURE_IDS_LENGTH = 256 -MAX_LECTURE_SHORT_LENGTH = 128 -MAX_LECTURE_NAME_LENGTH = 256 +MAX_LECTURE_SHORT_LENGTH = 512 +MAX_LECTURE_NAME_LENGTH = 512 MAX_LECTURE_TYPE_LENGTH = 64 diff --git a/ofu_app/apps/donar/utils/migrate_data_lectures.py b/ofu_app/apps/donar/utils/migrate_data_lectures.py index 6b85e57..15fce8e 100644 --- a/ofu_app/apps/donar/utils/migrate_data_lectures.py +++ b/ofu_app/apps/donar/utils/migrate_data_lectures.py @@ -4,6 +4,9 @@ import json from pprint import pprint from django.db.utils import IntegrityError from apps.donar.utils.parser import univis_lectures_parser +import logging + +logger = logging.getLogger(__name__) # CONFIG Fakultaet FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften" @@ -32,38 +35,35 @@ def writeUnivisLectureTermsInDB(lecture, lecture_obj): if type(lecture['terms']['term']) == list: for term in lecture['terms']['term']: try: - term_obj = Lecture_Terms.objects.create() starttime = "00:00" + term_obj = Lecture_Terms.objects.create(starttime=starttime) if 'starttime' in term: starttime = term['starttime'] term_obj.starttime = datetime.strptime(starttime, "%H:%M") + term_obj.save() if 'room' in term: room_id = term['room']['UnivISRef']['@key'] - term_obj.room = [Room.objects.get(key=room_id)] - term_obj.save() + term_obj.room.add(Room.objects.get(key=room_id)) lecture_obj.term.add(term_obj) except IntegrityError as err: - print("ROOM_ID: " + str(room_id)) - print(err.args) + logger.exception(err) else: try: - term_obj = Lecture_Terms.objects.create() univis_starttime = "00:00" + term_obj = Lecture_Terms.objects.create(starttime=univis_starttime) if 'starttime' in lecture['terms']['term']: univis_starttime = lecture['terms']['term']['starttime'] term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M') + term_obj.save() if 'room' in lecture['terms']['term']: room_id = lecture['terms']['term']['room']['UnivISRef']['@key'] - pprint("Room: " + room_id) Room.objects.get(key=room_id) - term_obj.room = [Room.objects.get(key=room_id)] - + term_obj.room.add(Room.objects.get(key=room_id)) term_obj.save() lecture_obj.term.add(term_obj) except IntegrityError as err: - print("ROOM_ID: " + str(room_id)) - print(err.args) + logger.exception(err) def writeUnivisLectureDataInDB(data): @@ -91,43 +91,56 @@ def writeUnivisLectureDataInDB(data): lecture_type = lecture['type'] if 'dozs' in lecture: lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] - print("Lecture: " + name) lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, type=lecture_type, lecturer_id=lecturer_id) writeUnivisLectureTermsInDB(lecture, lecture_obj) lecture_obj.save() + logger.info("Lecture: {}".format(lecture_obj.short)) except IntegrityError as err: - print(err.args) + logger.warning('Lecture already exists') + # logger.exception(err) return def showStatus(status: str): - print(status) - pprint("Lectures: " + str(Lecture.objects.count())) - pprint("Lecture Terms: " + str(Lecture_Terms.objects.count())) - pprint("Room: " + str(Room.objects.count())) + return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Terms: {lecture_terms}\n\tRoom: {room}".format( + status=status, + lectures=Lecture.objects.count(), + lecture_terms=Lecture_Terms.objects.count(), + room=Room.objects.count() + ) + + +def delete(): + lectures = Lecture.objects.all() + logger.info("Deleted following Lectures:") + for lecture in lectures: + logger.info("Lecture: {name}".format( + name=lecture.name) + ) + lecture.delete() def main(): # get food jsons - showStatus("Start with:") + logger.info(showStatus("Start SoWi:")) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi))) - pprint("----------------------------------------------------------------------------------------") + # pprint("----------------------------------------------------------------------------------------") - showStatus("After SoWi:") + logger.info(showStatus("Start GuK:")) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK))) - pprint("----------------------------------------------------------------------------------------") + # pprint("----------------------------------------------------------------------------------------") - showStatus("After GuK:") + logger.info(showStatus("Start HuWi:")) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi))) - pprint("----------------------------------------------------------------------------------------") + # pprint("----------------------------------------------------------------------------------------") - showStatus("After HuWi:") + logger.info(showStatus("Start WIAI:")) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI))) - pprint("----------------------------------------------------------------------------------------") - - showStatus("After WIAI:") + # pprint("----------------------------------------------------------------------------------------") + + logger.info(showStatus("Finished:")) if __name__ == '__main__': diff --git a/ofu_app/apps/donar/utils/migrate_data_rooms.py b/ofu_app/apps/donar/utils/migrate_data_rooms.py index 7957c82..f5ba439 100644 --- a/ofu_app/apps/donar/utils/migrate_data_rooms.py +++ b/ofu_app/apps/donar/utils/migrate_data_rooms.py @@ -6,6 +6,10 @@ from apps.donar.models import Room, Lecture_Terms, Lecture from apps.donar.utils.parser import univis_rooms_parser from apps.donar.utils.parser import univis_lectures_parser +import logging + +logger = logging.getLogger(__name__) + # CONFIG Fakultaet FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften" FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften" @@ -73,15 +77,28 @@ def writeUnivisRoomDataInDB(data): if 'description' in room: description = room['description'] - Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name, - orgname=orgname, short=short, size=size, description=description) + room = Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name, + orgname=orgname, short=short, size=size, description=description) + room.save() + logger.info('ROOM: {}'.format(room.short)) except IntegrityError as err: - pprint(err.args) + logger.warning('Room already exists') + + +def delete(): + rooms = Room.objects.all() + logger.info("Deleted following Rooms:") + for room in rooms: + logger.info("Room: {name}".format( + name=room.short) + ) + room.delete() def main(): # get food jsons - pprint("Begin: Room: " + str(Room.objects.count())) + logger.info("Start:\nRoom: {}".format(Room.objects.count())) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi))) @@ -102,7 +119,7 @@ def main(): writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x"))) - pprint("Now: Room: " + str(Room.objects.count())) + logger.info("Finished:\nRoom: {}".format(Room.objects.count())) if __name__ == '__main__': diff --git a/ofu_app/apps/events/utils/migrate_data.py b/ofu_app/apps/events/utils/migrate_data.py index b1f462a..685816c 100644 --- a/ofu_app/apps/events/utils/migrate_data.py +++ b/ofu_app/apps/events/utils/migrate_data.py @@ -7,6 +7,9 @@ from apps.events.utils.parser import univis_eventpage_parser from apps.events.utils.parser import fekide_eventpage_parser from apps.events.models import Event, Location +import logging + +logger = logging.getLogger(__name__) UNIVIS_CATEGORY = 'Univis' @@ -20,32 +23,17 @@ UNIVIS_RPG_WIAI = "http://univis.uni-bamberg.de/prg?search=events&department=Fak def writeFekideDataInDB(data): for date in data['dates']: for event in date['events']: - try: - Location.objects.create(name=event['location']) - except IntegrityError: - # print("Location %s already exists." % event['location']) - pass + location, _ = Location.objects.get_or_create(name=event['location']) + location.save() - try: - event_obj, new = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"), - title=event['title']) - if new: - event_obj.category = event['category'] - event_obj.link = event['link'] - event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M") - event_obj.locations.add(Location.objects.get(name=event['location'])) - event_obj.save() - Event.objects.filter(title="").delete() - else: - print("Event %s already exists. Start Update" % str(event_obj.title)) - event_obj.category = event['category'] - event_obj.link = event['link'] - event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M") - event_obj.locations.add(Location.objects.get(name=event['location'])) - event_obj.save() - except IntegrityError: - # ignored - pass + event_obj, _ = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"), + title=event['title']) + event_obj.category = event['category'] + event_obj.link = event['link'] + event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M") + event_obj.locations.add(Location.objects.get(name=event['location'])) + event_obj.save() + logger.info('CREATED - Event: {}'.format(event_obj.title)) def deleteUnivisObjects(): @@ -62,15 +50,13 @@ def writeUnivisLocationsInDB(rooms): for room in rooms: if '@key' in room and 'short' in room: try: - Location.objects.create(key=room['@key'], name=room['short']) - except IntegrityError: - print("Possible Duplicate! Start DB refresh") - try: - Location.objects.get(name=room['short']).key = room['@key'] - except Exception as harderr: - print("Failed to refresh object" + harderr.args) + location, _ = Location.objects.get_or_create(key=room['@key'], name=room['short']) + location.key = room['@key'] + location.name = room['short'] + location.save() + logger.info('CREATE - Location: {}'.format(location.name)) except Exception as err: - print(err.args) + logger.critical(err.args) def getLocationIDs(event): @@ -110,21 +96,24 @@ def writeUnivisEventsInDB(events: list): event_obj.orgname = event['orgname'] try: event_obj.save() + logger.info(event_obj.title) except IntegrityError: # TODO: Update DB Object if duplicate detected - print("Found Duplicate!") + logger.info("Found Duplicate!") except Exception as err: - print(err.args) + logger.exception(err.args) Event.objects.filter(title="").delete() def write_out_db_objects(): - pprint("Event: " + str(Event.objects.count())) - pprint("Location: " + str(Location.objects.count())) + return "\n\tEvent: {event}\n\tLocation: {location}".format( + event=Event.objects.count(), + location=Location.objects.count(), + ) def main(): - print("Aktueller Stand:") + logger.info("Aktueller Stand:") write_out_db_objects() # deleteUnivisObjects() @@ -139,7 +128,7 @@ def main(): writeFekideDataInDB(fekide_eventpage_parser.parsePage()) - print("Neuer Stand:") + logger.info("Neuer Stand:") write_out_db_objects() diff --git a/ofu_app/core/settings.py b/ofu_app/core/settings.py index 2714e4f..c05af6c 100755 --- a/ofu_app/core/settings.py +++ b/ofu_app/core/settings.py @@ -241,5 +241,9 @@ LOGGING = { 'handlers': ['console', 'file', 'mail_admins'], 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), }, + 'apps.donar.utils': { + 'handlers': ['console', 'file', 'mail_admins'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'), + }, }, }