Add logging, Refractor importer

ofu_app/apps/donar/management/commands/delete_lectures.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand, CommandError
+from apps.donar.models import Room
+from apps.donar.utils import migrate_data_lectures
+
+
+class Command(BaseCommand):
+    help = "Imports Lectures from UnivIS PRG. Requires Room import"
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        migrate_data_lectures.delete()

ofu_app/apps/donar/management/commands/delete_rooms.py

@@ -0,0 +1,13 @@
+from django.core.management.base import BaseCommand, CommandError
+from apps.donar.models import Room
+from apps.donar.utils import migrate_data_rooms
+
+
+class Command(BaseCommand):
+    help = "Imports Rooms from Univis PRG"
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        migrate_data_rooms.delete()

ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.1 on 2018-04-01 11:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('donar', '0004_auto_20180117_0137'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lecture',
+            name='short',
+            field=models.CharField(max_length=256),
+        ),
+    ]

ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.0.1 on 2018-04-01 11:39
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('donar', '0005_auto_20180401_1136'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lecture',
+            name='lecturer_id',
+            field=models.CharField(max_length=512),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_id',
+            field=models.CharField(max_length=512, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_ref',
+            field=models.CharField(max_length=512, unique=True),
+        ),
+    ]

ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.0.1 on 2018-04-01 11:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('donar', '0006_auto_20180401_1139'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lecture',
+            name='name',
+            field=models.CharField(max_length=512),
+        ),
+    ]

ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py

@@ -0,0 +1,28 @@
+# Generated by Django 2.0.1 on 2018-04-01 11:42
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('donar', '0007_auto_20180401_1140'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lecture',
+            name='lecturer_id',
+            field=models.CharField(max_length=1024),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_id',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_ref',
+            field=models.CharField(max_length=1024, unique=True),
+        ),
+    ]

ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py

@@ -0,0 +1,33 @@
+# Generated by Django 2.0.1 on 2018-04-01 11:43
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('donar', '0008_auto_20180401_1142'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='lecture',
+            name='lecturer_id',
+            field=models.CharField(max_length=256),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='short',
+            field=models.CharField(max_length=512),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_id',
+            field=models.CharField(max_length=256, unique=True),
+        ),
+        migrations.AlterField(
+            model_name='lecture',
+            name='univis_ref',
+            field=models.CharField(max_length=256, unique=True),
+        ),
+    ]

ofu_app/apps/donar/models.py

@@ -11,8 +11,8 @@ MAX_COORDS_NAME_LENGTH = 256
 MAX_COORDS_LENGTH = 256
 
 MAX_LECTURE_IDS_LENGTH = 256
-MAX_LECTURE_SHORT_LENGTH = 128
+MAX_LECTURE_SHORT_LENGTH = 512
-MAX_LECTURE_NAME_LENGTH = 256
+MAX_LECTURE_NAME_LENGTH = 512
 MAX_LECTURE_TYPE_LENGTH = 64
 
 

ofu_app/apps/donar/utils/migrate_data_lectures.py

@@ -4,6 +4,9 @@ import json
 from pprint import pprint
 from django.db.utils import IntegrityError
 from apps.donar.utils.parser import univis_lectures_parser
+import logging
+
+logger = logging.getLogger(__name__)
 
 # CONFIG Fakultaet
 FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
@@ -32,38 +35,35 @@ def writeUnivisLectureTermsInDB(lecture, lecture_obj):
         if type(lecture['terms']['term']) == list:
             for term in lecture['terms']['term']:
                 try:
-                    term_obj = Lecture_Terms.objects.create()
                     starttime = "00:00"
+                    term_obj = Lecture_Terms.objects.create(starttime=starttime)
                     if 'starttime' in term:
                         starttime = term['starttime']
                         term_obj.starttime = datetime.strptime(starttime, "%H:%M")
+                    term_obj.save()
                     if 'room' in term:
                         room_id = term['room']['UnivISRef']['@key']
-                        term_obj.room = [Room.objects.get(key=room_id)]
+                        term_obj.room.add(Room.objects.get(key=room_id))
-                    term_obj.save()
                     lecture_obj.term.add(term_obj)
                 except IntegrityError as err:
-                    print("ROOM_ID: " + str(room_id))
+                    logger.exception(err)
-                    print(err.args)
 
         else:
             try:
-                term_obj = Lecture_Terms.objects.create()
                 univis_starttime = "00:00"
+                term_obj = Lecture_Terms.objects.create(starttime=univis_starttime)
                 if 'starttime' in lecture['terms']['term']:
                     univis_starttime = lecture['terms']['term']['starttime']
                     term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M')
+                term_obj.save()
                 if 'room' in lecture['terms']['term']:
                     room_id = lecture['terms']['term']['room']['UnivISRef']['@key']
-                    pprint("Room: " + room_id)
                     Room.objects.get(key=room_id)
-                    term_obj.room = [Room.objects.get(key=room_id)]
+                    term_obj.room.add(Room.objects.get(key=room_id))
-
                 term_obj.save()
                 lecture_obj.term.add(term_obj)
             except IntegrityError as err:
-                print("ROOM_ID: " + str(room_id))
+                logger.exception(err)
-                print(err.args)
 
 
 def writeUnivisLectureDataInDB(data):
@@ -91,43 +91,56 @@ def writeUnivisLectureDataInDB(data):
                 lecture_type = lecture['type']
             if 'dozs' in lecture:
                 lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key']
-            print("Lecture: " + name)
             lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short,
                                                  type=lecture_type, lecturer_id=lecturer_id)
             writeUnivisLectureTermsInDB(lecture, lecture_obj)
             lecture_obj.save()
+            logger.info("Lecture: {}".format(lecture_obj.short))
         except IntegrityError as err:
-            print(err.args)
+            logger.warning('Lecture already exists')
+            # logger.exception(err)
 
     return
 
 
 def showStatus(status: str):
-    print(status)
+    return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Terms: {lecture_terms}\n\tRoom: {room}".format(
-    pprint("Lectures: " + str(Lecture.objects.count()))
+        status=status,
-    pprint("Lecture Terms: " + str(Lecture_Terms.objects.count()))
+        lectures=Lecture.objects.count(),
-    pprint("Room: " + str(Room.objects.count()))
+        lecture_terms=Lecture_Terms.objects.count(),
+        room=Room.objects.count()
+    )
+
+
+def delete():
+    lectures = Lecture.objects.all()
+    logger.info("Deleted following Lectures:")
+    for lecture in lectures:
+        logger.info("Lecture: {name}".format(
+            name=lecture.name)
+        )
+        lecture.delete()
 
 
 def main():
     # get food jsons
-    showStatus("Start with:")
+    logger.info(showStatus("Start SoWi:"))
     writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi)))
-    pprint("----------------------------------------------------------------------------------------")
+    # pprint("----------------------------------------------------------------------------------------")
 
-    showStatus("After SoWi:")
+    logger.info(showStatus("Start GuK:"))
     writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK)))
-    pprint("----------------------------------------------------------------------------------------")
+    # pprint("----------------------------------------------------------------------------------------")
 
-    showStatus("After GuK:")
+    logger.info(showStatus("Start HuWi:"))
     writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi)))
-    pprint("----------------------------------------------------------------------------------------")
+    # pprint("----------------------------------------------------------------------------------------")
 
-    showStatus("After HuWi:")
+    logger.info(showStatus("Start WIAI:"))
     writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI)))
-    pprint("----------------------------------------------------------------------------------------")
+    # pprint("----------------------------------------------------------------------------------------")
-    
+
-    showStatus("After WIAI:")
+    logger.info(showStatus("Finished:"))
 
 
 if __name__ == '__main__':

ofu_app/apps/donar/utils/migrate_data_rooms.py

@@ -6,6 +6,10 @@ from apps.donar.models import Room, Lecture_Terms, Lecture
 from apps.donar.utils.parser import univis_rooms_parser
 from apps.donar.utils.parser import univis_lectures_parser
 
+import logging
+
+logger = logging.getLogger(__name__)
+
 # CONFIG Fakultaet
 FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
 FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften"
@@ -73,15 +77,28 @@ def writeUnivisRoomDataInDB(data):
             if 'description' in room:
                 description = room['description']
 
-            Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name,
+            room = Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name,
-                                orgname=orgname, short=short, size=size, description=description)
+                                       orgname=orgname, short=short, size=size, description=description)
+            room.save()
+            logger.info('ROOM: {}'.format(room.short))
         except IntegrityError as err:
-            pprint(err.args)
+            logger.warning('Room already exists')
+
+
+def delete():
+    rooms = Room.objects.all()
+    logger.info("Deleted following Rooms:")
+    for room in rooms:
+        logger.info("Room: {name}".format(
+            name=room.short)
+        )
+        room.delete()
 
 
 def main():
     # get food jsons
-    pprint("Begin: Room: " + str(Room.objects.count()))
+    logger.info("Start:\nRoom: {}".format(Room.objects.count()))
+
     writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK)))
     writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi)))
     writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi)))
@@ -102,7 +119,7 @@ def main():
     writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d")))
     writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x")))
 
-    pprint("Now: Room: " + str(Room.objects.count()))
+    logger.info("Finished:\nRoom: {}".format(Room.objects.count()))
 
 
 if __name__ == '__main__':

ofu_app/apps/events/utils/migrate_data.py

@@ -7,6 +7,9 @@ from apps.events.utils.parser import univis_eventpage_parser
 from apps.events.utils.parser import fekide_eventpage_parser
 
 from apps.events.models import Event, Location
+import logging
+
+logger = logging.getLogger(__name__)
 
 UNIVIS_CATEGORY = 'Univis'
 
@@ -20,32 +23,17 @@ UNIVIS_RPG_WIAI = "http://univis.uni-bamberg.de/prg?search=events&department=Fak
 def writeFekideDataInDB(data):
     for date in data['dates']:
         for event in date['events']:
-            try:
+            location, _ = Location.objects.get_or_create(name=event['location'])
-                Location.objects.create(name=event['location'])
+            location.save()
-            except IntegrityError:
-                # print("Location %s already exists." % event['location'])
-                pass
 
-            try:
+            event_obj, _ = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"),
-                event_obj, new = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"),
+                                                       title=event['title'])
-                                                             title=event['title'])
+            event_obj.category = event['category']
-                if new:
+            event_obj.link = event['link']
-                    event_obj.category = event['category']
+            event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M")
-                    event_obj.link = event['link']
+            event_obj.locations.add(Location.objects.get(name=event['location']))
-                    event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M")
+            event_obj.save()
-                    event_obj.locations.add(Location.objects.get(name=event['location']))
+            logger.info('CREATED - Event: {}'.format(event_obj.title))
-                    event_obj.save()
-                    Event.objects.filter(title="").delete()
-                else:
-                    print("Event %s already exists. Start Update" % str(event_obj.title))
-                    event_obj.category = event['category']
-                    event_obj.link = event['link']
-                    event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M")
-                    event_obj.locations.add(Location.objects.get(name=event['location']))
-                    event_obj.save()
-            except IntegrityError:
-                # ignored
-                pass
 
 
 def deleteUnivisObjects():
@@ -62,15 +50,13 @@ def writeUnivisLocationsInDB(rooms):
     for room in rooms:
         if '@key' in room and 'short' in room:
             try:
-                Location.objects.create(key=room['@key'], name=room['short'])
+                location, _ = Location.objects.get_or_create(key=room['@key'], name=room['short'])
-            except IntegrityError:
+                location.key = room['@key']
-                print("Possible Duplicate! Start DB refresh")
+                location.name = room['short']
-                try:
+                location.save()
-                    Location.objects.get(name=room['short']).key = room['@key']
+                logger.info('CREATE - Location: {}'.format(location.name))
-                except Exception as harderr:
-                    print("Failed to refresh object" + harderr.args)
             except Exception as err:
-                print(err.args)
+                logger.critical(err.args)
 
 
 def getLocationIDs(event):
@@ -110,21 +96,24 @@ def writeUnivisEventsInDB(events: list):
                         event_obj.orgname = event['orgname']
                     try:
                         event_obj.save()
+                        logger.info(event_obj.title)
                     except IntegrityError:
                         # TODO: Update DB Object if duplicate detected
-                        print("Found Duplicate!")
+                        logger.info("Found Duplicate!")
                     except Exception as err:
-                        print(err.args)
+                        logger.exception(err.args)
                     Event.objects.filter(title="").delete()
 
 
 def write_out_db_objects():
-    pprint("Event: " + str(Event.objects.count()))
+    return "\n\tEvent: {event}\n\tLocation: {location}".format(
-    pprint("Location: " + str(Location.objects.count()))
+        event=Event.objects.count(),
+        location=Location.objects.count(),
+    )
 
 
 def main():
-    print("Aktueller Stand:")
+    logger.info("Aktueller Stand:")
     write_out_db_objects()
 
     # deleteUnivisObjects()
@@ -139,7 +128,7 @@ def main():
 
     writeFekideDataInDB(fekide_eventpage_parser.parsePage())
 
-    print("Neuer Stand:")
+    logger.info("Neuer Stand:")
     write_out_db_objects()
 
 

ofu_app/core/settings.py

@@ -241,5 +241,9 @@ LOGGING = {
             'handlers': ['console', 'file', 'mail_admins'],
             'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
         },
+        'apps.donar.utils': {
+            'handlers': ['console', 'file', 'mail_admins'],
+            'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
+        },
     },
 }