From 65b95cddfb51d1e4af1338365f210299409df8a7 Mon Sep 17 00:00:00 2001 From: michigg Date: Tue, 24 Oct 2017 00:02:17 +0200 Subject: [PATCH] start lecture mirgrations --- ofu_app/apps/donar/models.py | 20 +++- ofu_app/apps/donar/utils/migrate_data.py | 104 ++++++++++++++++-- .../utils/parser/univis_lectures_parser.py | 29 +++++ ofu_app/apps/events/utils/migrate_data.py | 8 +- 4 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 ofu_app/apps/donar/utils/parser/univis_lectures_parser.py diff --git a/ofu_app/apps/donar/models.py b/ofu_app/apps/donar/models.py index 8ec7b91..5768e93 100644 --- a/ofu_app/apps/donar/models.py +++ b/ofu_app/apps/donar/models.py @@ -1,4 +1,5 @@ from django.db import models +from django.utils import timezone MAX_LENGTH = 60 @@ -22,4 +23,21 @@ class VGN_Coords(models.Model): name = models.CharField(max_length=MAX_LENGTH, unique=True) coords = models.CharField(max_length=MAX_LENGTH, unique=True) longitude = models.CharField(max_length=MAX_LENGTH, unique=True) - latitude = models.CharField(max_length=MAX_LENGTH, unique=True) \ No newline at end of file + latitude = models.CharField(max_length=MAX_LENGTH, unique=True) + + +class Lecture(models.Model): + id = models.AutoField(primary_key=True) + univis_ref = models.CharField(max_length=MAX_LENGTH, unique=True) + univis_id = models.CharField(max_length=MAX_LENGTH, unique=True) + name = models.CharField(max_length=MAX_LENGTH) + short = models.CharField(max_length=MAX_LENGTH) + type = models.CharField(max_length=MAX_LENGTH) + lecturer_id = models.CharField(max_length=MAX_LENGTH) + term = models.ManyToManyField('Lecture_Terms', blank=False) + + +class Lecture_Terms(models.Model): + id = models.AutoField(primary_key=True) + starttime = time = models.TimeField(blank=False, default=timezone.now()) + room = models.ManyToManyField('Room', blank=False, blank=True, null=True) diff --git a/ofu_app/apps/donar/utils/migrate_data.py b/ofu_app/apps/donar/utils/migrate_data.py index fc1229f..b2176e8 100644 --- a/ofu_app/apps/donar/utils/migrate_data.py +++ b/ofu_app/apps/donar/utils/migrate_data.py @@ -1,15 +1,27 @@ import json from pprint import pprint from django.db.utils import IntegrityError +from datetime import datetime -from apps.donar.models import Room +from apps.donar.models import Room, Lecture_Terms, Lecture from apps.donar.utils.parser import univis_rooms_parser +from apps.donar.utils.parser import univis_lectures_parser -# CONFIG -UNIVIS_RPG_GuK = "http://univis.uni-bamberg.de/prg?search=rooms&department=Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften&show=xml" -UNIVIS_RPG_SoWi = "http://univis.uni-bamberg.de/prg?search=rooms&department=Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften&show=xml" -UNIVIS_RPG_HuWi = "http://univis.uni-bamberg.de/prg?search=rooms&department=Fakult%E4t%20Humanwissenschaften&show=xml" -UNIVIS_RPG_WIAI = "http://univis.uni-bamberg.de/prg?search=rooms&department=Fakult%E4t%20Wirtschaftsinformatik&show=xml" +# CONFIG Fakultaet +FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften" +FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften" +FAKULTAET_HuWi = "Fakult%E4t%20Humanwissenschaften" +FAKULTAET_WIAI = "Fakult%E4t%20Wirtschaftsinformatik" + + +# CONFIG ROOMS +def univis_rooms(fakultaet): + return "http://univis.uni-bamberg.de/prg?search=rooms&department=" + fakultaet + "&show=xml" + + +# CONFIG LECTURES +def univis_lectures(fakultaet): + return "http://univis.uni-bamberg.de/prg?search=lectures&department=" + fakultaet + "&show=exml" def getJsonFromFile(path): @@ -17,7 +29,7 @@ def getJsonFromFile(path): return json.load(file) -def writeFekideDataInDB(data): +def writeUnivisRoomDataInDB(data): for room in data: try: key = "" @@ -53,13 +65,83 @@ def writeFekideDataInDB(data): break +def writeUnivisLectureTermsInDB(lecture, lecture_obj): + if 'terms' in lecture: + if type(lecture['terms']['term']) == list: + for term in lecture['terms']['term']: + try: + term_obj = Lecture_Terms.objects.create() + starttime = "00:00" + if 'starttime' in term: + starttime = term['starttime'] + term_obj.starttime = datetime.strptime(starttime, "%H%H:%M%M") + if 'room' in term: + room_id = term['room']['UnivISRef']['@key'] + term_obj.room = Room.objects.get(key=room_id) + term_obj.save() + lecture_obj.term.add(term_obj) + except IntegrityError as err: + print(err.args) + + else: + try: + term_obj = Lecture_Terms.objects.create() + starttime = "00:00" + if 'starttime' in lecture['terms']['term']: + starttime = lecture['terms']['term']['starttime'] + term_obj.starttime = datetime.strptime(starttime, "%H%H:%M%M") + if 'room' in lecture['terms']['term']: + room_id = lecture['terms']['term']['room']['UnivISRef']['@key'] + term_obj.room = Room.objects.get(key=room_id) + term_obj.save() + lecture_obj.term.add(term_obj) + except IntegrityError as err: + print(err.args) + + +def writeUnivisLectureDataInDB(data): + for lecture in data: + try: + key = '' + name = '' + orgname = '' + short = '' + lecture_type = '' + lecturer_id = '' + + if '@key' in lecture: + key = lecture['@key'] + if 'name' in lecture: + # TODO Fix name bug + name = lecture['name'] + if 'id' in lecture: + univis_id = lecture['id'] + if 'orgname' in lecture: + orgname = lecture['orgname'] + if 'short' in lecture: + short = lecture['short'] + if 'type' in lecture: + lecture_type = lecture['type'] + if 'dozs' in lecture: + lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] + lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, + type=lecture_type, lecturer_id=lecture_type) + writeUnivisLectureTermsInDB(lecture, lecture_obj) + lecture_obj.save() + except IntegrityError as err: + print() + + return + + def main(): # get food jsons - writeFekideDataInDB(univis_rooms_parser.parsePage(UNIVIS_RPG_GuK)) - writeFekideDataInDB(univis_rooms_parser.parsePage(UNIVIS_RPG_SoWi)) - writeFekideDataInDB(univis_rooms_parser.parsePage(UNIVIS_RPG_HuWi)) - writeFekideDataInDB(univis_rooms_parser.parsePage(UNIVIS_RPG_WIAI)) + # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK))) + # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi))) + # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi))) + # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_WIAI))) pprint("Room: " + str(Room.objects.count())) + writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI))) if __name__ == '__main__': diff --git a/ofu_app/apps/donar/utils/parser/univis_lectures_parser.py b/ofu_app/apps/donar/utils/parser/univis_lectures_parser.py new file mode 100644 index 0000000..10ae33a --- /dev/null +++ b/ofu_app/apps/donar/utils/parser/univis_lectures_parser.py @@ -0,0 +1,29 @@ +import requests +import datetime +import xmltodict +import json +from pprint import pprint + + +def loadPage(url: str): + return requests.get(url).content + + +def getDay(): + return datetime.datetime.today().strftime("%A, %d.%m.%Y") + + +def getLectures(dict: dict): + lectures = [] + for lecture in dict['UnivIS']['Lecture']: + lectures.append(lecture) + return lectures + + +def parsePage(url): + page = loadPage(url) + dict = xmltodict.parse(page) + lectures = getLectures(dict) + return lectures + +# parsePage( "http://univis.uni-bamberg.de/prg?search=lectures&department=Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften&show=exml") diff --git a/ofu_app/apps/events/utils/migrate_data.py b/ofu_app/apps/events/utils/migrate_data.py index d304734..b1f462a 100644 --- a/ofu_app/apps/events/utils/migrate_data.py +++ b/ofu_app/apps/events/utils/migrate_data.py @@ -129,13 +129,13 @@ def main(): # deleteUnivisObjects() # events, rooms, persons = univis_eventpage_parser.parsePage(UNIVIS_RPG_HuWi) - # writeUnivisDataInDB(events, rooms, persons) + # writeUnivisRoomDataInDB(events, rooms, persons) # events, rooms, persons = univis_eventpage_parser.parsePage(UNIVIS_RPG_SoWi) - # writeUnivisDataInDB(events, rooms, persons) + # writeUnivisRoomDataInDB(events, rooms, persons) # events, rooms, persons = univis_eventpage_parser.parsePage(UNIVIS_RPG_GuK) - # writeUnivisDataInDB(events, rooms, persons) + # writeUnivisRoomDataInDB(events, rooms, persons) # events, rooms, persons = univis_eventpage_parser.parsePage(UNIVIS_RPG_WIAI) - # writeUnivisDataInDB(events, rooms, persons) + # writeUnivisRoomDataInDB(events, rooms, persons) writeFekideDataInDB(fekide_eventpage_parser.parsePage())