diff --git a/roofis2/core/settings.py b/roofis2/core/settings.py index 1d5f2a6..47c8c17 100644 --- a/roofis2/core/settings.py +++ b/roofis2/core/settings.py @@ -28,6 +28,8 @@ LOGIN_REDIRECT_URL = 'roomservice:home' ALLOWED_HOSTS = [] +DATA_UPLOAD_MAX_NUMBER_FIELDS = 40000 + # Application definition INSTALLED_APPS = [ diff --git a/roofis2/roomservice/admin.py b/roofis2/roomservice/admin.py index a4f2bfc..3945b3c 100644 --- a/roofis2/roomservice/admin.py +++ b/roofis2/roomservice/admin.py @@ -1,8 +1,10 @@ from django.contrib import admin -from .models import Room, Building, RoomType, OrgUnit, Equipment, Staff, Lecture, LectureSpecifiaction, LectureType +from .models import Room, Building, RoomType, OrgUnit, Equipment, Staff, Lecture, LectureTerm, LectureSpecifiaction, \ + LectureType # Register your models here. admin.site.register(Lecture) +admin.site.register(LectureTerm) admin.site.register(LectureSpecifiaction) admin.site.register(LectureType) admin.site.register(Staff) diff --git a/roofis2/roomservice/models.py b/roofis2/roomservice/models.py index abc28d5..3a80c72 100644 --- a/roofis2/roomservice/models.py +++ b/roofis2/roomservice/models.py @@ -101,6 +101,7 @@ class LectureSpecifiaction(models.Model): class LectureType(models.Model): + #univis_id = models.CharField(max_length=8) name = models.CharField(max_length=32) def __str__(self): @@ -129,16 +130,20 @@ class Lecture(models.Model): def __str__(self): return '{}'.format(self.title) -# class Lecture_Term(models.Model): -# room = models.ForeignKey(Room, on_delete=models.CASCADE) -# start_date = models.DateField() -# end_date = models.DateField() -# start_time = models.TimeField() -# end_time = models.TimeField() -# repeat_intervall = models.IntegerField() -# excludes = models.ManyToManyField(Exclude_Term) -# # TODO: Help please -# exclude_vacation = models.BooleanField() + +class LectureTerm(models.Model): + # room = models.ForeignKey(Room, on_delete=models.CASCADE) + room = models.CharField(max_length=64, null=True, blank=True) + start_date = models.DateField(null=True, blank=True) + end_date = models.DateField(null=True, blank=True) + start_time = models.TimeField(null=True, blank=True) + end_time = models.TimeField(null=True, blank=True) + exclude = models.CharField(max_length=128, null=True, blank=True) + repeat = models.CharField(max_length=16, null=True, blank=True) + lecture = models.ForeignKey(Lecture, on_delete=models.CASCADE, null=True, blank=True) + + def __str__(self): + return '{}'.format(self.lecture.title) # class Booking(models.Model): # DAY = 0 diff --git a/roofis2/roomservice/utils/migrate_data_lectures.py b/roofis2/roomservice/utils/migrate_data_lectures.py index bce1a76..1345aed 100644 --- a/roofis2/roomservice/utils/migrate_data_lectures.py +++ b/roofis2/roomservice/utils/migrate_data_lectures.py @@ -3,7 +3,7 @@ import json from pprint import pprint from django.db.utils import IntegrityError from roomservice.utils.parser import univis_lectures_parser -from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType +from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType, LectureTerm import logging @@ -31,128 +31,134 @@ def getJsonFromFile(path): return json.load(file) -def writeUnivisLectureTermsInDB(lecture): - logger.info('WTF') +def writeUnivisLectureTermsInDB(lecture, lecture_obj): if 'terms' in lecture: if type(lecture['terms']['term']) == list: + logger.info(lecture['terms']['term']) for term in lecture['terms']['term']: - if 'exclude' in term: - logger.info('EXCLUDE: {}'.format(term['exclude'])) - try: - starttime = "00:00" - # term_obj = Lecture_Terms.objects.create(starttime=starttime) - if 'starttime' in term: - starttime = term['starttime'] - # term_obj.starttime = datetime.strptime(starttime, "%H:%M") - # term_obj.save() - if 'room' in term: - room_id = term['room']['UnivISRef']['@key'] - # term_obj.room.add(Room.objects.get(key=room_id)) - # lecture_obj.term.add(term_obj) - except IntegrityError as err: - logger.exception(err) + logger.info(term) + exclude = dict(term).get('exclude') + starttime = dict(term).get('starttime') + endtime = dict(term).get('endtime') + startdate = dict(term).get('startdate') + enddate = dict(term).get('enddate') + repeat = dict(term).get('repeat') + room = dict(term).get('room') + if not room: + logger.info('ROOOOOOOOM ___________________________________________') + logger.info(lecture['terms']) + term_obj, _ = LectureTerm.objects.update_or_create(exclude=exclude, + repeat=repeat, + room=room, + lecture=lecture_obj) + + if startdate: + term_obj.start_date = dict(term).get('startdate') # datetime.strptime(startdate, '%Y-%m-%d') + if enddate: + term_obj.end_date = dict(term).get('enddate') # datetime.strptime(enddate, '%Y-%m-%d') + if starttime: + term_obj.start_time = dict(term).get('starttime') # datetime.strptime(starttime, '%H:%M') + if endtime: + term_obj.end_time = dict(term).get('endtime') # datetime.strptime(endtime, '%H:%M'), + term_obj.save() else: - try: - univis_starttime = "00:00" - # term_obj = Lecture_Terms.objects.create(starttime=univis_starttime) - if 'starttime' in lecture['terms']['term']: - univis_starttime = lecture['terms']['term']['starttime'] - # term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M') - # term_obj.save() - if 'room' in lecture['terms']['term']: - room_id = lecture['terms']['term']['room']['UnivISRef']['@key'] - # Room.objects.get(key=room_id) - # term_obj.room.add(Room.objects.get(key=room_id)) - # term_obj.save() - # lecture_obj.term.add(term_obj) - except IntegrityError as err: - logger.exception(err) + term = lecture['terms']['term'] + exclude = dict(term).get('exclude') + starttime = dict(term).get('starttime') + endtime = dict(term).get('endtime') + startdate = dict(term).get('startdate') + enddate = dict(term).get('enddate') + repeat = dict(term).get('repeat') + room = dict(term).get('room') + if not room: + logger.info('ROOOOOOOOM2 ___________________________________________') + logger.info(lecture) + + term_obj, _ = LectureTerm.objects.update_or_create(exclude=exclude, + repeat=repeat, + room=room, + lecture=lecture_obj) + if startdate: + term_obj.start_date = dict(term).get('startdate') # datetime.strptime(startdate, '%Y-%m-%d') + if enddate: + term_obj.end_date = dict(term).get('enddate') # datetime.strptime(enddate, '%Y-%m-%d') + if starttime: + term_obj.start_time = dict(term).get('starttime') # datetime.strptime(starttime, '%H:%M') + if endtime: + term_obj.end_time = dict(term).get('endtime') # datetime.strptime(endtime, '%H:%M'), + term_obj.save() def writeUnivisLectureDataInDB(data): for lecture in data: - univis_key = lecture['@key'] - univis_id = lecture['id'] - name = lecture['name'] - orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname']) + if not 'courses' in lecture: + lecture = dict(lecture) + univis_key = lecture['@key'] + univis_id = lecture['id'] + short = lecture.get('short') + name = lecture['name'] + url_description = lecture.get('url_description') + turnout = lecture.get('turnout') + if name == 'Projektpraktikum Mensch-Computer-Interaktion': + logger.info(lecture) + organizational = lecture.get('organizational') + time_description = lecture.get('time_description') + summary = lecture.get('summary') + orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname']) - short = None - if 'short' in lecture: - short = lecture['short'] + ects_cred = lecture.get('ects_cred') + if ects_cred: + ects_cred = float(str(lecture['ects_cred']).replace(',', '.')) - ects_cred = None - if 'ects_cred' in lecture: - ects_cred = float(str(lecture['ects_cred']).replace(',', '.')) + sws = None + if 'sws' in lecture: + sws = float(str(lecture['sws']).replace(',', '.')) - sws = None - if 'sws' in lecture: - sws = float(str(lecture['sws']).replace(',', '.')) + type = None + if 'type' in lecture: + type, _ = LectureType.objects.get_or_create(name=lecture['type']) - url_description = None - if 'url_description' in lecture: - url_description = lecture['url_description'] + # logger.info(name) + lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name, + orgname=orgname, short=short, ects=ects_cred, + sws=sws, url_description=url_description, + estimated_visitor=turnout, organizational=organizational, + time_description=time_description, summary=summary, + type=type) - turnout = None - if 'turnout' in lecture: - turnout = lecture['turnout'] + lecture_specs_options = LectureSpecifiaction.objects.all() + for lecture_spec in lecture_specs_options: + univis_id = lecture_spec.univis_id.lower() + if univis_id in lecture and lecture[univis_id] == 'ja': + lecture_obj.specification.add(lecture_spec) - organizational = None - if 'organizational' in lecture: - organizational = lecture['organizational'] + orgunits = None + if 'orgunits' in lecture: + for orgunit in lecture['orgunits']: + if len(orgunit) > 0: + for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','): + cleaned_orgunit = orgunit.strip().strip("'").strip("'") + orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit) + lecture_obj.orgunits.add(orgunit) - time_description = None - if 'time_description' in lecture: - time_description = lecture['time_description'] - - summary = None - if 'summary' in lecture: - summary = lecture['summary'] - - type = None - if 'type' in lecture: - type, _ = LectureType.objects.get_or_create(name=lecture['type']) - - logger.info(name) - lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name, - orgname=orgname, short=short, ects=ects_cred, - sws=sws, url_description=url_description, - estimated_visitor=turnout, organizational=organizational, - time_description=time_description, summary=summary, type=type) - - lecture_specs_options = LectureSpecifiaction.objects.all() - for lecture_spec in lecture_specs_options: - univis_id = lecture_spec.univis_id.lower() - if univis_id in lecture and lecture[univis_id] == 'ja': - lecture_obj.specification.add(lecture_spec) - - orgunits = None - if 'orgunits' in lecture: - for orgunit in lecture['orgunits']: - if len(orgunit) > 0: - for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','): - cleaned_orgunit = orgunit.strip().strip("'").strip("'") - orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit) + # TODO: Check xml syntax #orgunit problem + pass + else: + orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit) lecture_obj.orgunits.add(orgunit) - # TODO: Check xml syntax #orgunit problem - pass - else: - orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit) - lecture_obj.orgunits.add(orgunit) - - # logger.info(lecture_obj) - # if 'dozs' in lecture: - # lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] - # lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, - # type=lecture_type, lecturer_id=lecturer_id) - # writeUnivisLectureTermsInDB(lecture, lecture_obj) - # writeUnivisLectureTermsInDB(lecture) - # lecture_obj.save() - # logger.info("Lecture: {}".format(lecture_obj.short)) - # except IntegrityError as err: - # logger.warning('Lecture already exists') - # logger.exception(err) + # logger.info(lecture_obj) + # if 'dozs' in lecture: + # lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] + # lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, + # type=lecture_type, lecturer_id=lecturer_id) + writeUnivisLectureTermsInDB(lecture, lecture_obj) + # lecture_obj.save() + # logger.info("Lecture: {}".format(lecture_obj.short)) + # except IntegrityError as err: + # logger.warning('Lecture already exists') + # logger.exception(err) def showStatus(status: str): @@ -168,17 +174,17 @@ def showStatus(status: str): def main(): # get food jsons - logger.info(showStatus("Start SoWi:")) - writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi))) - logger.info("----------------------------------------------------------------------------------------") - - logger.info(showStatus("Start GuK:")) - writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK))) - logger.info("----------------------------------------------------------------------------------------") - - logger.info(showStatus("Start HuWi:")) - writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi))) - logger.info("----------------------------------------------------------------------------------------") + # logger.info(showStatus("Start SoWi:")) + # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi))) + # logger.info("----------------------------------------------------------------------------------------") + # + # logger.info(showStatus("Start GuK:")) + # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK))) + # logger.info("----------------------------------------------------------------------------------------") + # + # logger.info(showStatus("Start HuWi:")) + # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi))) + # logger.info("----------------------------------------------------------------------------------------") logger.info(showStatus("Start WIAI:")) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI))) diff --git a/roofis2/roomservice/utils/migrate_data_rooms.py b/roofis2/roomservice/utils/migrate_data_rooms.py index 16082c7..46ea783 100644 --- a/roofis2/roomservice/utils/migrate_data_rooms.py +++ b/roofis2/roomservice/utils/migrate_data_rooms.py @@ -138,25 +138,25 @@ def main(): Building.objects.count(), RoomType.objects.count())) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_WIAI))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_WIAI))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("k"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("z"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("u"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("f"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("r"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("h"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("l"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("m"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("o"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("p"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("v"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d"))) - # writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("z"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("u"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("f"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("r"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("h"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("l"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("m"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("o"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("p"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("v"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d"))) + writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x"))) logger.info("\nFinished:\nRoom: {}\nOrgunits: {}\nBuildings: {}\nRoomTypes: {}" .format(Room.objects.count(), diff --git a/roofis2/roomservice/utils/parser/univis_lectures_parser.py b/roofis2/roomservice/utils/parser/univis_lectures_parser.py index 10ae33a..8663af7 100644 --- a/roofis2/roomservice/utils/parser/univis_lectures_parser.py +++ b/roofis2/roomservice/utils/parser/univis_lectures_parser.py @@ -22,7 +22,9 @@ def getLectures(dict: dict): def parsePage(url): page = loadPage(url) + pprint(url) dict = xmltodict.parse(page) + # pprint(dict['UnivIS']['Lecture'], depth=4) lectures = getLectures(dict) return lectures