mgoetz/roofis2
Archived
1
1
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Drunk and Hangover

Browse Source
This commit is contained in:
Götz 2018-10-14 23:26:30 +02:00
parent 002aeff50a
commit 464fe5ba17
6 changed files with 162 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
roofis2/core/settings.py
View File

@ -28,6 +28,8 @@ LOGIN_REDIRECT_URL = 'roomservice:home'
ALLOWED_HOSTS = [] ALLOWED_HOSTS = []
DATA_UPLOAD_MAX_NUMBER_FIELDS = 40000
# Application definition # Application definition
INSTALLED_APPS = [ INSTALLED_APPS = [

4
roofis2/roomservice/admin.py
View File

@ -1,8 +1,10 @@
from django.contrib import admin from django.contrib import admin
from .models import Room, Building, RoomType, OrgUnit, Equipment, Staff, Lecture, LectureSpecifiaction, LectureType from .models import Room, Building, RoomType, OrgUnit, Equipment, Staff, Lecture, LectureTerm, LectureSpecifiaction, \
LectureType
# Register your models here. # Register your models here.
admin.site.register(Lecture) admin.site.register(Lecture)
admin.site.register(LectureTerm)
admin.site.register(LectureSpecifiaction) admin.site.register(LectureSpecifiaction)
admin.site.register(LectureType) admin.site.register(LectureType)
admin.site.register(Staff) admin.site.register(Staff)

25
roofis2/roomservice/models.py
View File

@ -101,6 +101,7 @@ class LectureSpecifiaction(models.Model):
class LectureType(models.Model): class LectureType(models.Model):
#univis_id = models.CharField(max_length=8)
name = models.CharField(max_length=32) name = models.CharField(max_length=32)
def __str__(self): def __str__(self):
@ -129,16 +130,20 @@ class Lecture(models.Model):
def __str__(self): def __str__(self):
return '{}'.format(self.title) return '{}'.format(self.title)
# class Lecture_Term(models.Model):
# room = models.ForeignKey(Room, on_delete=models.CASCADE) class LectureTerm(models.Model):
# start_date = models.DateField() # room = models.ForeignKey(Room, on_delete=models.CASCADE)
# end_date = models.DateField() room = models.CharField(max_length=64, null=True, blank=True)
# start_time = models.TimeField() start_date = models.DateField(null=True, blank=True)
# end_time = models.TimeField() end_date = models.DateField(null=True, blank=True)
# repeat_intervall = models.IntegerField() start_time = models.TimeField(null=True, blank=True)
# excludes = models.ManyToManyField(Exclude_Term) end_time = models.TimeField(null=True, blank=True)
# # TODO: Help please exclude = models.CharField(max_length=128, null=True, blank=True)
# exclude_vacation = models.BooleanField() repeat = models.CharField(max_length=16, null=True, blank=True)
lecture = models.ForeignKey(Lecture, on_delete=models.CASCADE, null=True, blank=True)
def __str__(self):
return '{}'.format(self.lecture.title)
# class Booking(models.Model): # class Booking(models.Model):
# DAY = 0 # DAY = 0

238
roofis2/roomservice/utils/migrate_data_lectures.py
View File

@ -3,7 +3,7 @@ import json
from pprint import pprint from pprint import pprint
from django.db.utils import IntegrityError from django.db.utils import IntegrityError
from roomservice.utils.parser import univis_lectures_parser from roomservice.utils.parser import univis_lectures_parser
from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType, LectureTerm
import logging import logging
@ -31,128 +31,134 @@ def getJsonFromFile(path):
return json.load(file) return json.load(file)
def writeUnivisLectureTermsInDB(lecture): def writeUnivisLectureTermsInDB(lecture, lecture_obj):
logger.info('WTF')
if 'terms' in lecture: if 'terms' in lecture:
if type(lecture['terms']['term']) == list: if type(lecture['terms']['term']) == list:
logger.info(lecture['terms']['term'])
for term in lecture['terms']['term']: for term in lecture['terms']['term']:
if 'exclude' in term: logger.info(term)
logger.info('EXCLUDE: {}'.format(term['exclude'])) exclude = dict(term).get('exclude')
try: starttime = dict(term).get('starttime')
starttime = "00:00" endtime = dict(term).get('endtime')
# term_obj = Lecture_Terms.objects.create(starttime=starttime) startdate = dict(term).get('startdate')
if 'starttime' in term: enddate = dict(term).get('enddate')
starttime = term['starttime'] repeat = dict(term).get('repeat')
# term_obj.starttime = datetime.strptime(starttime, "%H:%M") room = dict(term).get('room')
# term_obj.save() if not room:
if 'room' in term: logger.info('ROOOOOOOOM ___________________________________________')
room_id = term['room']['UnivISRef']['@key'] logger.info(lecture['terms'])
# term_obj.room.add(Room.objects.get(key=room_id)) term_obj, _ = LectureTerm.objects.update_or_create(exclude=exclude,
# lecture_obj.term.add(term_obj) repeat=repeat,
except IntegrityError as err: room=room,
logger.exception(err) lecture=lecture_obj)
if startdate:
term_obj.start_date = dict(term).get('startdate') # datetime.strptime(startdate, '%Y-%m-%d')
if enddate:
term_obj.end_date = dict(term).get('enddate') # datetime.strptime(enddate, '%Y-%m-%d')
if starttime:
term_obj.start_time = dict(term).get('starttime') # datetime.strptime(starttime, '%H:%M')
if endtime:
term_obj.end_time = dict(term).get('endtime') # datetime.strptime(endtime, '%H:%M'),
term_obj.save()
else: else:
try: term = lecture['terms']['term']
univis_starttime = "00:00" exclude = dict(term).get('exclude')
# term_obj = Lecture_Terms.objects.create(starttime=univis_starttime) starttime = dict(term).get('starttime')
if 'starttime' in lecture['terms']['term']: endtime = dict(term).get('endtime')
univis_starttime = lecture['terms']['term']['starttime'] startdate = dict(term).get('startdate')
# term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M') enddate = dict(term).get('enddate')
# term_obj.save() repeat = dict(term).get('repeat')
if 'room' in lecture['terms']['term']: room = dict(term).get('room')
room_id = lecture['terms']['term']['room']['UnivISRef']['@key'] if not room:
# Room.objects.get(key=room_id) logger.info('ROOOOOOOOM2 ___________________________________________')
# term_obj.room.add(Room.objects.get(key=room_id)) logger.info(lecture)
# term_obj.save()
# lecture_obj.term.add(term_obj) term_obj, _ = LectureTerm.objects.update_or_create(exclude=exclude,
except IntegrityError as err: repeat=repeat,
logger.exception(err) room=room,
lecture=lecture_obj)
if startdate:
term_obj.start_date = dict(term).get('startdate') # datetime.strptime(startdate, '%Y-%m-%d')
if enddate:
term_obj.end_date = dict(term).get('enddate') # datetime.strptime(enddate, '%Y-%m-%d')
if starttime:
term_obj.start_time = dict(term).get('starttime') # datetime.strptime(starttime, '%H:%M')
if endtime:
term_obj.end_time = dict(term).get('endtime') # datetime.strptime(endtime, '%H:%M'),
term_obj.save()
def writeUnivisLectureDataInDB(data): def writeUnivisLectureDataInDB(data):
for lecture in data: for lecture in data:
univis_key = lecture['@key'] if not 'courses' in lecture:
univis_id = lecture['id'] lecture = dict(lecture)
name = lecture['name'] univis_key = lecture['@key']
orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname']) univis_id = lecture['id']
short = lecture.get('short')
name = lecture['name']
url_description = lecture.get('url_description')
turnout = lecture.get('turnout')
if name == 'Projektpraktikum Mensch-Computer-Interaktion':
logger.info(lecture)
organizational = lecture.get('organizational')
time_description = lecture.get('time_description')
summary = lecture.get('summary')
orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname'])
short = None ects_cred = lecture.get('ects_cred')
if 'short' in lecture: if ects_cred:
short = lecture['short'] ects_cred = float(str(lecture['ects_cred']).replace(',', '.'))
ects_cred = None sws = None
if 'ects_cred' in lecture: if 'sws' in lecture:
ects_cred = float(str(lecture['ects_cred']).replace(',', '.')) sws = float(str(lecture['sws']).replace(',', '.'))
sws = None type = None
if 'sws' in lecture: if 'type' in lecture:
sws = float(str(lecture['sws']).replace(',', '.')) type, _ = LectureType.objects.get_or_create(name=lecture['type'])
url_description = None # logger.info(name)
if 'url_description' in lecture: lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name,
url_description = lecture['url_description'] orgname=orgname, short=short, ects=ects_cred,
sws=sws, url_description=url_description,
estimated_visitor=turnout, organizational=organizational,
time_description=time_description, summary=summary,
type=type)
turnout = None lecture_specs_options = LectureSpecifiaction.objects.all()
if 'turnout' in lecture: for lecture_spec in lecture_specs_options:
turnout = lecture['turnout'] univis_id = lecture_spec.univis_id.lower()
if univis_id in lecture and lecture[univis_id] == 'ja':
lecture_obj.specification.add(lecture_spec)
organizational = None orgunits = None
if 'organizational' in lecture: if 'orgunits' in lecture:
organizational = lecture['organizational'] for orgunit in lecture['orgunits']:
if len(orgunit) > 0:
for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','):
cleaned_orgunit = orgunit.strip().strip("'").strip("'")
orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit)
lecture_obj.orgunits.add(orgunit)
time_description = None # TODO: Check xml syntax #orgunit problem
if 'time_description' in lecture: pass
time_description = lecture['time_description'] else:
orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit)
summary = None
if 'summary' in lecture:
summary = lecture['summary']
type = None
if 'type' in lecture:
type, _ = LectureType.objects.get_or_create(name=lecture['type'])
logger.info(name)
lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name,
orgname=orgname, short=short, ects=ects_cred,
sws=sws, url_description=url_description,
estimated_visitor=turnout, organizational=organizational,
time_description=time_description, summary=summary, type=type)
lecture_specs_options = LectureSpecifiaction.objects.all()
for lecture_spec in lecture_specs_options:
univis_id = lecture_spec.univis_id.lower()
if univis_id in lecture and lecture[univis_id] == 'ja':
lecture_obj.specification.add(lecture_spec)
orgunits = None
if 'orgunits' in lecture:
for orgunit in lecture['orgunits']:
if len(orgunit) > 0:
for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','):
cleaned_orgunit = orgunit.strip().strip("'").strip("'")
orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit)
lecture_obj.orgunits.add(orgunit) lecture_obj.orgunits.add(orgunit)
# TODO: Check xml syntax #orgunit problem # logger.info(lecture_obj)
pass # if 'dozs' in lecture:
else: # lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key']
orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit) # lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short,
lecture_obj.orgunits.add(orgunit) # type=lecture_type, lecturer_id=lecturer_id)
writeUnivisLectureTermsInDB(lecture, lecture_obj)
# logger.info(lecture_obj) # lecture_obj.save()
# if 'dozs' in lecture: # logger.info("Lecture: {}".format(lecture_obj.short))
# lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] # except IntegrityError as err:
# lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, # logger.warning('Lecture already exists')
# type=lecture_type, lecturer_id=lecturer_id) # logger.exception(err)
# writeUnivisLectureTermsInDB(lecture, lecture_obj)
# writeUnivisLectureTermsInDB(lecture)
# lecture_obj.save()
# logger.info("Lecture: {}".format(lecture_obj.short))
# except IntegrityError as err:
# logger.warning('Lecture already exists')
# logger.exception(err)
def showStatus(status: str): def showStatus(status: str):
@ -168,17 +174,17 @@ def showStatus(status: str):
def main(): def main():
# get food jsons # get food jsons
logger.info(showStatus("Start SoWi:")) # logger.info(showStatus("Start SoWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi))) # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi)))
logger.info("----------------------------------------------------------------------------------------") # logger.info("----------------------------------------------------------------------------------------")
#
logger.info(showStatus("Start GuK:")) # logger.info(showStatus("Start GuK:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK))) # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK)))
logger.info("----------------------------------------------------------------------------------------") # logger.info("----------------------------------------------------------------------------------------")
#
logger.info(showStatus("Start HuWi:")) # logger.info(showStatus("Start HuWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi))) # writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi)))
logger.info("----------------------------------------------------------------------------------------") # logger.info("----------------------------------------------------------------------------------------")
logger.info(showStatus("Start WIAI:")) logger.info(showStatus("Start WIAI:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI))) writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI)))

36
roofis2/roomservice/utils/migrate_data_rooms.py
View File

@ -138,25 +138,25 @@ def main():
Building.objects.count(), Building.objects.count(),
RoomType.objects.count())) RoomType.objects.count()))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK)))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi)))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi)))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_WIAI))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_WIAI)))
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("k"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("k")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("z"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("z")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("u"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("u")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("f"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("f")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("r"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("r")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("h"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("h")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("l"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("l")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("m"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("m")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("o"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("o")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("p"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("p")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("v"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("v")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("w")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d")))
# writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x"))) writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x")))
logger.info("\nFinished:\nRoom: {}\nOrgunits: {}\nBuildings: {}\nRoomTypes: {}" logger.info("\nFinished:\nRoom: {}\nOrgunits: {}\nBuildings: {}\nRoomTypes: {}"
.format(Room.objects.count(), .format(Room.objects.count(),

2
roofis2/roomservice/utils/parser/univis_lectures_parser.py
View File

@ -22,7 +22,9 @@ def getLectures(dict: dict):
def parsePage(url): def parsePage(url):
page = loadPage(url) page = loadPage(url)
pprint(url)
dict = xmltodict.parse(page) dict = xmltodict.parse(page)
# pprint(dict['UnivIS']['Lecture'], depth=4)
lectures = getLectures(dict) lectures = getLectures(dict)
return lectures return lectures