mgoetz/roofis2
Archived
1
1
Fork 0
Code Issues 3 Pull Requests Releases Wiki Activity

Implement lecture import

Browse Source
This commit is contained in:
Götz 2018-06-19 10:08:07 +02:00
parent f0bca5e89a
commit 652e3d50eb
11 changed files with 355 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
roofis2/core/settings.py
View File

@ -183,11 +183,11 @@ LOGGING = {
# 'level': 'INFO',
# },
'roomservice': {
'roomservice.utils': {
'handlers': ['console'],
'level': 'INFO',
},
'roomservice.management.sample_data_creation': {
'roomservice.management': {
'handlers': ['console'],
'level': 'DEBUG'
}

1
roofis2/requirements.txt
View File

@ -7,3 +7,4 @@ django-extensions==2.0.7
requests==2.18.4
djangorestframework==3.8.2
coreapi==2.3.3
xmltodict==0.11.0

9
roofis2/roomservice/management/commands/import_univis_data.py Normal file
View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand
from roomservice.utils import migrate_data_rooms
class Command(BaseCommand):
help = "Import room data from univis prg api"
def handle(self, *args, **options):
migrate_data_rooms.main()

31
roofis2/roomservice/management/commands/import_univis_lecture_specs.py Normal file
View File

@ -0,0 +1,31 @@
from django.core.management.base import BaseCommand
from roomservice.utils import migrate_data_rooms
from roomservice.models import LectureSpecifiaction
import logging
logger = logging.getLogger(__name__)
UNIVIS_IDS_MAP = [{'id': 'OBLIG', 'name': 'obligatorische Lehrveranstaltung'},
{'id': 'GENERALE', 'name': 'Studium Generale'},
{'id': 'ENGLISH', 'name': 'Englischsprachig'},
{'id': 'ETCS', 'name': 'ECTS-Studium'},
{'id': 'WOMSPE', 'name': 'Frauenspezifisch/Geschlechtervergl'},
{'id': 'ANPFLICHT', 'name': 'Anwesenheitspflicht'},
{'id': 'MODULSTUD', 'name': 'Modulstudium'},
{'id': 'FRUEH', 'name': 'Frühstudium'},
{'id': 'ZENIS', 'name': 'Zentrum für Interreligiöse Studien'},
{'id': 'BENSCHEIN', 'name': 'benoteter Schein'},
{'id': 'ERWEI', 'name': 'Erweiterungsbereich'},
{'id': 'ZENMAS', 'name': ' Zentrum für Mittelalterstudien'},
]
class Command(BaseCommand):
help = "Import room data from univis prg api"
def handle(self, *args, **options):
logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count()))
for elem in UNIVIS_IDS_MAP:
equip, _ = LectureSpecifiaction.objects.update_or_create(univis_id=elem['id'], name=elem['name'])
logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count()))

55
roofis2/roomservice/management/commands/import_univis_lecture_type.py Normal file
View File

@ -0,0 +1,55 @@
from django.core.management.base import BaseCommand
from roomservice.utils import migrate_data_rooms
from roomservice.models import LectureType
import logging
logger = logging.getLogger(__name__)
UNIVIS_IDS_MAP = [{'id': 'V', 'name': 'Vorlesung'},
{'id': 'Vorlesung', 'name': 'Vorlesung'},
{'id': 'Vorlesung/Seminar', 'name': 'Vorlesung/Seminar'},
{'id': 'Seminar', 'name': 'Seminar'},
{'id': 'Hauptseminar', 'name': 'Hauptseminar'},
{'id': 'Oberseminar', 'name': 'Oberseminar'},
{'id': 'Blockseminar', 'name': 'Blockseminar'},
{'id': 'Seminar/Oberseminar', 'name': 'Seminar/Oberseminar'},
{'id': 'Seminaristischer Unterricht', 'name': 'Seminaristischer Unterricht'},
{'id': 'Vertiefungsseminar', 'name': 'Vertiefungsseminar'},
{'id': 'Proseminar', 'name': 'Proseminar'},
{'id': 'Seminar/Hauptseminar', 'name': 'Seminar/Hauptseminar'},
{'id': 'Forschungsseminar', 'name': 'Forschungsseminar'},
{'id': 'Ü', 'name': 'Übung'},
{'id': 'Übung', 'name': 'Übung'},
{'id': 'Übung/Blockseminar', 'name': 'Übung/Blockseminar'},
{'id': 'Übung/Tutorium', 'name': 'Übung/Tutorium'},
{'id': 'Praktikum', 'name': 'Praktikum'},
{'id': 'Forschungspraktikum', 'name': 'Forschungspraktikum'},
{'id': 'Sonstige Lehrveranstaltung', 'name': 'Sonstige Lehrveranstaltung'},
{'id': 'Tutorien', 'name': 'Tutorien'},
{'id': 'Kolloquium', 'name': 'Kolloquium'},
{'id': 'Repetitorium', 'name': 'Repetitorium'},
{'id': 'Vorlesung und Übung', 'name': 'Vorlesung und Übung'},
{'id': 'Seminar/Proseminar', 'name': 'Seminar/Proseminar'},
{'id': 'Seminar/Übung', 'name': 'Seminar/Übung'},
{'id': 'feldarchäologisches Praktikum', 'name': 'feldarchäologisches Praktikum'},
{'id': 'Exkursion', 'name': 'Exkursion'},
# {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'},
# {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'},
]
class Command(BaseCommand):
help = "Import room data from univis prg api"
def handle(self, *args, **options):
logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count()))
for elem in UNIVIS_IDS_MAP:
equip, _ = LectureType.objects.update_or_create(univis_id=elem['id'], name=elem['name'])
logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count()))

9
roofis2/roomservice/management/commands/import_univis_terms.py Normal file
View File

@ -0,0 +1,9 @@
from django.core.management.base import BaseCommand
from roomservice.utils import migrate_data_lectures
class Command(BaseCommand):
help = "Import room data from univis prg api"
def handle(self, *args, **options):
migrate_data_lectures.main()

153
roofis2/roomservice/management/sample_data_creation/data_creator.py
View File

@ -1,153 +0,0 @@
from roomservice.models import RoomType, Room, NumEquipment, Building, Location, Equipment, Booking, BookingGroup, \
Staff, AccessPoint, Favorite
from django.contrib.auth.models import User
import logging
import random
import datetime
logger = logging.getLogger(__name__)
LOCATIONS = ['Bamberg', 'Erlangen', 'Crailsheim', 'Hamburg', 'Chemnitz', 'Dresden', 'Sebnitz', 'Gunzenhausen',
'Nürnberg']
BUILDINGS = ['253', '274', '134', '256', '142', '432', '652', '333', '233', '444', '332']
ROOMTYPE = ['Office', 'Conference Room', 'Teaching Room']
BOOKING_GROUP = ['Boss', 'SectionA', 'SectionB', 'SectionC', 'Standard']
EQUIPMENT = ['Beamer', 'Flipchart']
def create():
logger.info('Location Count: {}'.format(Location.objects.count()))
create_locations(LOCATIONS)
logger.info('Location Count: {}'.format(Location.objects.count()))
logger.info('Building Count: {}'.format(Building.objects.count()))
create_buildings(BUILDINGS)
logger.info('Building Count: {}'.format(Building.objects.count()))
logger.info('RoomType Count: {}'.format(RoomType.objects.count()))
create_room_type(ROOMTYPE)
logger.info('RoomType Count: {}'.format(RoomType.objects.count()))
logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count()))
create_booking_group(BOOKING_GROUP)
logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count()))
logger.info('Equipment Count: {}'.format(Equipment.objects.count()))
create_equipment(EQUIPMENT)
logger.info('Equipment Count: {}'.format(Equipment.objects.count()))
logger.info('Staff Count: {}'.format(Staff.objects.count()))
create_staff()
logger.info('Staff Count: {}'.format(Staff.objects.count()))
logger.info('Room Count: {}'.format(Room.objects.count()))
create_room()
logger.info('Room Count: {}'.format(Room.objects.count()))
logger.info('Booking Count: {}'.format(Booking.objects.count()))
create_booking()
logger.info('Booking Count: {}'.format(Booking.objects.count()))
logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count()))
create_num_equipment()
logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count()))
logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count()))
create_access_point()
logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count()))
logger.info('Favorites Count: {}'.format(Favorite.objects.count()))
create_favorite()
logger.info('Favorites Count: {}'.format(Favorite.objects.count()))
def create_locations(names):
for name in names:
location, _ = Location.objects.get_or_create(name=name)
def create_buildings(names):
for name in names:
building, _ = Building.objects.get_or_create(name=name, location=random.choice(Location.objects.all()))
def create_room_type(types):
for type in types:
building, _ = RoomType.objects.get_or_create(type=type)
def create_booking_group(names):
for name in names:
booking_group, _ = BookingGroup.objects.get_or_create(name=name)
def create_equipment(names):
for name in names:
equipment, _ = Equipment.objects.get_or_create(name=name)
def create_staff():
for i in range(1, 30):
user, _ = User.objects.get_or_create(username='MannFrau{}'.format(i), email='mann{}@frau.de'.format(i),
password='1234abcdef#')
staff, _ = Staff.objects.get_or_create(user=random.choice(User.objects.all()))
staff.booking_group.add(random.choice(BookingGroup.objects.all()))
# booking_group = random.choice(BookingGroup.objects.all())
staff.save()
def create_room():
for i in range(1, 40):
room, _ = Room.objects.get_or_create(building=random.choice(Building.objects.all()),
room_number=random.randint(100, 956),
capacity=random.randint(10, 400), seating=random.randint(0, 1),
barrier_free=random.randint(0, 1), cooling=random.randint(0, 1),
room_type=random.choice(RoomType.objects.all()),
floor=random.randint(1, 6),
admin=random.choice(BookingGroup.objects.all()),
service_staff=random.choice(Staff.objects.all()))
def create_booking():
for i in range(1, 80):
year = random.choice(range(2018, 2019))
month = random.choice(range(1, 12))
day = random.choice(range(1, 28))
random_start_date = datetime.datetime(year, month, day)
delta = random.choice(range(0, 2))
random_enddate_date = random_start_date + datetime.timedelta(delta)
hour = random.choice(range(6, 22))
minute = random.choice(range(1, 59))
start_time = datetime.datetime(year, month, day, hour=hour, minute=minute)
delta = random.choice(range(0, 3))
end_time_time = start_time + datetime.timedelta(hours=delta)
booking, _ = Booking.objects.get_or_create(room=random.choice(Room.objects.all()),
staff=random.choice(Staff.objects.all()),
start_date=random_start_date, end_date=random_enddate_date,
start_time=start_time, end_time=end_time_time,
intervall=random.choice(range(0, 3)))
def create_num_equipment():
for i in range(1, 80):
num_equipment = NumEquipment.objects.get_or_create(room=random.choice(Room.objects.all()),
equipment=random.choice(Equipment.objects.all()),
count=random.choice(range(1, 3)))
def create_access_point():
for i in range(1, 25):
access_point, _ = AccessPoint.objects.get_or_create(mac_address=''.join(
random.choices(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'], k=12)))
for i in range(1, random.randint(1, 4)):
access_point.rooms.add(random.choice(Room.objects.all()))
access_point.save()
def create_favorite():
for staff in Staff.objects.all():
favorite, _ = Favorite.objects.get_or_create(staff=staff, room=random.choice(Room.objects.all()))

191
roofis2/roomservice/utils/migrate_data_lectures.py Normal file
View File

@ -0,0 +1,191 @@
from datetime import datetime
import json
from pprint import pprint
from django.db.utils import IntegrityError
from roomservice.utils.parser import univis_lectures_parser
from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType
import logging
logger = logging.getLogger(__name__)
# CONFIG Fakultaet
FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften"
FAKULTAET_HuWi = "Fakult%E4t%20Humanwissenschaften"
FAKULTAET_WIAI = "Fakult%E4t%20Wirtschaftsinformatik"
# CONFIG ROOMS
def univis_rooms(fakultaet):
return "http://univis.uni-bamberg.de/prg?search=rooms&department=" + fakultaet + "&show=xml"
# CONFIG LECTURES
def univis_lectures(fakultaet):
return "http://univis.uni-bamberg.de/prg?search=lectures&department=" + fakultaet + "&show=exml"
def getJsonFromFile(path):
with open(path, "r") as file:
return json.load(file)
def writeUnivisLectureTermsInDB(lecture):
logger.info('WTF')
if 'terms' in lecture:
if type(lecture['terms']['term']) == list:
for term in lecture['terms']['term']:
if 'exclude' in term:
logger.info('EXCLUDE: {}'.format(term['exclude']))
try:
starttime = "00:00"
# term_obj = Lecture_Terms.objects.create(starttime=starttime)
if 'starttime' in term:
starttime = term['starttime']
# term_obj.starttime = datetime.strptime(starttime, "%H:%M")
# term_obj.save()
if 'room' in term:
room_id = term['room']['UnivISRef']['@key']
# term_obj.room.add(Room.objects.get(key=room_id))
# lecture_obj.term.add(term_obj)
except IntegrityError as err:
logger.exception(err)
else:
try:
univis_starttime = "00:00"
# term_obj = Lecture_Terms.objects.create(starttime=univis_starttime)
if 'starttime' in lecture['terms']['term']:
univis_starttime = lecture['terms']['term']['starttime']
# term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M')
# term_obj.save()
if 'room' in lecture['terms']['term']:
room_id = lecture['terms']['term']['room']['UnivISRef']['@key']
# Room.objects.get(key=room_id)
# term_obj.room.add(Room.objects.get(key=room_id))
# term_obj.save()
# lecture_obj.term.add(term_obj)
except IntegrityError as err:
logger.exception(err)
def writeUnivisLectureDataInDB(data):
for lecture in data:
univis_key = lecture['@key']
univis_id = lecture['id']
name = lecture['name']
orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname'])
short = None
if 'short' in lecture:
short = lecture['short']
ects_cred = None
if 'ects_cred' in lecture:
ects_cred = float(str(lecture['ects_cred']).replace(',', '.'))
sws = None
if 'sws' in lecture:
sws = float(str(lecture['sws']).replace(',', '.'))
url_description = None
if 'url_description' in lecture:
url_description = lecture['url_description']
turnout = None
if 'turnout' in lecture:
turnout = lecture['turnout']
organizational = None
if 'organizational' in lecture:
organizational = lecture['organizational']
time_description = None
if 'time_description' in lecture:
time_description = lecture['time_description']
summary = None
if 'summary' in lecture:
summary = lecture['summary']
type = None
if 'type' in lecture:
type, _ = LectureType.objects.get_or_create(name=lecture['type'])
logger.info(name)
lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name,
orgname=orgname, short=short, ects=ects_cred,
sws=sws, url_description=url_description,
estimated_visitor=turnout, organizational=organizational,
time_description=time_description, summary=summary, type=type)
lecture_specs_options = LectureSpecifiaction.objects.all()
for lecture_spec in lecture_specs_options:
univis_id = lecture_spec.univis_id.lower()
if univis_id in lecture and lecture[univis_id] == 'ja':
lecture_obj.specification.add(lecture_spec)
orgunits = None
if 'orgunits' in lecture:
for orgunit in lecture['orgunits']:
if len(orgunit) > 0:
for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','):
cleaned_orgunit = orgunit.strip().strip("'").strip("'")
orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit)
lecture_obj.orgunits.add(orgunit)
# TODO: Check xml syntax #orgunit problem
pass
else:
orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit)
lecture_obj.orgunits.add(orgunit)
# logger.info(lecture_obj)
# if 'dozs' in lecture:
# lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key']
# lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short,
# type=lecture_type, lecturer_id=lecturer_id)
# writeUnivisLectureTermsInDB(lecture, lecture_obj)
# writeUnivisLectureTermsInDB(lecture)
# lecture_obj.save()
# logger.info("Lecture: {}".format(lecture_obj.short))
# except IntegrityError as err:
# logger.warning('Lecture already exists')
# logger.exception(err)
def showStatus(status: str):
return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Specifcation: {lecture_specs}\n\tLectureType: {lecture_type}\n\tOrg Unit: {orgunit}\n\n" \
.format(
status=status,
lectures=Lecture.objects.count(),
lecture_specs=LectureSpecifiaction.objects.count(),
lecture_type=LectureType.objects.count(),
orgunit=OrgUnit.objects.count()
)
def main():
# get food jsons
logger.info(showStatus("Start SoWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi)))
logger.info("----------------------------------------------------------------------------------------")
logger.info(showStatus("Start GuK:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK)))
logger.info("----------------------------------------------------------------------------------------")
logger.info(showStatus("Start HuWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi)))
logger.info("----------------------------------------------------------------------------------------")
logger.info(showStatus("Start WIAI:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI)))
pprint("----------------------------------------------------------------------------------------")
logger.info(showStatus("Finished:"))
if __name__ == '__main__':
main()

0
roofis2/roomservice/management/sample_data_creation/__init__.py → roofis2/roomservice/utils/parser/__init__.py
View File

29
roofis2/roomservice/utils/parser/univis_lectures_parser.py Normal file
View File

@ -0,0 +1,29 @@
import requests
import datetime
import xmltodict
import json
from pprint import pprint
def loadPage(url: str):
return requests.get(url).content
def getDay():
return datetime.datetime.today().strftime("%A, %d.%m.%Y")
def getLectures(dict: dict):
lectures = []
for lecture in dict['UnivIS']['Lecture']:
lectures.append(lecture)
return lectures
def parsePage(url):
page = loadPage(url)
dict = xmltodict.parse(page)
lectures = getLectures(dict)
return lectures
# parsePage( "http://univis.uni-bamberg.de/prg?search=lectures&department=Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften&show=exml")

27
roofis2/roomservice/utils/parser/univis_rooms_parser.py Normal file
View File

@ -0,0 +1,27 @@
import requests
import xmltodict
def loadPage(url: str):
return requests.get(url).content
def getRoom(dict: dict):
rooms = []
for room in dict['UnivIS']['Room']:
rooms.append(room)
return rooms
def getPersons(dict: dict):
persons = []
for person in dict['UnivIS']['Person']:
persons.append(person)
return persons
def parsePage(url):
page = loadPage(url)
dict = xmltodict.parse(page)
rooms = getRoom(dict)
return rooms