From 652e3d50eb1c91b7b6719f08b37d3bc44ef60cf4 Mon Sep 17 00:00:00 2001 From: MG Date: Tue, 19 Jun 2018 10:08:07 +0200 Subject: [PATCH] Implement lecture import --- roofis2/core/settings.py | 4 +- roofis2/requirements.txt | 3 +- .../management/commands/import_univis_data.py | 9 + .../commands/import_univis_lecture_specs.py | 31 +++ .../commands/import_univis_lecture_type.py | 55 +++++ .../commands/import_univis_terms.py | 9 + .../sample_data_creation/data_creator.py | 153 -------------- .../utils/migrate_data_lectures.py | 191 ++++++++++++++++++ .../parser}/__init__.py | 0 .../utils/parser/univis_lectures_parser.py | 29 +++ .../utils/parser/univis_rooms_parser.py | 27 +++ 11 files changed, 355 insertions(+), 156 deletions(-) create mode 100644 roofis2/roomservice/management/commands/import_univis_data.py create mode 100644 roofis2/roomservice/management/commands/import_univis_lecture_specs.py create mode 100644 roofis2/roomservice/management/commands/import_univis_lecture_type.py create mode 100644 roofis2/roomservice/management/commands/import_univis_terms.py delete mode 100644 roofis2/roomservice/management/sample_data_creation/data_creator.py create mode 100644 roofis2/roomservice/utils/migrate_data_lectures.py rename roofis2/roomservice/{management/sample_data_creation => utils/parser}/__init__.py (100%) create mode 100644 roofis2/roomservice/utils/parser/univis_lectures_parser.py create mode 100644 roofis2/roomservice/utils/parser/univis_rooms_parser.py diff --git a/roofis2/core/settings.py b/roofis2/core/settings.py index 77883a8..1d5f2a6 100644 --- a/roofis2/core/settings.py +++ b/roofis2/core/settings.py @@ -183,11 +183,11 @@ LOGGING = { # 'level': 'INFO', # }, - 'roomservice': { + 'roomservice.utils': { 'handlers': ['console'], 'level': 'INFO', }, - 'roomservice.management.sample_data_creation': { + 'roomservice.management': { 'handlers': ['console'], 'level': 'DEBUG' } diff --git a/roofis2/requirements.txt b/roofis2/requirements.txt index 6ef23da..2d9937c 100644 --- a/roofis2/requirements.txt +++ b/roofis2/requirements.txt @@ -6,4 +6,5 @@ Pillow==5.1.0 django-extensions==2.0.7 requests==2.18.4 djangorestframework==3.8.2 -coreapi==2.3.3 \ No newline at end of file +coreapi==2.3.3 +xmltodict==0.11.0 \ No newline at end of file diff --git a/roofis2/roomservice/management/commands/import_univis_data.py b/roofis2/roomservice/management/commands/import_univis_data.py new file mode 100644 index 0000000..7ff1ce7 --- /dev/null +++ b/roofis2/roomservice/management/commands/import_univis_data.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from roomservice.utils import migrate_data_rooms + + +class Command(BaseCommand): + help = "Import room data from univis prg api" + + def handle(self, *args, **options): + migrate_data_rooms.main() diff --git a/roofis2/roomservice/management/commands/import_univis_lecture_specs.py b/roofis2/roomservice/management/commands/import_univis_lecture_specs.py new file mode 100644 index 0000000..af2d182 --- /dev/null +++ b/roofis2/roomservice/management/commands/import_univis_lecture_specs.py @@ -0,0 +1,31 @@ +from django.core.management.base import BaseCommand +from roomservice.utils import migrate_data_rooms +from roomservice.models import LectureSpecifiaction + +import logging + +logger = logging.getLogger(__name__) + +UNIVIS_IDS_MAP = [{'id': 'OBLIG', 'name': 'obligatorische Lehrveranstaltung'}, + {'id': 'GENERALE', 'name': 'Studium Generale'}, + {'id': 'ENGLISH', 'name': 'Englischsprachig'}, + {'id': 'ETCS', 'name': 'ECTS-Studium'}, + {'id': 'WOMSPE', 'name': 'Frauenspezifisch/Geschlechtervergl'}, + {'id': 'ANPFLICHT', 'name': 'Anwesenheitspflicht'}, + {'id': 'MODULSTUD', 'name': 'Modulstudium'}, + {'id': 'FRUEH', 'name': 'Frühstudium'}, + {'id': 'ZENIS', 'name': 'Zentrum für Interreligiöse Studien'}, + {'id': 'BENSCHEIN', 'name': 'benoteter Schein'}, + {'id': 'ERWEI', 'name': 'Erweiterungsbereich'}, + {'id': 'ZENMAS', 'name': ' Zentrum für Mittelalterstudien'}, + ] + + +class Command(BaseCommand): + help = "Import room data from univis prg api" + + def handle(self, *args, **options): + logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count())) + for elem in UNIVIS_IDS_MAP: + equip, _ = LectureSpecifiaction.objects.update_or_create(univis_id=elem['id'], name=elem['name']) + logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count())) diff --git a/roofis2/roomservice/management/commands/import_univis_lecture_type.py b/roofis2/roomservice/management/commands/import_univis_lecture_type.py new file mode 100644 index 0000000..1d82a23 --- /dev/null +++ b/roofis2/roomservice/management/commands/import_univis_lecture_type.py @@ -0,0 +1,55 @@ +from django.core.management.base import BaseCommand +from roomservice.utils import migrate_data_rooms +from roomservice.models import LectureType + +import logging + +logger = logging.getLogger(__name__) + +UNIVIS_IDS_MAP = [{'id': 'V', 'name': 'Vorlesung'}, + + {'id': 'Vorlesung', 'name': 'Vorlesung'}, + {'id': 'Vorlesung/Seminar', 'name': 'Vorlesung/Seminar'}, + + {'id': 'Seminar', 'name': 'Seminar'}, + {'id': 'Hauptseminar', 'name': 'Hauptseminar'}, + {'id': 'Oberseminar', 'name': 'Oberseminar'}, + {'id': 'Blockseminar', 'name': 'Blockseminar'}, + {'id': 'Seminar/Oberseminar', 'name': 'Seminar/Oberseminar'}, + {'id': 'Seminaristischer Unterricht', 'name': 'Seminaristischer Unterricht'}, + {'id': 'Vertiefungsseminar', 'name': 'Vertiefungsseminar'}, + {'id': 'Proseminar', 'name': 'Proseminar'}, + {'id': 'Seminar/Hauptseminar', 'name': 'Seminar/Hauptseminar'}, + {'id': 'Forschungsseminar', 'name': 'Forschungsseminar'}, + + {'id': 'Ü', 'name': 'Übung'}, + {'id': 'Übung', 'name': 'Übung'}, + {'id': 'Übung/Blockseminar', 'name': 'Übung/Blockseminar'}, + {'id': 'Übung/Tutorium', 'name': 'Übung/Tutorium'}, + + {'id': 'Praktikum', 'name': 'Praktikum'}, + {'id': 'Forschungspraktikum', 'name': 'Forschungspraktikum'}, + {'id': 'Sonstige Lehrveranstaltung', 'name': 'Sonstige Lehrveranstaltung'}, + {'id': 'Tutorien', 'name': 'Tutorien'}, + + {'id': 'Kolloquium', 'name': 'Kolloquium'}, + {'id': 'Repetitorium', 'name': 'Repetitorium'}, + + {'id': 'Vorlesung und Übung', 'name': 'Vorlesung und Übung'}, + {'id': 'Seminar/Proseminar', 'name': 'Seminar/Proseminar'}, + {'id': 'Seminar/Übung', 'name': 'Seminar/Übung'}, + {'id': 'feldarchäologisches Praktikum', 'name': 'feldarchäologisches Praktikum'}, + {'id': 'Exkursion', 'name': 'Exkursion'}, + # {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'}, + # {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'}, + ] + + +class Command(BaseCommand): + help = "Import room data from univis prg api" + + def handle(self, *args, **options): + logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count())) + for elem in UNIVIS_IDS_MAP: + equip, _ = LectureType.objects.update_or_create(univis_id=elem['id'], name=elem['name']) + logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count())) diff --git a/roofis2/roomservice/management/commands/import_univis_terms.py b/roofis2/roomservice/management/commands/import_univis_terms.py new file mode 100644 index 0000000..1069af1 --- /dev/null +++ b/roofis2/roomservice/management/commands/import_univis_terms.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand +from roomservice.utils import migrate_data_lectures + + +class Command(BaseCommand): + help = "Import room data from univis prg api" + + def handle(self, *args, **options): + migrate_data_lectures.main() diff --git a/roofis2/roomservice/management/sample_data_creation/data_creator.py b/roofis2/roomservice/management/sample_data_creation/data_creator.py deleted file mode 100644 index 4bb32b7..0000000 --- a/roofis2/roomservice/management/sample_data_creation/data_creator.py +++ /dev/null @@ -1,153 +0,0 @@ -from roomservice.models import RoomType, Room, NumEquipment, Building, Location, Equipment, Booking, BookingGroup, \ - Staff, AccessPoint, Favorite -from django.contrib.auth.models import User -import logging -import random -import datetime - -logger = logging.getLogger(__name__) - -LOCATIONS = ['Bamberg', 'Erlangen', 'Crailsheim', 'Hamburg', 'Chemnitz', 'Dresden', 'Sebnitz', 'Gunzenhausen', - 'Nürnberg'] -BUILDINGS = ['253', '274', '134', '256', '142', '432', '652', '333', '233', '444', '332'] -ROOMTYPE = ['Office', 'Conference Room', 'Teaching Room'] -BOOKING_GROUP = ['Boss', 'SectionA', 'SectionB', 'SectionC', 'Standard'] -EQUIPMENT = ['Beamer', 'Flipchart'] - - -def create(): - logger.info('Location Count: {}'.format(Location.objects.count())) - create_locations(LOCATIONS) - logger.info('Location Count: {}'.format(Location.objects.count())) - - logger.info('Building Count: {}'.format(Building.objects.count())) - create_buildings(BUILDINGS) - logger.info('Building Count: {}'.format(Building.objects.count())) - - logger.info('RoomType Count: {}'.format(RoomType.objects.count())) - create_room_type(ROOMTYPE) - logger.info('RoomType Count: {}'.format(RoomType.objects.count())) - - logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count())) - create_booking_group(BOOKING_GROUP) - logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count())) - - logger.info('Equipment Count: {}'.format(Equipment.objects.count())) - create_equipment(EQUIPMENT) - logger.info('Equipment Count: {}'.format(Equipment.objects.count())) - - logger.info('Staff Count: {}'.format(Staff.objects.count())) - create_staff() - logger.info('Staff Count: {}'.format(Staff.objects.count())) - - logger.info('Room Count: {}'.format(Room.objects.count())) - create_room() - logger.info('Room Count: {}'.format(Room.objects.count())) - - logger.info('Booking Count: {}'.format(Booking.objects.count())) - create_booking() - logger.info('Booking Count: {}'.format(Booking.objects.count())) - - logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count())) - create_num_equipment() - logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count())) - - logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count())) - create_access_point() - logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count())) - - logger.info('Favorites Count: {}'.format(Favorite.objects.count())) - create_favorite() - logger.info('Favorites Count: {}'.format(Favorite.objects.count())) - - -def create_locations(names): - for name in names: - location, _ = Location.objects.get_or_create(name=name) - - -def create_buildings(names): - for name in names: - building, _ = Building.objects.get_or_create(name=name, location=random.choice(Location.objects.all())) - - -def create_room_type(types): - for type in types: - building, _ = RoomType.objects.get_or_create(type=type) - - -def create_booking_group(names): - for name in names: - booking_group, _ = BookingGroup.objects.get_or_create(name=name) - - -def create_equipment(names): - for name in names: - equipment, _ = Equipment.objects.get_or_create(name=name) - - -def create_staff(): - for i in range(1, 30): - user, _ = User.objects.get_or_create(username='MannFrau{}'.format(i), email='mann{}@frau.de'.format(i), - password='1234abcdef#') - staff, _ = Staff.objects.get_or_create(user=random.choice(User.objects.all())) - staff.booking_group.add(random.choice(BookingGroup.objects.all())) - # booking_group = random.choice(BookingGroup.objects.all()) - staff.save() - - -def create_room(): - for i in range(1, 40): - room, _ = Room.objects.get_or_create(building=random.choice(Building.objects.all()), - room_number=random.randint(100, 956), - capacity=random.randint(10, 400), seating=random.randint(0, 1), - barrier_free=random.randint(0, 1), cooling=random.randint(0, 1), - room_type=random.choice(RoomType.objects.all()), - floor=random.randint(1, 6), - admin=random.choice(BookingGroup.objects.all()), - service_staff=random.choice(Staff.objects.all())) - - -def create_booking(): - for i in range(1, 80): - year = random.choice(range(2018, 2019)) - month = random.choice(range(1, 12)) - day = random.choice(range(1, 28)) - random_start_date = datetime.datetime(year, month, day) - - delta = random.choice(range(0, 2)) - random_enddate_date = random_start_date + datetime.timedelta(delta) - - hour = random.choice(range(6, 22)) - minute = random.choice(range(1, 59)) - start_time = datetime.datetime(year, month, day, hour=hour, minute=minute) - - delta = random.choice(range(0, 3)) - end_time_time = start_time + datetime.timedelta(hours=delta) - - booking, _ = Booking.objects.get_or_create(room=random.choice(Room.objects.all()), - staff=random.choice(Staff.objects.all()), - start_date=random_start_date, end_date=random_enddate_date, - start_time=start_time, end_time=end_time_time, - intervall=random.choice(range(0, 3))) - - -def create_num_equipment(): - for i in range(1, 80): - num_equipment = NumEquipment.objects.get_or_create(room=random.choice(Room.objects.all()), - equipment=random.choice(Equipment.objects.all()), - count=random.choice(range(1, 3))) - - -def create_access_point(): - for i in range(1, 25): - access_point, _ = AccessPoint.objects.get_or_create(mac_address=''.join( - random.choices(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'], k=12))) - for i in range(1, random.randint(1, 4)): - access_point.rooms.add(random.choice(Room.objects.all())) - access_point.save() - - -def create_favorite(): - for staff in Staff.objects.all(): - favorite, _ = Favorite.objects.get_or_create(staff=staff, room=random.choice(Room.objects.all())) diff --git a/roofis2/roomservice/utils/migrate_data_lectures.py b/roofis2/roomservice/utils/migrate_data_lectures.py new file mode 100644 index 0000000..bce1a76 --- /dev/null +++ b/roofis2/roomservice/utils/migrate_data_lectures.py @@ -0,0 +1,191 @@ +from datetime import datetime +import json +from pprint import pprint +from django.db.utils import IntegrityError +from roomservice.utils.parser import univis_lectures_parser +from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType + +import logging + +logger = logging.getLogger(__name__) + +# CONFIG Fakultaet +FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften" +FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften" +FAKULTAET_HuWi = "Fakult%E4t%20Humanwissenschaften" +FAKULTAET_WIAI = "Fakult%E4t%20Wirtschaftsinformatik" + + +# CONFIG ROOMS +def univis_rooms(fakultaet): + return "http://univis.uni-bamberg.de/prg?search=rooms&department=" + fakultaet + "&show=xml" + + +# CONFIG LECTURES +def univis_lectures(fakultaet): + return "http://univis.uni-bamberg.de/prg?search=lectures&department=" + fakultaet + "&show=exml" + + +def getJsonFromFile(path): + with open(path, "r") as file: + return json.load(file) + + +def writeUnivisLectureTermsInDB(lecture): + logger.info('WTF') + if 'terms' in lecture: + if type(lecture['terms']['term']) == list: + for term in lecture['terms']['term']: + if 'exclude' in term: + logger.info('EXCLUDE: {}'.format(term['exclude'])) + try: + starttime = "00:00" + # term_obj = Lecture_Terms.objects.create(starttime=starttime) + if 'starttime' in term: + starttime = term['starttime'] + # term_obj.starttime = datetime.strptime(starttime, "%H:%M") + # term_obj.save() + if 'room' in term: + room_id = term['room']['UnivISRef']['@key'] + # term_obj.room.add(Room.objects.get(key=room_id)) + # lecture_obj.term.add(term_obj) + except IntegrityError as err: + logger.exception(err) + + else: + try: + univis_starttime = "00:00" + # term_obj = Lecture_Terms.objects.create(starttime=univis_starttime) + if 'starttime' in lecture['terms']['term']: + univis_starttime = lecture['terms']['term']['starttime'] + # term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M') + # term_obj.save() + if 'room' in lecture['terms']['term']: + room_id = lecture['terms']['term']['room']['UnivISRef']['@key'] + # Room.objects.get(key=room_id) + # term_obj.room.add(Room.objects.get(key=room_id)) + # term_obj.save() + # lecture_obj.term.add(term_obj) + except IntegrityError as err: + logger.exception(err) + + +def writeUnivisLectureDataInDB(data): + for lecture in data: + univis_key = lecture['@key'] + univis_id = lecture['id'] + name = lecture['name'] + orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname']) + + short = None + if 'short' in lecture: + short = lecture['short'] + + ects_cred = None + if 'ects_cred' in lecture: + ects_cred = float(str(lecture['ects_cred']).replace(',', '.')) + + sws = None + if 'sws' in lecture: + sws = float(str(lecture['sws']).replace(',', '.')) + + url_description = None + if 'url_description' in lecture: + url_description = lecture['url_description'] + + turnout = None + if 'turnout' in lecture: + turnout = lecture['turnout'] + + organizational = None + if 'organizational' in lecture: + organizational = lecture['organizational'] + + time_description = None + if 'time_description' in lecture: + time_description = lecture['time_description'] + + summary = None + if 'summary' in lecture: + summary = lecture['summary'] + + type = None + if 'type' in lecture: + type, _ = LectureType.objects.get_or_create(name=lecture['type']) + + logger.info(name) + lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name, + orgname=orgname, short=short, ects=ects_cred, + sws=sws, url_description=url_description, + estimated_visitor=turnout, organizational=organizational, + time_description=time_description, summary=summary, type=type) + + lecture_specs_options = LectureSpecifiaction.objects.all() + for lecture_spec in lecture_specs_options: + univis_id = lecture_spec.univis_id.lower() + if univis_id in lecture and lecture[univis_id] == 'ja': + lecture_obj.specification.add(lecture_spec) + + orgunits = None + if 'orgunits' in lecture: + for orgunit in lecture['orgunits']: + if len(orgunit) > 0: + for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','): + cleaned_orgunit = orgunit.strip().strip("'").strip("'") + orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit) + lecture_obj.orgunits.add(orgunit) + + # TODO: Check xml syntax #orgunit problem + pass + else: + orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit) + lecture_obj.orgunits.add(orgunit) + + # logger.info(lecture_obj) + # if 'dozs' in lecture: + # lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key'] + # lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short, + # type=lecture_type, lecturer_id=lecturer_id) + # writeUnivisLectureTermsInDB(lecture, lecture_obj) + # writeUnivisLectureTermsInDB(lecture) + # lecture_obj.save() + # logger.info("Lecture: {}".format(lecture_obj.short)) + # except IntegrityError as err: + # logger.warning('Lecture already exists') + # logger.exception(err) + + +def showStatus(status: str): + return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Specifcation: {lecture_specs}\n\tLectureType: {lecture_type}\n\tOrg Unit: {orgunit}\n\n" \ + .format( + status=status, + lectures=Lecture.objects.count(), + lecture_specs=LectureSpecifiaction.objects.count(), + lecture_type=LectureType.objects.count(), + orgunit=OrgUnit.objects.count() + ) + + +def main(): + # get food jsons + logger.info(showStatus("Start SoWi:")) + writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi))) + logger.info("----------------------------------------------------------------------------------------") + + logger.info(showStatus("Start GuK:")) + writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK))) + logger.info("----------------------------------------------------------------------------------------") + + logger.info(showStatus("Start HuWi:")) + writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi))) + logger.info("----------------------------------------------------------------------------------------") + + logger.info(showStatus("Start WIAI:")) + writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI))) + pprint("----------------------------------------------------------------------------------------") + + logger.info(showStatus("Finished:")) + + +if __name__ == '__main__': + main() diff --git a/roofis2/roomservice/management/sample_data_creation/__init__.py b/roofis2/roomservice/utils/parser/__init__.py similarity index 100% rename from roofis2/roomservice/management/sample_data_creation/__init__.py rename to roofis2/roomservice/utils/parser/__init__.py diff --git a/roofis2/roomservice/utils/parser/univis_lectures_parser.py b/roofis2/roomservice/utils/parser/univis_lectures_parser.py new file mode 100644 index 0000000..10ae33a --- /dev/null +++ b/roofis2/roomservice/utils/parser/univis_lectures_parser.py @@ -0,0 +1,29 @@ +import requests +import datetime +import xmltodict +import json +from pprint import pprint + + +def loadPage(url: str): + return requests.get(url).content + + +def getDay(): + return datetime.datetime.today().strftime("%A, %d.%m.%Y") + + +def getLectures(dict: dict): + lectures = [] + for lecture in dict['UnivIS']['Lecture']: + lectures.append(lecture) + return lectures + + +def parsePage(url): + page = loadPage(url) + dict = xmltodict.parse(page) + lectures = getLectures(dict) + return lectures + +# parsePage( "http://univis.uni-bamberg.de/prg?search=lectures&department=Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften&show=exml") diff --git a/roofis2/roomservice/utils/parser/univis_rooms_parser.py b/roofis2/roomservice/utils/parser/univis_rooms_parser.py new file mode 100644 index 0000000..21514ce --- /dev/null +++ b/roofis2/roomservice/utils/parser/univis_rooms_parser.py @@ -0,0 +1,27 @@ +import requests +import xmltodict + + +def loadPage(url: str): + return requests.get(url).content + + +def getRoom(dict: dict): + rooms = [] + for room in dict['UnivIS']['Room']: + rooms.append(room) + return rooms + + +def getPersons(dict: dict): + persons = [] + for person in dict['UnivIS']['Person']: + persons.append(person) + return persons + + +def parsePage(url): + page = loadPage(url) + dict = xmltodict.parse(page) + rooms = getRoom(dict) + return rooms