Implement lecture import

roofis2/core/settings.py

@@ -183,11 +183,11 @@ LOGGING = {
         #     'level': 'INFO',
         # },
 
-        'roomservice': {
+        'roomservice.utils': {
             'handlers': ['console'],
             'level': 'INFO',
         },
-        'roomservice.management.sample_data_creation': {
+        'roomservice.management': {
             'handlers': ['console'],
             'level': 'DEBUG'
         }

roofis2/requirements.txt

@@ -6,4 +6,5 @@ Pillow==5.1.0
 django-extensions==2.0.7
 requests==2.18.4
 djangorestframework==3.8.2
-coreapi==2.3.3
+coreapi==2.3.3
+xmltodict==0.11.0

roofis2/roomservice/management/commands/import_univis_data.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from roomservice.utils import migrate_data_rooms
+
+
+class Command(BaseCommand):
+    help = "Import room data from univis prg api"
+
+    def handle(self, *args, **options):
+        migrate_data_rooms.main()

roofis2/roomservice/management/commands/import_univis_lecture_specs.py

@@ -0,0 +1,31 @@
+from django.core.management.base import BaseCommand
+from roomservice.utils import migrate_data_rooms
+from roomservice.models import LectureSpecifiaction
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+UNIVIS_IDS_MAP = [{'id': 'OBLIG', 'name': 'obligatorische Lehrveranstaltung'},
+                  {'id': 'GENERALE', 'name': 'Studium Generale'},
+                  {'id': 'ENGLISH', 'name': 'Englischsprachig'},
+                  {'id': 'ETCS', 'name': 'ECTS-Studium'},
+                  {'id': 'WOMSPE', 'name': 'Frauenspezifisch/Geschlechtervergl'},
+                  {'id': 'ANPFLICHT', 'name': 'Anwesenheitspflicht'},
+                  {'id': 'MODULSTUD', 'name': 'Modulstudium'},
+                  {'id': 'FRUEH', 'name': 'Frühstudium'},
+                  {'id': 'ZENIS', 'name': 'Zentrum für Interreligiöse Studien'},
+                  {'id': 'BENSCHEIN', 'name': 'benoteter Schein'},
+                  {'id': 'ERWEI', 'name': 'Erweiterungsbereich'},
+                  {'id': 'ZENMAS', 'name': ' Zentrum für Mittelalterstudien'},
+                  ]
+
+
+class Command(BaseCommand):
+    help = "Import room data from univis prg api"
+
+    def handle(self, *args, **options):
+        logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count()))
+        for elem in UNIVIS_IDS_MAP:
+            equip, _ = LectureSpecifiaction.objects.update_or_create(univis_id=elem['id'], name=elem['name'])
+        logger.info("Start:\nLecture Specs: {}".format(LectureSpecifiaction.objects.count()))

roofis2/roomservice/management/commands/import_univis_lecture_type.py

@@ -0,0 +1,55 @@
+from django.core.management.base import BaseCommand
+from roomservice.utils import migrate_data_rooms
+from roomservice.models import LectureType
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+UNIVIS_IDS_MAP = [{'id': 'V', 'name': 'Vorlesung'},
+
+                  {'id': 'Vorlesung', 'name': 'Vorlesung'},
+                  {'id': 'Vorlesung/Seminar', 'name': 'Vorlesung/Seminar'},
+
+                  {'id': 'Seminar', 'name': 'Seminar'},
+                  {'id': 'Hauptseminar', 'name': 'Hauptseminar'},
+                  {'id': 'Oberseminar', 'name': 'Oberseminar'},
+                  {'id': 'Blockseminar', 'name': 'Blockseminar'},
+                  {'id': 'Seminar/Oberseminar', 'name': 'Seminar/Oberseminar'},
+                  {'id': 'Seminaristischer Unterricht', 'name': 'Seminaristischer Unterricht'},
+                  {'id': 'Vertiefungsseminar', 'name': 'Vertiefungsseminar'},
+                  {'id': 'Proseminar', 'name': 'Proseminar'},
+                  {'id': 'Seminar/Hauptseminar', 'name': 'Seminar/Hauptseminar'},
+                  {'id': 'Forschungsseminar', 'name': 'Forschungsseminar'},
+
+                  {'id': 'Ü', 'name': 'Übung'},
+                  {'id': 'Übung', 'name': 'Übung'},
+                  {'id': 'Übung/Blockseminar', 'name': 'Übung/Blockseminar'},
+                  {'id': 'Übung/Tutorium', 'name': 'Übung/Tutorium'},
+
+                  {'id': 'Praktikum', 'name': 'Praktikum'},
+                  {'id': 'Forschungspraktikum', 'name': 'Forschungspraktikum'},
+                  {'id': 'Sonstige Lehrveranstaltung', 'name': 'Sonstige Lehrveranstaltung'},
+                  {'id': 'Tutorien', 'name': 'Tutorien'},
+
+                  {'id': 'Kolloquium', 'name': 'Kolloquium'},
+                  {'id': 'Repetitorium', 'name': 'Repetitorium'},
+
+                  {'id': 'Vorlesung und Übung', 'name': 'Vorlesung und Übung'},
+                  {'id': 'Seminar/Proseminar', 'name': 'Seminar/Proseminar'},
+                  {'id': 'Seminar/Übung', 'name': 'Seminar/Übung'},
+                  {'id': 'feldarchäologisches Praktikum', 'name': 'feldarchäologisches Praktikum'},
+                  {'id': 'Exkursion', 'name': 'Exkursion'},
+                  # {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'},
+                  # {'id': 'Klausurenkurs', 'name': 'Klausurenkurs'},
+                  ]
+
+
+class Command(BaseCommand):
+    help = "Import room data from univis prg api"
+
+    def handle(self, *args, **options):
+        logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count()))
+        for elem in UNIVIS_IDS_MAP:
+            equip, _ = LectureType.objects.update_or_create(univis_id=elem['id'], name=elem['name'])
+        logger.info("Start:\nLecture Types: {}".format(LectureType.objects.count()))

roofis2/roomservice/management/commands/import_univis_terms.py

@@ -0,0 +1,9 @@
+from django.core.management.base import BaseCommand
+from roomservice.utils import migrate_data_lectures
+
+
+class Command(BaseCommand):
+    help = "Import room data from univis prg api"
+
+    def handle(self, *args, **options):
+        migrate_data_lectures.main()

roofis2/roomservice/management/sample_data_creation/data_creator.py

@@ -1,153 +0,0 @@
-from roomservice.models import RoomType, Room, NumEquipment, Building, Location, Equipment, Booking, BookingGroup, \
-    Staff, AccessPoint, Favorite
-from django.contrib.auth.models import User
-import logging
-import random
-import datetime
-
-logger = logging.getLogger(__name__)
-
-LOCATIONS = ['Bamberg', 'Erlangen', 'Crailsheim', 'Hamburg', 'Chemnitz', 'Dresden', 'Sebnitz', 'Gunzenhausen',
-             'Nürnberg']
-BUILDINGS = ['253', '274', '134', '256', '142', '432', '652', '333', '233', '444', '332']
-ROOMTYPE = ['Office', 'Conference Room', 'Teaching Room']
-BOOKING_GROUP = ['Boss', 'SectionA', 'SectionB', 'SectionC', 'Standard']
-EQUIPMENT = ['Beamer', 'Flipchart']
-
-
-def create():
-    logger.info('Location Count: {}'.format(Location.objects.count()))
-    create_locations(LOCATIONS)
-    logger.info('Location Count: {}'.format(Location.objects.count()))
-
-    logger.info('Building Count: {}'.format(Building.objects.count()))
-    create_buildings(BUILDINGS)
-    logger.info('Building Count: {}'.format(Building.objects.count()))
-
-    logger.info('RoomType Count: {}'.format(RoomType.objects.count()))
-    create_room_type(ROOMTYPE)
-    logger.info('RoomType Count: {}'.format(RoomType.objects.count()))
-
-    logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count()))
-    create_booking_group(BOOKING_GROUP)
-    logger.info('BookingGroup Count: {}'.format(BookingGroup.objects.count()))
-
-    logger.info('Equipment Count: {}'.format(Equipment.objects.count()))
-    create_equipment(EQUIPMENT)
-    logger.info('Equipment Count: {}'.format(Equipment.objects.count()))
-
-    logger.info('Staff Count: {}'.format(Staff.objects.count()))
-    create_staff()
-    logger.info('Staff Count: {}'.format(Staff.objects.count()))
-
-    logger.info('Room Count: {}'.format(Room.objects.count()))
-    create_room()
-    logger.info('Room Count: {}'.format(Room.objects.count()))
-
-    logger.info('Booking Count: {}'.format(Booking.objects.count()))
-    create_booking()
-    logger.info('Booking Count: {}'.format(Booking.objects.count()))
-
-    logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count()))
-    create_num_equipment()
-    logger.info('NumEquipment Count: {}'.format(NumEquipment.objects.count()))
-
-    logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count()))
-    create_access_point()
-    logger.info('AccessPoint Count: {}'.format(AccessPoint.objects.count()))
-
-    logger.info('Favorites Count: {}'.format(Favorite.objects.count()))
-    create_favorite()
-    logger.info('Favorites Count: {}'.format(Favorite.objects.count()))
-
-
-def create_locations(names):
-    for name in names:
-        location, _ = Location.objects.get_or_create(name=name)
-
-
-def create_buildings(names):
-    for name in names:
-        building, _ = Building.objects.get_or_create(name=name, location=random.choice(Location.objects.all()))
-
-
-def create_room_type(types):
-    for type in types:
-        building, _ = RoomType.objects.get_or_create(type=type)
-
-
-def create_booking_group(names):
-    for name in names:
-        booking_group, _ = BookingGroup.objects.get_or_create(name=name)
-
-
-def create_equipment(names):
-    for name in names:
-        equipment, _ = Equipment.objects.get_or_create(name=name)
-
-
-def create_staff():
-    for i in range(1, 30):
-        user, _ = User.objects.get_or_create(username='MannFrau{}'.format(i), email='mann{}@frau.de'.format(i),
-                                             password='1234abcdef#')
-        staff, _ = Staff.objects.get_or_create(user=random.choice(User.objects.all()))
-        staff.booking_group.add(random.choice(BookingGroup.objects.all()))
-        # booking_group = random.choice(BookingGroup.objects.all())
-        staff.save()
-
-
-def create_room():
-    for i in range(1, 40):
-        room, _ = Room.objects.get_or_create(building=random.choice(Building.objects.all()),
-                                             room_number=random.randint(100, 956),
-                                             capacity=random.randint(10, 400), seating=random.randint(0, 1),
-                                             barrier_free=random.randint(0, 1), cooling=random.randint(0, 1),
-                                             room_type=random.choice(RoomType.objects.all()),
-                                             floor=random.randint(1, 6),
-                                             admin=random.choice(BookingGroup.objects.all()),
-                                             service_staff=random.choice(Staff.objects.all()))
-
-
-def create_booking():
-    for i in range(1, 80):
-        year = random.choice(range(2018, 2019))
-        month = random.choice(range(1, 12))
-        day = random.choice(range(1, 28))
-        random_start_date = datetime.datetime(year, month, day)
-
-        delta = random.choice(range(0, 2))
-        random_enddate_date = random_start_date + datetime.timedelta(delta)
-
-        hour = random.choice(range(6, 22))
-        minute = random.choice(range(1, 59))
-        start_time = datetime.datetime(year, month, day, hour=hour, minute=minute)
-
-        delta = random.choice(range(0, 3))
-        end_time_time = start_time + datetime.timedelta(hours=delta)
-
-        booking, _ = Booking.objects.get_or_create(room=random.choice(Room.objects.all()),
-                                                   staff=random.choice(Staff.objects.all()),
-                                                   start_date=random_start_date, end_date=random_enddate_date,
-                                                   start_time=start_time, end_time=end_time_time,
-                                                   intervall=random.choice(range(0, 3)))
-
-
-def create_num_equipment():
-    for i in range(1, 80):
-        num_equipment = NumEquipment.objects.get_or_create(room=random.choice(Room.objects.all()),
-                                                           equipment=random.choice(Equipment.objects.all()),
-                                                           count=random.choice(range(1, 3)))
-
-
-def create_access_point():
-    for i in range(1, 25):
-        access_point, _ = AccessPoint.objects.get_or_create(mac_address=''.join(
-            random.choices(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'], k=12)))
-        for i in range(1, random.randint(1, 4)):
-            access_point.rooms.add(random.choice(Room.objects.all()))
-        access_point.save()
-
-
-def create_favorite():
-    for staff in Staff.objects.all():
-        favorite, _ = Favorite.objects.get_or_create(staff=staff, room=random.choice(Room.objects.all()))

roofis2/roomservice/utils/migrate_data_lectures.py

@@ -0,0 +1,191 @@
+from datetime import datetime
+import json
+from pprint import pprint
+from django.db.utils import IntegrityError
+from roomservice.utils.parser import univis_lectures_parser
+from roomservice.models import Lecture, LectureSpecifiaction, OrgUnit, LectureType
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+# CONFIG Fakultaet
+FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
+FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften"
+FAKULTAET_HuWi = "Fakult%E4t%20Humanwissenschaften"
+FAKULTAET_WIAI = "Fakult%E4t%20Wirtschaftsinformatik"
+
+
+# CONFIG ROOMS
+def univis_rooms(fakultaet):
+    return "http://univis.uni-bamberg.de/prg?search=rooms&department=" + fakultaet + "&show=xml"
+
+
+# CONFIG LECTURES
+def univis_lectures(fakultaet):
+    return "http://univis.uni-bamberg.de/prg?search=lectures&department=" + fakultaet + "&show=exml"
+
+
+def getJsonFromFile(path):
+    with open(path, "r") as file:
+        return json.load(file)
+
+
+def writeUnivisLectureTermsInDB(lecture):
+    logger.info('WTF')
+    if 'terms' in lecture:
+        if type(lecture['terms']['term']) == list:
+            for term in lecture['terms']['term']:
+                if 'exclude' in term:
+                    logger.info('EXCLUDE: {}'.format(term['exclude']))
+                try:
+                    starttime = "00:00"
+                    # term_obj = Lecture_Terms.objects.create(starttime=starttime)
+                    if 'starttime' in term:
+                        starttime = term['starttime']
+                        # term_obj.starttime = datetime.strptime(starttime, "%H:%M")
+                    # term_obj.save()
+                    if 'room' in term:
+                        room_id = term['room']['UnivISRef']['@key']
+                        # term_obj.room.add(Room.objects.get(key=room_id))
+                    # lecture_obj.term.add(term_obj)
+                except IntegrityError as err:
+                    logger.exception(err)
+
+        else:
+            try:
+                univis_starttime = "00:00"
+                # term_obj = Lecture_Terms.objects.create(starttime=univis_starttime)
+                if 'starttime' in lecture['terms']['term']:
+                    univis_starttime = lecture['terms']['term']['starttime']
+                    # term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M')
+                # term_obj.save()
+                if 'room' in lecture['terms']['term']:
+                    room_id = lecture['terms']['term']['room']['UnivISRef']['@key']
+                    # Room.objects.get(key=room_id)
+                    # term_obj.room.add(Room.objects.get(key=room_id))
+                # term_obj.save()
+                # lecture_obj.term.add(term_obj)
+            except IntegrityError as err:
+                logger.exception(err)
+
+
+def writeUnivisLectureDataInDB(data):
+    for lecture in data:
+        univis_key = lecture['@key']
+        univis_id = lecture['id']
+        name = lecture['name']
+        orgname, _ = OrgUnit.objects.get_or_create(title=lecture['orgname'])
+
+        short = None
+        if 'short' in lecture:
+            short = lecture['short']
+
+        ects_cred = None
+        if 'ects_cred' in lecture:
+            ects_cred = float(str(lecture['ects_cred']).replace(',', '.'))
+
+        sws = None
+        if 'sws' in lecture:
+            sws = float(str(lecture['sws']).replace(',', '.'))
+
+        url_description = None
+        if 'url_description' in lecture:
+            url_description = lecture['url_description']
+
+        turnout = None
+        if 'turnout' in lecture:
+            turnout = lecture['turnout']
+
+        organizational = None
+        if 'organizational' in lecture:
+            organizational = lecture['organizational']
+
+        time_description = None
+        if 'time_description' in lecture:
+            time_description = lecture['time_description']
+
+        summary = None
+        if 'summary' in lecture:
+            summary = lecture['summary']
+
+        type = None
+        if 'type' in lecture:
+            type, _ = LectureType.objects.get_or_create(name=lecture['type'])
+
+        logger.info(name)
+        lecture_obj, _ = Lecture.objects.update_or_create(univis_id=univis_id, univis_key=univis_key, title=name,
+                                                          orgname=orgname, short=short, ects=ects_cred,
+                                                          sws=sws, url_description=url_description,
+                                                          estimated_visitor=turnout, organizational=organizational,
+                                                          time_description=time_description, summary=summary, type=type)
+
+        lecture_specs_options = LectureSpecifiaction.objects.all()
+        for lecture_spec in lecture_specs_options:
+            univis_id = lecture_spec.univis_id.lower()
+            if univis_id in lecture and lecture[univis_id] == 'ja':
+                lecture_obj.specification.add(lecture_spec)
+
+        orgunits = None
+        if 'orgunits' in lecture:
+            for orgunit in lecture['orgunits']:
+                if len(orgunit) > 0:
+                    for orgunit in str(lecture['orgunits']['orgunit']).strip("[]").split(','):
+                        cleaned_orgunit = orgunit.strip().strip("'").strip("'")
+                        orgunit, _ = OrgUnit.objects.get_or_create(title=cleaned_orgunit)
+                        lecture_obj.orgunits.add(orgunit)
+
+                    # TODO: Check xml syntax #orgunit problem
+                    pass
+                else:
+                    orgunit, _ = OrgUnit.objects.get_or_create(title=orgunit)
+                    lecture_obj.orgunits.add(orgunit)
+
+        # logger.info(lecture_obj)
+        #     if 'dozs' in lecture:
+        #         lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key']
+        # lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short,
+        #                                      type=lecture_type, lecturer_id=lecturer_id)
+        # writeUnivisLectureTermsInDB(lecture, lecture_obj)
+        # writeUnivisLectureTermsInDB(lecture)
+        # lecture_obj.save()
+        # logger.info("Lecture: {}".format(lecture_obj.short))
+        # except IntegrityError as err:
+        #     logger.warning('Lecture already exists')
+        # logger.exception(err)
+
+
+def showStatus(status: str):
+    return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Specifcation: {lecture_specs}\n\tLectureType: {lecture_type}\n\tOrg Unit: {orgunit}\n\n" \
+        .format(
+        status=status,
+        lectures=Lecture.objects.count(),
+        lecture_specs=LectureSpecifiaction.objects.count(),
+        lecture_type=LectureType.objects.count(),
+        orgunit=OrgUnit.objects.count()
+    )
+
+
+def main():
+    # get food jsons
+    logger.info(showStatus("Start SoWi:"))
+    writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi)))
+    logger.info("----------------------------------------------------------------------------------------")
+
+    logger.info(showStatus("Start GuK:"))
+    writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK)))
+    logger.info("----------------------------------------------------------------------------------------")
+
+    logger.info(showStatus("Start HuWi:"))
+    writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi)))
+    logger.info("----------------------------------------------------------------------------------------")
+
+    logger.info(showStatus("Start WIAI:"))
+    writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI)))
+    pprint("----------------------------------------------------------------------------------------")
+
+    logger.info(showStatus("Finished:"))
+
+
+if __name__ == '__main__':
+    main()

roofis2/roomservice/utils/parser/univis_lectures_parser.py

@@ -0,0 +1,29 @@
+import requests
+import datetime
+import xmltodict
+import json
+from pprint import pprint
+
+
+def loadPage(url: str):
+    return requests.get(url).content
+
+
+def getDay():
+    return datetime.datetime.today().strftime("%A, %d.%m.%Y")
+
+
+def getLectures(dict: dict):
+    lectures = []
+    for lecture in dict['UnivIS']['Lecture']:
+        lectures.append(lecture)
+    return lectures
+
+
+def parsePage(url):
+    page = loadPage(url)
+    dict = xmltodict.parse(page)
+    lectures = getLectures(dict)
+    return lectures
+
+# parsePage(    "http://univis.uni-bamberg.de/prg?search=lectures&department=Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften&show=exml")

roofis2/roomservice/utils/parser/univis_rooms_parser.py

@@ -0,0 +1,27 @@
+import requests
+import xmltodict
+
+
+def loadPage(url: str):
+    return requests.get(url).content
+
+
+def getRoom(dict: dict):
+    rooms = []
+    for room in dict['UnivIS']['Room']:
+        rooms.append(room)
+    return rooms
+
+
+def getPersons(dict: dict):
+    persons = []
+    for person in dict['UnivIS']['Person']:
+        persons.append(person)
+    return persons
+
+
+def parsePage(url):
+    page = loadPage(url)
+    dict = xmltodict.parse(page)
+    rooms = getRoom(dict)
+    return rooms