mgoetz/basta-server
Archived
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add logging, Refractor importer

Browse Source
This commit is contained in:
michigg 2018-04-01 17:35:55 +02:00
parent e9bc6da951
commit 213496fa3e
12 changed files with 247 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ofu_app/apps/donar/management/commands/delete_lectures.py Normal file
View File

@ -0,0 +1,13 @@
from django.core.management.base import BaseCommand, CommandError
from apps.donar.models import Room
from apps.donar.utils import migrate_data_lectures
class Command(BaseCommand):
help = "Imports Lectures from UnivIS PRG. Requires Room import"
def add_arguments(self, parser):
pass
def handle(self, *args, **options):
migrate_data_lectures.delete()

13
ofu_app/apps/donar/management/commands/delete_rooms.py Normal file
View File

@ -0,0 +1,13 @@
from django.core.management.base import BaseCommand, CommandError
from apps.donar.models import Room
from apps.donar.utils import migrate_data_rooms
class Command(BaseCommand):
help = "Imports Rooms from Univis PRG"
def add_arguments(self, parser):
pass
def handle(self, *args, **options):
migrate_data_rooms.delete()

18
ofu_app/apps/donar/migrations/0005_auto_20180401_1136.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.0.1 on 2018-04-01 11:36
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('donar', '0004_auto_20180117_0137'),
]
operations = [
migrations.AlterField(
model_name='lecture',
name='short',
field=models.CharField(max_length=256),
),
]

28
ofu_app/apps/donar/migrations/0006_auto_20180401_1139.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Django 2.0.1 on 2018-04-01 11:39
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('donar', '0005_auto_20180401_1136'),
]
operations = [
migrations.AlterField(
model_name='lecture',
name='lecturer_id',
field=models.CharField(max_length=512),
),
migrations.AlterField(
model_name='lecture',
name='univis_id',
field=models.CharField(max_length=512, unique=True),
),
migrations.AlterField(
model_name='lecture',
name='univis_ref',
field=models.CharField(max_length=512, unique=True),
),
]

18
ofu_app/apps/donar/migrations/0007_auto_20180401_1140.py Normal file
View File

@ -0,0 +1,18 @@
# Generated by Django 2.0.1 on 2018-04-01 11:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('donar', '0006_auto_20180401_1139'),
]
operations = [
migrations.AlterField(
model_name='lecture',
name='name',
field=models.CharField(max_length=512),
),
]

28
ofu_app/apps/donar/migrations/0008_auto_20180401_1142.py Normal file
View File

@ -0,0 +1,28 @@
# Generated by Django 2.0.1 on 2018-04-01 11:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('donar', '0007_auto_20180401_1140'),
]
operations = [
migrations.AlterField(
model_name='lecture',
name='lecturer_id',
field=models.CharField(max_length=1024),
),
migrations.AlterField(
model_name='lecture',
name='univis_id',
field=models.CharField(max_length=1024, unique=True),
),
migrations.AlterField(
model_name='lecture',
name='univis_ref',
field=models.CharField(max_length=1024, unique=True),
),
]

33
ofu_app/apps/donar/migrations/0009_auto_20180401_1143.py Normal file
View File

@ -0,0 +1,33 @@
# Generated by Django 2.0.1 on 2018-04-01 11:43
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('donar', '0008_auto_20180401_1142'),
]
operations = [
migrations.AlterField(
model_name='lecture',
name='lecturer_id',
field=models.CharField(max_length=256),
),
migrations.AlterField(
model_name='lecture',
name='short',
field=models.CharField(max_length=512),
),
migrations.AlterField(
model_name='lecture',
name='univis_id',
field=models.CharField(max_length=256, unique=True),
),
migrations.AlterField(
model_name='lecture',
name='univis_ref',
field=models.CharField(max_length=256, unique=True),
),
]

4
ofu_app/apps/donar/models.py
View File

@ -11,8 +11,8 @@ MAX_COORDS_NAME_LENGTH = 256
MAX_COORDS_LENGTH = 256
MAX_LECTURE_IDS_LENGTH = 256
MAX_LECTURE_SHORT_LENGTH = 128
MAX_LECTURE_NAME_LENGTH = 256
MAX_LECTURE_SHORT_LENGTH = 512
MAX_LECTURE_NAME_LENGTH = 512
MAX_LECTURE_TYPE_LENGTH = 64

65
ofu_app/apps/donar/utils/migrate_data_lectures.py
View File

@ -4,6 +4,9 @@ import json
from pprint import pprint
from django.db.utils import IntegrityError
from apps.donar.utils.parser import univis_lectures_parser
import logging
logger = logging.getLogger(__name__)
# CONFIG Fakultaet
FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
@ -32,38 +35,35 @@ def writeUnivisLectureTermsInDB(lecture, lecture_obj):
if type(lecture['terms']['term']) == list:
for term in lecture['terms']['term']:
try:
term_obj = Lecture_Terms.objects.create()
starttime = "00:00"
term_obj = Lecture_Terms.objects.create(starttime=starttime)
if 'starttime' in term:
starttime = term['starttime']
term_obj.starttime = datetime.strptime(starttime, "%H:%M")
term_obj.save()
if 'room' in term:
room_id = term['room']['UnivISRef']['@key']
term_obj.room = [Room.objects.get(key=room_id)]
term_obj.save()
term_obj.room.add(Room.objects.get(key=room_id))
lecture_obj.term.add(term_obj)
except IntegrityError as err:
print("ROOM_ID: " + str(room_id))
print(err.args)
logger.exception(err)
else:
try:
term_obj = Lecture_Terms.objects.create()
univis_starttime = "00:00"
term_obj = Lecture_Terms.objects.create(starttime=univis_starttime)
if 'starttime' in lecture['terms']['term']:
univis_starttime = lecture['terms']['term']['starttime']
term_obj.starttime = datetime.strptime(univis_starttime, '%H:%M')
term_obj.save()
if 'room' in lecture['terms']['term']:
room_id = lecture['terms']['term']['room']['UnivISRef']['@key']
pprint("Room: " + room_id)
Room.objects.get(key=room_id)
term_obj.room = [Room.objects.get(key=room_id)]
term_obj.room.add(Room.objects.get(key=room_id))
term_obj.save()
lecture_obj.term.add(term_obj)
except IntegrityError as err:
print("ROOM_ID: " + str(room_id))
print(err.args)
logger.exception(err)
def writeUnivisLectureDataInDB(data):
@ -91,43 +91,56 @@ def writeUnivisLectureDataInDB(data):
lecture_type = lecture['type']
if 'dozs' in lecture:
lecturer_id = dict(lecture['dozs']['doz']['UnivISRef'])['@key']
print("Lecture: " + name)
lecture_obj = Lecture.objects.create(univis_ref=key, univis_id=univis_id, name=name, short=short,
type=lecture_type, lecturer_id=lecturer_id)
writeUnivisLectureTermsInDB(lecture, lecture_obj)
lecture_obj.save()
logger.info("Lecture: {}".format(lecture_obj.short))
except IntegrityError as err:
print(err.args)
logger.warning('Lecture already exists')
# logger.exception(err)
return
def showStatus(status: str):
print(status)
pprint("Lectures: " + str(Lecture.objects.count()))
pprint("Lecture Terms: " + str(Lecture_Terms.objects.count()))
pprint("Room: " + str(Room.objects.count()))
return "\nStatus: {status}\n\tLectures: {lectures}\n\tLecture Terms: {lecture_terms}\n\tRoom: {room}".format(
status=status,
lectures=Lecture.objects.count(),
lecture_terms=Lecture_Terms.objects.count(),
room=Room.objects.count()
)
def delete():
lectures = Lecture.objects.all()
logger.info("Deleted following Lectures:")
for lecture in lectures:
logger.info("Lecture: {name}".format(
name=lecture.name)
)
lecture.delete()
def main():
# get food jsons
showStatus("Start with:")
logger.info(showStatus("Start SoWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_SoWi)))
pprint("----------------------------------------------------------------------------------------")
# pprint("----------------------------------------------------------------------------------------")
showStatus("After SoWi:")
logger.info(showStatus("Start GuK:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_GuK)))
pprint("----------------------------------------------------------------------------------------")
# pprint("----------------------------------------------------------------------------------------")
showStatus("After GuK:")
logger.info(showStatus("Start HuWi:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_HuWi)))
pprint("----------------------------------------------------------------------------------------")
# pprint("----------------------------------------------------------------------------------------")
showStatus("After HuWi:")
logger.info(showStatus("Start WIAI:"))
writeUnivisLectureDataInDB(univis_lectures_parser.parsePage(univis_lectures(FAKULTAET_WIAI)))
pprint("----------------------------------------------------------------------------------------")
# pprint("----------------------------------------------------------------------------------------")
showStatus("After WIAI:")
logger.info(showStatus("Finished:"))
if __name__ == '__main__':

25
ofu_app/apps/donar/utils/migrate_data_rooms.py
View File

@ -6,6 +6,10 @@ from apps.donar.models import Room, Lecture_Terms, Lecture
from apps.donar.utils.parser import univis_rooms_parser
from apps.donar.utils.parser import univis_lectures_parser
import logging
logger = logging.getLogger(__name__)
# CONFIG Fakultaet
FAKULTAET_GuK = "Fakult%E4t%20Geistes-%20und%20Kulturwissenschaften"
FAKULTAET_SoWi = "Fakult%E4t%20Sozial-%20und%20Wirtschaftswissenschaften"
@ -73,15 +77,28 @@ def writeUnivisRoomDataInDB(data):
if 'description' in room:
description = room['description']
Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name,
room = Room.objects.create(key=key, address=address, building_key=building_key, floor=floor, name=name,
orgname=orgname, short=short, size=size, description=description)
room.save()
logger.info('ROOM: {}'.format(room.short))
except IntegrityError as err:
pprint(err.args)
logger.warning('Room already exists')
def delete():
rooms = Room.objects.all()
logger.info("Deleted following Rooms:")
for room in rooms:
logger.info("Room: {name}".format(
name=room.short)
)
room.delete()
def main():
# get food jsons
pprint("Begin: Room: " + str(Room.objects.count()))
logger.info("Start:\nRoom: {}".format(Room.objects.count()))
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_GuK)))
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_SoWi)))
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms(FAKULTAET_HuWi)))
@ -102,7 +119,7 @@ def main():
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("d")))
writeUnivisRoomDataInDB(univis_rooms_parser.parsePage(univis_rooms_loc("x")))
pprint("Now: Room: " + str(Room.objects.count()))
logger.info("Finished:\nRoom: {}".format(Room.objects.count()))
if __name__ == '__main__':

55
ofu_app/apps/events/utils/migrate_data.py
View File

@ -7,6 +7,9 @@ from apps.events.utils.parser import univis_eventpage_parser
from apps.events.utils.parser import fekide_eventpage_parser
from apps.events.models import Event, Location
import logging
logger = logging.getLogger(__name__)
UNIVIS_CATEGORY = 'Univis'
@ -20,32 +23,17 @@ UNIVIS_RPG_WIAI = "http://univis.uni-bamberg.de/prg?search=events&department=Fak
def writeFekideDataInDB(data):
for date in data['dates']:
for event in date['events']:
try:
Location.objects.create(name=event['location'])
except IntegrityError:
# print("Location %s already exists." % event['location'])
pass
location, _ = Location.objects.get_or_create(name=event['location'])
location.save()
try:
event_obj, new = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"),
event_obj, _ = Event.objects.get_or_create(date=datetime.strptime(date['date'], "%d.%m.%Y"),
title=event['title'])
if new:
event_obj.category = event['category']
event_obj.link = event['link']
event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M")
event_obj.locations.add(Location.objects.get(name=event['location']))
event_obj.save()
Event.objects.filter(title="").delete()
else:
print("Event %s already exists. Start Update" % str(event_obj.title))
event_obj.category = event['category']
event_obj.link = event['link']
event_obj.time = datetime.strptime(str(event['time']).split()[1], "%H:%M")
event_obj.locations.add(Location.objects.get(name=event['location']))
event_obj.save()
except IntegrityError:
# ignored
pass
logger.info('CREATED - Event: {}'.format(event_obj.title))
def deleteUnivisObjects():
@ -62,15 +50,13 @@ def writeUnivisLocationsInDB(rooms):
for room in rooms:
if '@key' in room and 'short' in room:
try:
Location.objects.create(key=room['@key'], name=room['short'])
except IntegrityError:
print("Possible Duplicate! Start DB refresh")
try:
Location.objects.get(name=room['short']).key = room['@key']
except Exception as harderr:
print("Failed to refresh object" + harderr.args)
location, _ = Location.objects.get_or_create(key=room['@key'], name=room['short'])
location.key = room['@key']
location.name = room['short']
location.save()
logger.info('CREATE - Location: {}'.format(location.name))
except Exception as err:
print(err.args)
logger.critical(err.args)
def getLocationIDs(event):
@ -110,21 +96,24 @@ def writeUnivisEventsInDB(events: list):
event_obj.orgname = event['orgname']
try:
event_obj.save()
logger.info(event_obj.title)
except IntegrityError:
# TODO: Update DB Object if duplicate detected
print("Found Duplicate!")
logger.info("Found Duplicate!")
except Exception as err:
print(err.args)
logger.exception(err.args)
Event.objects.filter(title="").delete()
def write_out_db_objects():
pprint("Event: " + str(Event.objects.count()))
pprint("Location: " + str(Location.objects.count()))
return "\n\tEvent: {event}\n\tLocation: {location}".format(
event=Event.objects.count(),
location=Location.objects.count(),
)
def main():
print("Aktueller Stand:")
logger.info("Aktueller Stand:")
write_out_db_objects()
# deleteUnivisObjects()
@ -139,7 +128,7 @@ def main():
writeFekideDataInDB(fekide_eventpage_parser.parsePage())
print("Neuer Stand:")
logger.info("Neuer Stand:")
write_out_db_objects()

4
ofu_app/core/settings.py
View File

@ -241,5 +241,9 @@ LOGGING = {
'handlers': ['console', 'file', 'mail_admins'],
'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
},
'apps.donar.utils': {
'handlers': ['console', 'file', 'mail_admins'],
'level': os.getenv('DJANGO_LOG_LEVEL', 'DEBUG'),
},
},
}