feat(migration): legacy data load tweaks

fiesta/apps/utils/management/commands/loadlegacydata.py

@@ -8,7 +8,7 @@
 import requests
 from click import secho
 from django.core.files.images import ImageFile
-from django.db import connections, transaction
+from django.db import connections
 from django.db.backends.utils import CursorWrapper
 from django.utils import timezone
 from django.utils.text import slugify
@@ -17,15 +17,18 @@
 
 from apps.accounts.hashers import LegacyBCryptSHA256PasswordHasher
 from apps.accounts.models import User, UserProfile
+from apps.accounts.services.user_profile_state_synchronizer import synchronizer
 from apps.buddy_system.models import BuddyRequest, BuddyRequestMatch
 from apps.sections.models import Section, SectionMembership, SectionUniversity
+from apps.sections.models.services.section_plugins_reconciler import SectionPluginsReconciler
 from apps.universities.models import Faculty, University
 
 LOAD_MEMBERS = """
 SELECT
     du.user_id, u.password,
     du.name, du.surname,
     du.registered,
+    du.gender,
     u.status,
     u.signature,
     json_array(ra.role),
@@ -47,30 +50,83 @@
 SELECT
     br.data_user,
     br.description,
-    u.university,
+    issuer_u.university,
+    issuer.faculty_id,
+    matcher_u.university,
+    matcher.faculty_id,
     bm.member,
     bm.created
 FROM buddy_request br
-INNER JOIN user u on br.data_user = u.user_id
+INNER JOIN data_user issuer on br.data_user = issuer.user_id
+INNER JOIN user issuer_u on br.data_user = issuer_u.user_id
 LEFT JOIN buddy_match bm ON bm.international = br.data_user
+LEFT JOIN data_user matcher ON bm.member = matcher.user_id
+LEFT JOIN user matcher_u ON bm.member = matcher_u.user_id;
 """
 
-ID_TO_USER = {}
+LOAD_FACULTIES = """
+select
+    faculty.id, -- id
+    faculty.faculty_shortcut, -- FIT
+    faculty.university_id, -- BUT
+faculty.faculty -- faculty of
+from faculty
+inner join fiesta.university u on faculty.university_id = u.id
+"""
+
+ID_TO_USER: dict[str, User] = {}
+ID_TO_FACULTY: dict[str, Faculty] = {}
 
 WORKER_POOL = ProcessPoolExecutor()
 
 
+def load_faculties(*, cursor: CursorWrapper):
+    cursor.execute(LOAD_FACULTIES)
+
+    for f_id, f_abbr, uni_abbr, f_name in cursor.fetchall():
+        u, _ = University.objects.get_or_create(
+            abbr=uni_abbr,
+            defaults=dict(name=uni_abbr),
+        )
+        f, _ = Faculty.objects.update_or_create(
+            abbr=f_abbr,
+            university=u,
+            defaults=dict(name=f_name),
+        )
+        ID_TO_FACULTY[f_id] = f
+        secho(f"Processed faculty {f_abbr}.")
+
+
+def unknown_faculty_for_university(uni_abbr: str):
+    return Faculty.objects.get_or_create(
+        abbr="UND",
+        university=University.objects.get(abbr=uni_abbr),
+        defaults=dict(name="Unknown"),
+    )[0]
+
+
 def load_requests(*, cursor: CursorWrapper):
+    secho("Loading buddy requests.")
     cursor.execute(LOAD_BUDDY_REQUESTS)
     for i, row in enumerate(cursor.fetchall(), start=1):
         (
             issuer_email,
             description,
             issuer_university,
+            issuer_faculty,
+            matcher_university,
+            matcher_faculty,
             matched_by_email,
             matched_at,
         ) = row
 
+        secho(f"Processing {i: >4} {row=}.")
+
+        if issuer_faculty:
+            issuer_f = ID_TO_FACULTY[issuer_faculty]
+        else:
+            issuer_f = unknown_faculty_for_university(issuer_university)
+
         responsible_section = Section.objects.filter(universities__abbr=issuer_university).first()
 
         br, _ = BuddyRequest.objects.update_or_create(
@@ -79,25 +135,32 @@ def load_requests(*, cursor: CursorWrapper):
             defaults=dict(
                 note=description,
                 state=BuddyRequest.State.MATCHED if matched_by_email else BuddyRequest.State.CREATED,
+                issuer_faculty=issuer_f,
+                created=ID_TO_USER[issuer_email].date_joined,
             ),
         )
 
         if matched_by_email:
-            BuddyRequestMatch.objects.create_or_update(
+            if matcher_faculty:
+                matcher_f = ID_TO_FACULTY.get(matcher_faculty)
+            else:
+                matcher_f = unknown_faculty_for_university(matcher_university)
+
+            BuddyRequestMatch.objects.update_or_create(
                 request=br,
                 defaults=dict(
                     matcher=ID_TO_USER[matched_by_email],
                     created=make_aware(matched_at) if matched_at else None,
+                    matcher_faculty=matcher_f,
                 ),
             )
 
-        secho("Processing {i: >4}: {desc}.".format(i=i, desc=description[:32].replace("\n", " ")))
-
 
 def load_users(*, cursor: CursorWrapper):
     cursor.execute(LOAD_MEMBERS)
 
-    for email, user in WORKER_POOL.map(process_user_row, cursor.fetchall(), itertools.count(1)):
+    # for email, user in WORKER_POOL.map(process_user_row, cursor.fetchall(), itertools.count(1)):
+    for email, user in map(process_user_row, cursor.fetchall(), itertools.count(1)):
         ID_TO_USER[email] = user
 
 
@@ -108,6 +171,7 @@ def process_user_row(row, i):
         first_name,
         last_name,
         registered,
+        gender,
         status,
         avatar_name,
         roles,
@@ -118,12 +182,12 @@ def process_user_row(row, i):
         uni_id,
         uni_name,
     ) = row
-
     secho(f"Processing {i: >4}: {email}.")
+
     # password = password.split('$')[3]
     # is:        $2y$10$t9haqcKqlgXKrCv1pVTAxuqKh7vDtwksh0w7PXb44eNjRQOvY58Mu
     # wanted: alg$2y$10$iqdxi8RXWCADYYFsfyHD1u6rrXBW4v73.AGHZ32uUYu9gVjD0x7IG
-    user, created = User.objects.update_or_create(
+    user, created = User.objects.select_for_update().update_or_create(
         username=email,
         defaults=dict(
             email=email,
@@ -145,14 +209,14 @@ def process_user_row(row, i):
     # if not created:
     #     continue
 
-    section, _ = Section.objects.update_or_create(
+    section, _ = Section.objects.select_for_update().update_or_create(
         name=section_short,
         defaults=dict(
             country=Country("CZ"),
             space_slug=slugify(section_short.replace(" ", "")),
         ),
     )
-    SectionMembership.objects.update_or_create(
+    SectionMembership.objects.select_for_update().update_or_create(
         section=section,
         user=user,
         role=(
@@ -168,28 +232,30 @@ def process_user_row(row, i):
         state=SectionMembership.State.ACTIVE,
         defaults=dict(created=make_aware(registered) if registered else timezone.now()),
     )
-    university, _ = University.objects.update_or_create(
-        abbr=uni_id, defaults=dict(name=uni_name, country=Country("CZ"))
-    )
-    SectionUniversity.objects.update_or_create(
+    university, _ = University.objects.get_or_create(abbr=uni_id, defaults=dict(name=uni_name, country=Country("CZ")))
+    SectionUniversity.objects.select_for_update().update_or_create(
         section=section,
         university=university,
     )
     faculty = None
     if faculty_name:
-        faculty, _ = Faculty.objects.update_or_create(
+        faculty, _ = Faculty.objects.select_for_update().update_or_create(
             university=university,
             abbr=faculty_short,
             defaults=dict(name=faculty_name),
         )
 
     # is_international = highest_role == SectionMembership.Role.INTERNATIONAL
-    user_profile, _ = UserProfile.objects.update_or_create(
+    user_profile, _ = UserProfile.objects.select_for_update().update_or_create(
         user=user,
         defaults=dict(
             nationality=Country(country_code),
             university=university if not faculty else None,
             faculty=faculty,
+            gender={
+                "f": UserProfile.Gender.FEMALE,
+                "m": UserProfile.Gender.MALE,
+            }[gender],
         ),
     )
 
@@ -205,16 +271,28 @@ def process_user_row(row, i):
         )
         user_profile.save(update_fields=["picture"])
 
-    return (email, user)
+    return email, user
+
+
+def fill_id_to_user():
+    for user in User.objects.all():
+        ID_TO_USER[user.username] = user
+
+
+def reconcile_sections():
+    for s in Section.objects.all():
+        SectionPluginsReconciler.reconcile(s)
+        secho(f"Reconciled section {s}.")
 
 
 def load(cursor: CursorWrapper):
-    with transaction.atomic(using="legacydb"):
-        load_users(cursor=cursor)
-        load_requests(cursor=cursor)
+    load_faculties(cursor=cursor)
+    # fill_id_to_user()
+    load_users(cursor=cursor)
+    load_requests(cursor=cursor)
 
 
 @click.command()
 def load_legacy_data():
-    with connections["legacydb"].cursor() as cursor:
+    with connections["legacydb"].cursor() as cursor, synchronizer.without_profile_revalidation():
         load(cursor=cursor)