Merge pull request #27 from shannonturner/v6_2024

Version 6 (Spring 2024)

.gitignore

@@ -1,6 +1,7 @@
 *.pyc
 bin/
 lib/
+media/
 include/
 .DS_Store
 .Python

metro_map_saver/citysuggester/apps.py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.apps import AppConfig
 
 
 class CitysuggesterConfig(AppConfig):
     name = 'citysuggester'
+    default_auto_field = 'django.db.models.AutoField'

metro_map_saver/citysuggester/utils.py

@@ -23,7 +23,6 @@ def load_systems():
 
     return systems
 
-systems = load_systems()
 
 def suggest_city(map_stations, station_overlap=MINIMUM_STATION_OVERLAP):
 
@@ -37,6 +36,8 @@ def suggest_city(map_stations, station_overlap=MINIMUM_STATION_OVERLAP):
     if not station_overlap:
         station_overlap = MINIMUM_STATION_OVERLAP
 
+    systems = load_systems()
+
     for name, system_stations in systems.items():
         common_stations = system_stations.intersection(map_stations)
         if len(common_stations) > station_overlap:

metro_map_saver/map_saver/apps.py

@@ -1,8 +1,6 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
 from django.apps import AppConfig
 
 
 class MapSaverConfig(AppConfig):
     name = 'map_saver'
+    default_auto_field = 'django.db.models.BigAutoField'

metro_map_saver/map_saver/forms.py

@@ -0,0 +1,81 @@
+from django import forms
+
+from .models import SavedMap, IdentifyMap, MAP_TYPE_CHOICES
+from .validator import (
+    hex64,
+    validate_metro_map,
+    validate_metro_map_v2,
+)
+
+import hashlib
+import json
+import random
+
+
+RATING_CHOICES = (
+    ('likes', 'likes'),
+    ('dislikes', 'dislikes'),
+)
+
+class CreateMapForm(forms.Form):
+    mapdata = forms.JSONField()
+
+    def clean_mapdata(self):
+        mapdata = self.cleaned_data['mapdata']
+
+        data_version = mapdata.get('global', {}).get('data_version', 1)
+        if data_version == 2:
+            mapdata = validate_metro_map_v2(mapdata)
+            mapdata['global']['data_version'] = 2
+        else:
+            try:
+                mapdata = validate_metro_map(mapdata)
+                mapdata['global']['data_version'] = 1
+            except AssertionError as exc:
+                raise forms.ValidationError(exc)
+
+        return mapdata
+
+    def clean(self):
+        data = self.cleaned_data
+        if data.get('mapdata'):
+            data['urlhash'] = hex64(hashlib.sha256(str(data['mapdata']).encode('utf-8')).hexdigest()[:12])
+            data['naming_token'] = hashlib.sha256('{0}'.format(random.randint(1, 100000)).encode('utf-8')).hexdigest()
+            data['data_version'] = data['mapdata']['global']['data_version'] # convenience
+        return data
+
+class RateForm(forms.Form):
+
+    choice = forms.ChoiceField(widget=forms.HiddenInput, choices=RATING_CHOICES)
+    urlhash = forms.CharField(widget=forms.HiddenInput)
+
+    def clean(self):
+        data = self.cleaned_data
+        data['g-recaptcha-response'] = self.data.get('g-recaptcha-response')
+        return data
+
+class IdentifyForm(forms.ModelForm):
+
+    urlhash = forms.CharField(widget=forms.HiddenInput)
+    map_type = forms.ChoiceField(choices=(('', '--------'), *MAP_TYPE_CHOICES), required=False)
+
+    def clean_name(self):
+        name = self.cleaned_data['name'] or ''
+        return name.strip()
+
+    def clean_map_type(self):
+        map_type = self.cleaned_data['map_type'] or ''
+        return map_type.strip()
+
+    def clean(self):
+        data = self.cleaned_data
+        data['g-recaptcha-response'] = self.data.get('g-recaptcha-response')
+        return data
+
+    class Meta:
+        model = IdentifyMap
+        fields = [
+            'urlhash',
+            'name',
+            'map_type',
+        ]

metro_map_saver/map_saver/management/commands/count_stations.py

@@ -0,0 +1,19 @@
+from django.core.management.base import BaseCommand
+from map_saver.models import SavedMap
+
+class Command(BaseCommand):
+    help = """
+        Run on a regular schedule to:
+            count stations into .station_count
+            and populate .stations
+    """
+
+    def handle(self, *args, **kwargs):
+        needs_stations = SavedMap.objects.filter(station_count=-1)
+
+        for mmap in needs_stations:
+            mmap.stations = mmap._get_stations()
+            mmap.station_count = mmap._station_count()
+            mmap.save()
+
+        self.stdout.write(f'Counted stations for {needs_stations.count()} maps.')

metro_map_saver/map_saver/management/commands/make_images.py

@@ -0,0 +1,106 @@
+from django.core.management.base import BaseCommand
+from django.db.models import Q
+from map_saver.models import SavedMap
+
+import json
+import logging
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = """
+        Run on a regular schedule to generate images and thumbnails.
+
+            This really has multiple modes:
+                * ongoing (no args), less memory-efficient but with fewer database hits,
+                    meant to generate maps for the first time automatically on a schedule
+                * urlhash, meant to (re-)generate a single map
+                * start/end, like alltime but meant to handle picking up from a starting point
+    """
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-s',
+            '--start',
+            type=int,
+            dest='start',
+            default=0,
+            help='(Re-)Calculate images and thumbnails for maps starting with this PK.',
+        )
+        parser.add_argument(
+            '-e',
+            '--end',
+            type=int,
+            dest='end',
+            default=0,
+            help='Calculate images and thumbnails for maps with a PK lower than this value. Does NOT re-calculate.',
+        )
+        parser.add_argument(
+            '-l',
+            '--limit',
+            type=int,
+            dest='limit',
+            default=500,
+            help='Only calculate images and thumbnails for this many maps at once. Not in use if --alltime is set.',
+        )
+        parser.add_argument(
+            '-u',
+            '--urlhash',
+            type=str,
+            dest='urlhash',
+            default=False,
+            help='Calculate images and thumbnails for only one map in particular.',
+        )
+
+    def handle(self, *args, **kwargs):
+        urlhash = kwargs['urlhash']
+        start = kwargs['start']
+        end = kwargs['end']
+        limit = kwargs['limit']
+
+        if urlhash:
+            limit = total = 1
+            needs_images = SavedMap.objects.filter(urlhash=urlhash)
+            self.stdout.write(f"Generating images and thumbnails for {urlhash}.")
+            limit = 1
+        elif start or end:
+            start = start or 1
+            end = end or (start + limit + 1)
+            needs_images = SavedMap.objects.filter(pk__in=range(start, end))
+            self.stdout.write(f"(Re-)Generating images and thumbnails for {limit} maps starting with PK {start}.")
+        else:
+            # .filter(thumbnail_svg__in=[None, '']) worked great on staging until it didn't;
+            #   then staging would only match on thumbnail_svg=None;
+            #   using Q() objects works equally well on both
+            needs_images = SavedMap.objects.filter(Q(thumbnail_svg=None) | Q(thumbnail_svg=''))
+            self.stdout.write(f"Generating images and thumbnails for up to {limit} maps that don't have them.")
+
+        needs_images = needs_images.order_by('id')[:limit]
+
+        errors = []
+        t0 = time.time()
+        for mmap in needs_images:
+            t1 = time.time()
+
+            try:
+                self.stdout.write(mmap.generate_images())
+            except json.decoder.JSONDecodeError as exc:
+                self.stdout.write(f'[ERROR] Failed to generate images and thumbnails for #{mmap.id} ({mmap.urlhash}): JSONDecodeError: {exc}')
+                errors.append(mmap.urlhash)
+            except Exception as exc:
+                self.stdout.write(f'[ERROR] Failed to generate images and thumbnails for #{mmap.id} ({mmap.urlhash}): Exception: {exc}')
+                errors.append(mmap.urlhash)
+
+            t2 = time.time()
+            dt = (t2 - t1)
+            if dt > 5:
+                self.stdout.write(f'Generating image for {mmap.urlhash} took a very long time: {dt:.2f}s')
+                logger.warn(f'Generating image for {mmap.urlhash} took a very long time: {dt:.2f}s')
+
+        t3 = time.time()
+        dt = (t3 - t0)
+        self.stdout.write(f'Made images and thumbnails in {dt:.2f}s')
+        if errors:
+            self.stdout.write(f'Failed to generate images and thumbnails for {len(errors)} maps: {errors}')

metro_map_saver/map_saver/management/commands/oneoff_backfill_createdat.py

@@ -0,0 +1,83 @@
+from django.core.management.base import BaseCommand
+from map_saver.models import SavedMap
+
+import datetime
+import math
+import pytz
+
+
+class Command(BaseCommand):
+    help = """
+        Backdates old maps (from prior to addition of .created_at, sigh)
+        based on best-available data I have.
+    """
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '--dry-run',
+            action='store_true',
+            dest='dry_run',
+            default=False,
+            help='TK.',
+        )
+
+    def handle(self, *args, **kwargs):
+
+        dry_run = kwargs['dry_run']
+
+        UTC = pytz.timezone('UTC')
+
+        key_dates = {
+            "2018-04-11": 218,
+            "2018-06-20": 70,
+            "2018-07-14": 24,
+            "2018-07-25": 11,
+            "2018-07-31": 6,
+            "2018-08-11": 11,
+            "2018-08-30": 19,
+            "2018-09-12": 13,
+        }
+
+        for old_date, days_elapsed in key_dates.items():
+
+            old_date = datetime.datetime.strptime(old_date, '%Y-%m-%d')
+            old_date = old_date.replace(tzinfo=UTC)
+
+            days = days_elapsed - 1
+            maps_this_date = list(SavedMap.objects.filter(created_at=old_date).order_by('id'))
+            map_count = len(maps_this_date)
+            maps_to_backdate = []
+
+            if map_count < days_elapsed:
+                raise ValueError('This command has already been run! Exiting now.')
+
+            # Make sure at least this many maps get created each day;
+            # though we will need to add the remainders on still
+            maps_per_day = [math.floor(map_count / days_elapsed)] * days_elapsed
+            remainder = map_count - sum(maps_per_day)
+            index = 0
+            while remainder > 0:
+                maps_per_day[index] += 1
+                remainder -= 1
+                index += 1
+
+            self.stdout.write(f"Found {map_count} maps for {old_date}")
+
+            for mmap in maps_this_date:
+                maps_to_backdate.append(mmap)
+                if len(maps_to_backdate) >= maps_per_day[days - 1]:
+                    new_date = old_date - datetime.timedelta(days=days)
+                    self.stdout.write(f"{'[DRY-RUN] ' if dry_run else ''} Set date from {old_date.date()} to {new_date.date()} for {[m.pk for m in maps_to_backdate]}")
+                    if not dry_run:
+                        SavedMap.objects.filter(pk__in=[m.pk for m in maps_to_backdate]).update(
+                            created_at=new_date,
+                        )
+                    days = days - 1
+                    maps_to_backdate = []
+            else:
+                if maps_to_backdate:
+                    new_date = old_date - datetime.timedelta(days=days_elapsed)
+                    self.stdout.write(f"{'[DRY-RUN] ' if dry_run else ''} Set date from {old_date.date()} to {new_date.date()} for {[m.pk for m in maps_to_backdate]}")
+                    if not dry_run:
+                        SavedMap.objects.filter(pk__in=[m.pk for m in maps_to_backdate]).update(
+                            created_at=new_date,
+                        )