diff --git a/dashboard/management/commands/fix_trac_metrics.py b/dashboard/management/commands/fix_trac_metrics.py new file mode 100644 index 000000000..573b0aa7f --- /dev/null +++ b/dashboard/management/commands/fix_trac_metrics.py @@ -0,0 +1,93 @@ +from datetime import date, timedelta + +import time_machine +from django.core.management.base import CommandError, LabelCommand +from django.db.models import Case, Max, Min, When + +from ...models import TracTicketMetric + + +def _get_data(metric, options): + """ + Return a queryset of Datum instances for the given metric, taking into + account the from_date/to_date keys of the given options dict. + """ + queryset = metric.data.all() + if options["from_date"]: + queryset = queryset.filter(timestamp__date__gte=options["from_date"]) + if options["to_date"]: + queryset = queryset.filter(timestamp__date__lte=options["to_date"]) + return queryset + + +def _daterange(queryset): + """ + Given a queryset of Datum objects, generate all dates (as date objects) + between the earliest and latest data points in the queryset. + """ + aggregated = queryset.aggregate( + start=Min("timestamp__date"), end=Max("timestamp__date") + ) + if aggregated["start"] is None or aggregated["end"] is None: + raise ValueError("queryset cannot be empty") + + d = aggregated["start"] + while d <= aggregated["end"]: + yield d + d += timedelta(days=1) + + +def _refetched_case_when(dates, metric): + """ + Refetch the given metric for all the given dates and build a CASE database + expression with one WHEN per date. + """ + whens = [] + for d in dates: + with time_machine.travel(d): + whens.append(When(timestamp__date=d, then=metric.fetch())) + return Case(*whens) + + +class Command(LabelCommand): + help = "Retroactively refetch measurements for Trac metrics." + label = "slug" + + def add_arguments(self, parser): + super().add_arguments(parser) + parser.add_argument( + "--yes", action="store_true", help="Commit the changes to the database" + ) + parser.add_argument( + "--from-date", + type=date.fromisoformat, + help="Restrict the timestamp range (ISO format)", + ) + parser.add_argument( + "--to-date", + type=date.fromisoformat, + help="Restrict the timestamp range (ISO format)", + ) + + def handle_label(self, label, **options): + try: + metric = TracTicketMetric.objects.get(slug=label) + except TracTicketMetric.DoesNotExist as e: + raise CommandError from e + + verbose = int(options["verbosity"]) > 0 + + if verbose: + self.stdout.write(f"Fixing metric {label}...") + dataset = _get_data(metric, options) + + if options["yes"]: + dates = _daterange(dataset) + updated_measurement_expression = _refetched_case_when(dates, metric) + updated = dataset.update(measurement=updated_measurement_expression) + if verbose: + self.stdout.write(self.style.SUCCESS(f"{updated} rows updated")) + else: + if verbose: + self.stdout.write(f"{dataset.count()} rows will be updated.") + self.stdout.write("Re-run the command with --yes to apply the change") diff --git a/dashboard/tests.py b/dashboard/tests.py index 1d357ffc3..5b4f1baa9 100644 --- a/dashboard/tests.py +++ b/dashboard/tests.py @@ -1,5 +1,6 @@ import datetime import json +from operator import attrgetter from unittest import mock import requests_mock @@ -10,6 +11,7 @@ from tracdb.models import Ticket from tracdb.testutils import TracDBCreateDatabaseMixin +from tracdb.tractime import UTC, datetime_to_timestamp from .models import ( METRIC_PERIOD_DAILY, @@ -178,3 +180,115 @@ def test_update_metric(self, mocker, mock_reset_generation_key): self.assertTrue(mock_reset_generation_key.called) data = GithubItemCountMetric.objects.last().data.last() self.assertEqual(data.measurement, 10) + + +class FixTracMetricsCommandTestCase(TracDBCreateDatabaseMixin, TestCase): + databases = {"default", "trac"} + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + + def dt(*args, **kwargs): + kwargs.setdefault("tzinfo", UTC) + return datetime.datetime(*args, **kwargs) + + def ts(*args, **kwargs): + return datetime_to_timestamp(dt(*args, **kwargs)) + + for day in range(7): + Ticket.objects.create(_time=ts(2024, 1, day + 1)) + + cls.metric_today = TracTicketMetric.objects.create( + slug="today", query="time=today.." + ) + cls.metric_week = TracTicketMetric.objects.create( + slug="week", query="time=thisweek.." + ) + + def test_command_today(self): + datum = self.metric_today.data.create( + measurement=0, timestamp="2024-01-01T00:00:00" + ) + management.call_command("fix_trac_metrics", "today", yes=True, verbosity=0) + datum.refresh_from_db() + self.assertEqual(datum.measurement, 1) + + def test_command_week(self): + datum = self.metric_week.data.create( + measurement=0, timestamp="2024-01-07T00:00:00" + ) + management.call_command("fix_trac_metrics", "week", yes=True, verbosity=0) + datum.refresh_from_db() + self.assertEqual(datum.measurement, 7) + + def test_command_safe_by_default(self): + datum = self.metric_today.data.create( + measurement=0, timestamp="2024-01-01T00:00:00" + ) + management.call_command("fix_trac_metrics", "today", verbosity=0) + datum.refresh_from_db() + self.assertEqual(datum.measurement, 0) + + def test_multiple_measurements(self): + self.metric_today.data.create(measurement=0, timestamp="2024-01-01T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-02T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-03T00:00:00") + management.call_command("fix_trac_metrics", "today", yes=True, verbosity=0) + self.assertQuerySetEqual( + self.metric_today.data.order_by("timestamp"), + [1, 1, 1], + transform=attrgetter("measurement"), + ) + + def test_option_from_date(self): + self.metric_today.data.create(measurement=0, timestamp="2024-01-01T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-02T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-03T00:00:00") + management.call_command( + "fix_trac_metrics", + "today", + yes=True, + from_date=datetime.date(2024, 1, 2), + verbosity=0, + ) + self.assertQuerySetEqual( + self.metric_today.data.order_by("timestamp"), + [0, 1, 1], + transform=attrgetter("measurement"), + ) + + def test_option_to_date(self): + self.metric_today.data.create(measurement=0, timestamp="2024-01-01T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-02T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-03T00:00:00") + management.call_command( + "fix_trac_metrics", + "today", + yes=True, + to_date=datetime.date(2024, 1, 2), + verbosity=0, + ) + self.assertQuerySetEqual( + self.metric_today.data.order_by("timestamp"), + [1, 1, 0], + transform=attrgetter("measurement"), + ) + + def test_option_both_to_and_from_date(self): + self.metric_today.data.create(measurement=0, timestamp="2024-01-01T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-02T00:00:00") + self.metric_today.data.create(measurement=0, timestamp="2024-01-03T00:00:00") + management.call_command( + "fix_trac_metrics", + "today", + yes=True, + from_date=datetime.date(2024, 1, 2), + to_date=datetime.date(2024, 1, 2), + verbosity=0, + ) + self.assertQuerySetEqual( + self.metric_today.data.order_by("timestamp"), + [0, 1, 0], + transform=attrgetter("measurement"), + ) diff --git a/requirements/common.txt b/requirements/common.txt index f824ada21..ac175c8a3 100644 --- a/requirements/common.txt +++ b/requirements/common.txt @@ -21,3 +21,4 @@ requests==2.32.3 sorl-thumbnail==12.11.0 Sphinx==7.1.2 stripe==3.1.0 +time-machine==2.15.0 diff --git a/tracdb/models.py b/tracdb/models.py index 24f5a8bd4..21e1fbfa5 100644 --- a/tracdb/models.py +++ b/tracdb/models.py @@ -43,41 +43,14 @@ """ -import datetime +from datetime import date from functools import reduce from operator import and_, or_ from urllib.parse import parse_qs from django.db import models -try: - _epoc = datetime.datetime(1970, 1, 1, tzinfo=datetime.UTC) -except AttributeError: - # TODO: Remove when dropping support for Python 3.8 - _epoc = datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc) - - -class time_property: - """ - Convert Trac timestamps into UTC datetimes. - - See http://trac.edgewall.org/browser//branches/0.12-stable/trac/util/datefmt.py - for Trac's version of all this. Mine's something of a simplification. - - Like the rest of this module this is far from perfect -- no setters, for - example! That's good enough for now. - """ - - def __init__(self, fieldname): - self.fieldname = fieldname - - def __get__(self, instance, owner): - if instance is None: - return self - timestamp = getattr(instance, self.fieldname) - if timestamp is None: - return None - return _epoc + datetime.timedelta(microseconds=timestamp) +from .tractime import dayrange, time_property class JSONBObjectAgg(models.Aggregate): @@ -101,7 +74,17 @@ def from_querystring(self, querystring): filter_kwargs, exclude_kwargs = {}, {} for field, (value,) in parsed.items(): - if field not in model_fields: + if field == "time": + if value == "today..": + timestamp_range = dayrange(date.today(), 1) + elif value == "thisweek..": + timestamp_range = dayrange(date.today(), 7) + else: + raise ValueError(f"Unsupported time value {value}") + + filter_kwargs["_time__range"] = timestamp_range + continue + elif field not in model_fields: custom_lookup_required = True field = f"custom__{field}" if value.startswith("!"): diff --git a/tracdb/tests.py b/tracdb/tests.py index e7b7f9c24..3d072e089 100644 --- a/tracdb/tests.py +++ b/tracdb/tests.py @@ -1,9 +1,18 @@ +from datetime import date, datetime from operator import attrgetter -from django.test import TestCase +import time_machine +from django.test import SimpleTestCase, TestCase from .models import Revision, Ticket, TicketCustom from .testutils import TracDBCreateDatabaseMixin +from .tractime import ( + UTC, + datetime_to_timestamp, + dayrange, + time_property, + timestamp_to_datetime, +) class TestModels(TestCase): @@ -20,6 +29,9 @@ def _create_ticket(self, custom=None, **kwargs): """ if custom is None: custom = {} + if "time" in kwargs: + assert "_time" not in kwargs + kwargs["_time"] = datetime_to_timestamp(kwargs.pop("time")) ticket = Ticket.objects.create(**kwargs) TicketCustom.objects.bulk_create( @@ -157,3 +169,97 @@ def test_from_querystring_model_and_custom_field_together(self): Ticket.objects.from_querystring("severity=high&stage=unreviewed"), ["test1"], ) + + @time_machine.travel("2024-10-24T14:30:00+00:00") + def test_from_querystring_time_today_same_day(self): + self._create_ticket( + summary="test", + time=datetime.fromisoformat("2024-10-24T10:30:00+00:00"), + ) + self.assertTicketsEqual( + Ticket.objects.from_querystring("time=today.."), ["test"] + ) + + @time_machine.travel("2024-10-24T14:30:00+00:00") + def test_from_querystring_time_today_previous_day_less_than_24h(self): + self._create_ticket( + summary="test", + # previous day, but still within 24h + time=datetime.fromisoformat("2024-10-23T20:30:00+00:00"), + ) + self.assertTicketsEqual(Ticket.objects.from_querystring("time=today.."), []) + + @time_machine.travel("2024-10-24T14:30:00+00:00") + def test_from_querystring_time_today_previous_day_more_than_24h(self): + self._create_ticket( + summary="test", + # previous day, more than 24h ago + time=datetime.fromisoformat("2024-10-23T10:30:00+00:00"), + ) + self.assertTicketsEqual(Ticket.objects.from_querystring("time=today.."), []) + + @time_machine.travel("2024-10-24T14:30:00+00:00") + def test_from_querystring_time_thisweek(self): + self._create_ticket( + summary="test", + time=datetime.fromisoformat("2024-10-21T10:30:00+00:00"), + ) + self._create_ticket( + summary="too old", + time=datetime.fromisoformat("2024-10-15T10:30:00+00:00"), + ) + self.assertTicketsEqual( + Ticket.objects.from_querystring("time=thisweek.."), ["test"] + ) + + def test_from_querystring_invalid_time(self): + with self.assertRaises(ValueError): + Ticket.objects.from_querystring("time=2024-10-24..") + + +class TracTimeTestCase(SimpleTestCase): + def test_datetime_to_timestamp(self): + testdata = [ + (datetime(1970, 1, 1, microsecond=1, tzinfo=UTC), 1), + (datetime(1970, 1, 1, 0, 0, 1, tzinfo=UTC), 1_000_000), + (datetime(1970, 1, 2, tzinfo=UTC), 24 * 3600 * 1_000_000), + ] + for dt, expected in testdata: + with self.subTest(dt=dt): + self.assertEqual(datetime_to_timestamp(dt), expected) + + def test_timestamp_to_datetime(self): + testdata = [ + (1, datetime(1970, 1, 1, microsecond=1, tzinfo=UTC)), + (1_000_000, datetime(1970, 1, 1, second=1, tzinfo=UTC)), + (24 * 3600 * 1_000_000, datetime(1970, 1, 2, tzinfo=UTC)), + ] + for ts, expected in testdata: + with self.subTest(ts=ts): + self.assertEqual(timestamp_to_datetime(ts), expected) + + def test_time_property(self): + class T: + timestamp = 1 + prop = time_property("timestamp") + + self.assertEqual(T().prop.date(), date(1970, 1, 1)) + + def test_dayrange_error_negative_day(self): + with self.assertRaises(ValueError): + dayrange(date.today(), -1) + + def test_dayrange_error_zero_day(self): + with self.assertRaises(ValueError): + dayrange(date.today(), 0) + + def test_dayrange_error_datetime(self): + with self.assertRaises(TypeError): + dayrange(datetime.now(), 1) + + def test_dayrange_1_day(self): + offset = 6 * 3600 * 1_000_000 # offset between utc and chicago + self.assertEqual( + dayrange(date(1970, 1, 1), days=1), + (offset, offset + 24 * 3600 * 1_000_000 - 1), + ) diff --git a/tracdb/tractime.py b/tracdb/tractime.py new file mode 100644 index 000000000..3ac41738a --- /dev/null +++ b/tracdb/tractime.py @@ -0,0 +1,71 @@ +import datetime + +from django.utils import timezone + +try: + UTC = datetime.UTC +except AttributeError: + # TODO: Remove when dropping support for Python 3.8 + UTC = datetime.timezone.utc + +_epoc = datetime.datetime(1970, 1, 1, tzinfo=UTC) + + +class time_property: + """ + Convert Trac timestamps into UTC datetimes. + + See http://trac.edgewall.org/browser//branches/0.12-stable/trac/util/datefmt.py + for Trac's version of all this. Mine's something of a simplification. + + Like the rest of this module this is far from perfect -- no setters, for + example! That's good enough for now. + """ + + def __init__(self, fieldname): + self.fieldname = fieldname + + def __get__(self, instance, owner): + if instance is None: + return self + return timestamp_to_datetime(getattr(instance, self.fieldname)) + + +def datetime_to_timestamp(dt): + """ + Convert a python datetime object to a Trac-style timestamp. + """ + return (dt - _epoc).total_seconds() * 1000000 + + +def timestamp_to_datetime(ts): + """ + Convert a Trac-style timestamp to a python datetime object. + """ + if ts is None: + return None + return _epoc + datetime.timedelta(microseconds=ts) + + +def dayrange(d, days): + """ + Return a tuple of two timestamps (Trac-style) corresponding to the bounds + of the range of `days` days before the given date (included). That is to + say, `dayrange(TODAY, 1)` will return the range for today, and + `dayrange(TODAY, 7)` will be the last 7 days. + """ + if days <= 0: + raise ValueError(f"days must be greater than 0, not {days!r}") + if type(d) is not datetime.date: + raise TypeError(f"d must be a date object, not {d.__class__.__name__}") + + tz = timezone.get_current_timezone() + start = datetime.datetime.combine( + d - datetime.timedelta(days=days - 1), datetime.time(0, 0, 0), tzinfo=tz + ) + end = datetime.datetime.combine(d, datetime.time(23, 59, 59, 999_999), tzinfo=tz) + + return ( + datetime_to_timestamp(start), + datetime_to_timestamp(end), + ) diff --git a/tracdb/views.py b/tracdb/views.py index 9e725f593..8240677a3 100644 --- a/tracdb/views.py +++ b/tracdb/views.py @@ -1,9 +1,7 @@ -import datetime - from django import db from django.shortcuts import render -from .models import _epoc +from .tractime import timestamp_to_datetime def bouncing_tickets(request): @@ -17,7 +15,7 @@ def bouncing_tickets(request): # Fix timestamps. LOLTrac. for t in tickets: - t["last_reopen_time"] = ts2dt(t["last_reopen_time"]) + t["last_reopen_time"] = timestamp_to_datetime(t["last_reopen_time"]) return render( request, @@ -28,10 +26,6 @@ def bouncing_tickets(request): ) -def ts2dt(ts): - return _epoc + datetime.timedelta(microseconds=ts) - - def dictfetchall(cursor): desc = cursor.description return [dict(zip([col[0] for col in desc], row)) for row in cursor.fetchall()]