From bd6a08addcf4ac3c925505373ed1677a15037250 Mon Sep 17 00:00:00 2001 From: Cristhian Garcia Date: Thu, 26 Oct 2023 15:29:13 -0500 Subject: [PATCH] test: add test for event bus backend fix: only generate metadata fix: assert for subset ignoring runtime generated only metadata chore: quality fixes chore: improve docstring fix: convert timestamp str to datetime object test: add test for str datetime fix: rename tracking log event emitted fix: rename tracking log event emitted config fix: rename tracking event chore: handle comments feat: add settings for allowed events in the event bus chore: use opt_in for tracking logs event bus toggle fix: serialize tracking log data and context dates as logger chore: use send_event instead of custom metadata test: update unit test docs: add event bus routing documentation --- README.rst | 55 +++++++++++++++++ doc/user_guide/api/eventtracking.backends.rst | 8 +++ eventtracking/backends/event_bus.py | 55 ++++++++--------- eventtracking/backends/routing.py | 4 +- .../backends/tests/test_event_bus.py | 60 ++++++++++++++----- eventtracking/config.py | 11 +++- eventtracking/django/apps.py | 2 - requirements/base.txt | 32 ++++++++-- requirements/dev.txt | 13 +++- requirements/test.txt | 37 +++++++++--- 10 files changed, 209 insertions(+), 68 deletions(-) diff --git a/README.rst b/README.rst index 5ada9141..6d80a2b4 100644 --- a/README.rst +++ b/README.rst @@ -149,6 +149,61 @@ An example configuration for ``AsyncRoutingBackend`` is provided below:: } +Event Bus Routing +----------------- + +``event-tracking`` provides a solution for routing events to the Event Bus +using the ``EventBusBackend``. It extends ``RoutingBackend`` but sends events +to the Event Bus. + +It can: + +* Process event through the configured processors. +* If the event is allowed via `EVENT_BUS_TRACKING_LOGS`, send it to the Event Bus. + +Make sure to enable the setting: ``SEND_TRACKING_EVENT_EMITTED_SIGNAL`` to allow the +``EventBusBackend`` to send events to the Event Bus. + +An example configuration for ``EventBusBackend`` is provided below:: + + EVENT_TRACKING_BACKENDS = { + 'xapi': { + 'ENGINE': 'eventtracking.backends.event_bus.EventBusBackend', + 'OPTIONS': { + 'backend_name': 'xapi', + 'processors': [ + { + 'ENGINE': 'eventtracking.processors.regex_filter.RegexFilter', + 'OPTIONS':{ + 'filter_type': 'allowlist', + 'regular_expressions': [ + 'edx.course.enrollment.activated', + 'edx.course.enrollment.deactivated', + ] + } + } + ], + 'backends': { + 'xapi': { + 'ENGINE': 'dummy.backend.engine', + 'OPTIONS': { + ... + } + } + }, + }, + }, + 'tracking_logs': { + ... + } + ... + } + + EVENT_BUS_TRACKING_LOGS = [ + 'edx.course.enrollment.activated', + 'edx.course.enrollment.deactivated', + ] + Roadmap ------- diff --git a/doc/user_guide/api/eventtracking.backends.rst b/doc/user_guide/api/eventtracking.backends.rst index 51a037ae..f2f17fce 100644 --- a/doc/user_guide/api/eventtracking.backends.rst +++ b/doc/user_guide/api/eventtracking.backends.rst @@ -42,3 +42,11 @@ eventtracking.backends.segment :undoc-members: :show-inheritance: + +eventtracking.backends.event_bus +------------------------------ + +.. automodule:: eventtracking.backends.event_bus + :members: + :undoc-members: + :show-inheritance: diff --git a/eventtracking/backends/event_bus.py b/eventtracking/backends/event_bus.py index baba6e29..7be870d0 100644 --- a/eventtracking/backends/event_bus.py +++ b/eventtracking/backends/event_bus.py @@ -1,59 +1,52 @@ """Event tracker backend that emits events to the event-bus.""" + import json +import logging +from datetime import datetime +from django.conf import settings from openedx_events.analytics.data import TrackingLogData from openedx_events.analytics.signals import TRACKING_EVENT_EMITTED +from eventtracking.backends.logger import DateTimeJSONEncoder from eventtracking.backends.routing import RoutingBackend from eventtracking.config import SEND_TRACKING_EVENT_EMITTED_SIGNAL -from openedx_events.data import EventsMetadata -from openedx_events.event_bus import get_producer -from attrs import asdict -import logging logger = logging.getLogger(__name__) -EVENT_BUS_SOURCE = "openedx/eventtracking" class EventBusRoutingBackend(RoutingBackend): """ - Event tracker backend that emits an Open edX public signal. + Event tracker backend for the event bus. """ def send(self, event): """ - Emit the TRACKING_EVENT_EMITTED Open edX public signal to allow - other apps to listen for tracking events. + Send the tracking log event to the event bus by emitting the + TRACKING_EVENT_EMITTED signal using custom metadata. """ if not SEND_TRACKING_EVENT_EMITTED_SIGNAL.is_enabled(): return - data = json.dumps(event.get("data")) - context = json.dumps(event.get("context")) + name = event.get("name") + + if name not in getattr(settings, "EVENT_BUS_TRACKING_LOGS", []): + return + + data = json.dumps(event.get("data"), cls=DateTimeJSONEncoder) + context = json.dumps(event.get("context"), cls=DateTimeJSONEncoder) + + timestamp = event.get("timestamp") - tracking_log=TrackingLogData( + if isinstance(timestamp, str): + timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%f%z") + + tracking_log = TrackingLogData( name=event.get("name"), - timestamp=event.get("timestamp"), + timestamp=timestamp, data=data, context=context, ) + TRACKING_EVENT_EMITTED.send_event(tracking_log=tracking_log) - logger.info(f"Sending tracking event emitted signal for event for {tracking_log.name}") - get_producer().send( - signal=TRACKING_EVENT_EMITTED, - topic="analytics", - event_key_field="tracking_log.name", - event_data={"tracking_log": tracking_log}, - event_metadata=generate_signal_metadata() - ) - - -def generate_signal_metadata(): - """ - Generate the metadata for the signal with a custom source. - """ - metadata = TRACKING_EVENT_EMITTED.generate_signal_metadata() - medata_dict = asdict(metadata) - medata_dict["source"] = EVENT_BUS_SOURCE - metadata = EventsMetadata(**medata_dict) - return metadata + logger.info(f"Tracking log {tracking_log.name} emitted to the event bus.") diff --git a/eventtracking/backends/routing.py b/eventtracking/backends/routing.py index c8238a91..d59d941e 100644 --- a/eventtracking/backends/routing.py +++ b/eventtracking/backends/routing.py @@ -118,7 +118,7 @@ def process_event(self, event): processed_event = modified_event except EventEmissionExit: raise - except Exception: # pylint: disable=broad-except + except Exception: LOG.exception( 'Failed to execute processor: %s', str(processor) ) @@ -142,7 +142,7 @@ def send_to_backends(self, event): LOG.info('[send_to_backends] Failed to send edx event "%s" to "%s" backend. "%s" backend has' ' not been enabled, [%s]', event["name"], name, name, repr(exc) ) - except Exception: # pylint: disable=broad-except + except Exception: LOG.exception( 'Unable to send edx event "%s" to backend: %s', event["name"], name ) diff --git a/eventtracking/backends/tests/test_event_bus.py b/eventtracking/backends/tests/test_event_bus.py index 952d8535..bd170f2f 100644 --- a/eventtracking/backends/tests/test_event_bus.py +++ b/eventtracking/backends/tests/test_event_bus.py @@ -1,12 +1,18 @@ """ Test the async routing backend. """ + +import json +from datetime import datetime from unittest import TestCase +from unittest.mock import patch -from unittest.mock import sentinel, patch -from eventtracking.backends.event_bus import EventBusRoutingBackend +from django.test import override_settings from openedx_events.analytics.data import TrackingLogData +from eventtracking.backends.event_bus import EventBusRoutingBackend + + class TestAsyncRoutingBackend(TestCase): """ Test the async routing backend. @@ -15,27 +21,49 @@ class TestAsyncRoutingBackend(TestCase): def setUp(self): super().setUp() self.sample_event = { - 'name': str(sentinel.name), - 'data': 'data', - 'timestamp': '2020-01-01T12:12:12.000000+00:00', - 'context': {}, + "name": "sample_event", + "data": {"foo": "bar"}, + "timestamp": "2020-01-01T12:12:12.000000+00:00", + "context": {"baz": "qux"}, } - @patch('eventtracking.backends.event_bus.EventBusRoutingBackend.send') + @patch("eventtracking.backends.event_bus.EventBusRoutingBackend.send") def test_successful_send(self, mocked_send_event): backend = EventBusRoutingBackend() backend.send(self.sample_event) mocked_send_event.assert_called_once_with(self.sample_event) - @patch('eventtracking.backends.event_bus.TRACKING_EVENT_EMITTED.send_event') - def test_successful_send_event(self, mocked_send_event): + @override_settings( + SEND_TRACKING_EVENT_EMITTED_SIGNAL=True, + EVENT_BUS_TRACKING_LOGS=["sample_event"], + ) + @patch("eventtracking.backends.event_bus.TRACKING_EVENT_EMITTED.send_event") + def test_successful_send_event(self, mock_send_event): backend = EventBusRoutingBackend() backend.send(self.sample_event) - mocked_send_event.assert_called_once_with( - tracking_log=TrackingLogData( - name=self.sample_event['name'], - timestamp=self.sample_event['timestamp'], - data=self.sample_event['data'], - context=self.sample_event['context'] - ) + + mock_send_event.assert_called() + self.assertDictContainsSubset( + { + "tracking_log": TrackingLogData( + name=self.sample_event["name"], + timestamp=datetime.strptime( + self.sample_event["timestamp"], "%Y-%m-%dT%H:%M:%S.%f%z" + ), + data=json.dumps(self.sample_event["data"]), + context=json.dumps(self.sample_event["context"]), + ) + }, + mock_send_event.call_args.kwargs, ) + + @patch( + "eventtracking.backends.event_bus.SEND_TRACKING_EVENT_EMITTED_SIGNAL.is_enabled" + ) + @patch("eventtracking.backends.event_bus.TRACKING_EVENT_EMITTED.send_event") + def test_event_is_disabled(self, mock_send_event, mock_is_enabled): + mock_is_enabled.return_value = False + backend = EventBusRoutingBackend() + backend.send(self.sample_event) + mock_is_enabled.assert_called_once() + mock_send_event.assert_not_called() diff --git a/eventtracking/config.py b/eventtracking/config.py index f316a2f6..0dd5ef0f 100644 --- a/eventtracking/config.py +++ b/eventtracking/config.py @@ -1,6 +1,6 @@ """ This module contains various configuration settings via -waffle switches for the Certificates app. +waffle switches for the event-tracking app. """ from edx_toggles.toggles import SettingToggle @@ -10,5 +10,10 @@ # .. toggle_default: False # .. toggle_description: When True, the system will publish `TRACKING_EVENT_EMITTED` signals to the event bus. The # `TRACKING_EVENT_EMITTED` signal is emit when a tracking log is emitted. -# .. toggle_use_cases: publish -SEND_TRACKING_EVENT_EMITTED_SIGNAL = SettingToggle('SEND_TRACKING_EVENT_EMITTED_SIGNAL', default=True, module_name=__name__) +# .. toggle_use_cases: opt_in +# .. toggle_creation_date: 2023-10-26 +SEND_TRACKING_EVENT_EMITTED_SIGNAL = SettingToggle( + 'SEND_TRACKING_EVENT_EMITTED_SIGNAL', + default=False, + module_name=__name__ +) diff --git a/eventtracking/django/apps.py b/eventtracking/django/apps.py index f830140d..c2fc320a 100644 --- a/eventtracking/django/apps.py +++ b/eventtracking/django/apps.py @@ -21,5 +21,3 @@ def ready(self): # pylint: disable=import-outside-toplevel from eventtracking.django.django_tracker import override_default_tracker override_default_tracker() - - import eventtracking.handlers # pylint: disable=unused-import diff --git a/requirements/base.txt b/requirements/base.txt index 02c2c5ca..4e16d633 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -28,6 +28,7 @@ click==8.1.7 # click-didyoumean # click-plugins # click-repl + # code-annotations # edx-django-utils click-didyoumean==0.3.0 # via celery @@ -35,6 +36,8 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery +code-annotations==1.6.0 + # via edx-toggles django==3.2.24 # via # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt @@ -42,22 +45,36 @@ django==3.2.24 # django-crum # django-waffle # edx-django-utils + # edx-toggles # openedx-events django-crum==0.7.9 - # via edx-django-utils + # via + # edx-django-utils + # edx-toggles django-waffle==4.1.0 - # via edx-django-utils + # via + # edx-django-utils + # edx-toggles edx-django-utils==5.10.1 - # via -r requirements/base.in + # via + # -r requirements/base.in + # edx-toggles + # openedx-events edx-opaque-keys[django]==2.5.1 # via openedx-events +edx-toggles==5.1.1 + # via -r requirements/base.in fastavro==1.9.3 # via openedx-events +jinja2==3.1.3 + # via code-annotations kombu==5.3.5 # via celery +markupsafe==2.1.5 + # via jinja2 newrelic==9.6.0 # via edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.1 # via -r requirements/base.in pbr==6.0.0 # via stevedore @@ -75,10 +92,14 @@ pynacl==1.5.0 # via edx-django-utils python-dateutil==2.8.2 # via celery +python-slugify==8.0.4 + # via code-annotations pytz==2024.1 # via # -r requirements/base.in # django +pyyaml==6.0.1 + # via code-annotations six==1.16.0 # via # -r requirements/base.in @@ -87,8 +108,11 @@ sqlparse==0.4.4 # via django stevedore==5.1.0 # via + # code-annotations # edx-django-utils # edx-opaque-keys +text-unidecode==1.3 + # via python-slugify typing-extensions==4.9.0 # via # asgiref diff --git a/requirements/dev.txt b/requirements/dev.txt index 568a4e8a..41d8d3b8 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -92,6 +92,7 @@ code-annotations==1.6.0 # via # -r requirements/test.txt # edx-lint + # edx-toggles colorama==0.4.6 # via # -r requirements/ci.txt @@ -119,19 +120,25 @@ django==3.2.24 # django-crum # django-waffle # edx-django-utils + # edx-toggles # openedx-events django-crum==0.7.9 # via # -r requirements/test.txt # edx-django-utils + # edx-toggles django-waffle==4.1.0 # via # -r requirements/test.txt # edx-django-utils + # edx-toggles docutils==0.20.1 # via sphinx edx-django-utils==5.10.1 - # via -r requirements/test.txt + # via + # -r requirements/test.txt + # edx-toggles + # openedx-events edx-lint==5.3.6 # via # -r requirements/dev.in @@ -141,6 +148,8 @@ edx-opaque-keys[django]==2.5.1 # -r requirements/test.txt # edx-opaque-keys # openedx-events +edx-toggles==5.1.1 + # via -r requirements/test.txt exceptiongroup==1.2.0 # via # -r requirements/test.txt @@ -194,7 +203,7 @@ newrelic==9.6.0 # via # -r requirements/test.txt # edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.1 # via -r requirements/test.txt packaging==23.2 # via diff --git a/requirements/test.txt b/requirements/test.txt index ed6eed86..a71e64bf 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -58,7 +58,10 @@ click-plugins==1.1.1 # -r requirements/base.txt # celery code-annotations==1.6.0 - # via edx-lint + # via + # -r requirements/base.txt + # edx-lint + # edx-toggles coverage[toml]==7.4.1 # via # coverage @@ -73,17 +76,23 @@ dill==0.3.8 # django-crum # django-waffle # edx-django-utils + # edx-toggles # openedx-events django-crum==0.7.9 # via # -r requirements/base.txt # edx-django-utils + # edx-toggles django-waffle==4.1.0 # via # -r requirements/base.txt # edx-django-utils + # edx-toggles edx-django-utils==5.10.1 - # via -r requirements/base.txt + # via + # -r requirements/base.txt + # edx-toggles + # openedx-events edx-lint==5.3.6 # via -r requirements/test.in edx-opaque-keys[django]==2.5.1 @@ -91,6 +100,8 @@ edx-opaque-keys[django]==2.5.1 # -r requirements/base.txt # edx-opaque-keys # openedx-events +edx-toggles==5.1.1 + # via -r requirements/base.txt exceptiongroup==1.2.0 # via pytest fastavro==1.9.3 @@ -102,12 +113,16 @@ iniconfig==2.0.0 isort==5.13.2 # via pylint jinja2==3.1.3 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations # via # -r requirements/base.txt # celery markupsafe==2.1.5 - # via jinja2 + # via + # -r requirements/base.txt + # jinja2 mccabe==0.7.0 # via pylint mock==5.1.0 @@ -116,7 +131,7 @@ newrelic==9.6.0 # via # -r requirements/base.txt # edx-django-utils -openedx-events==9.4.0 +openedx-events==9.5.1 # via -r requirements/base.txt packaging==23.2 # via pytest @@ -172,13 +187,17 @@ python-dateutil==2.8.2 # -r requirements/base.txt # celery python-slugify==8.0.4 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations pytz==2024.1 # via # -r requirements/base.txt # django pyyaml==6.0.1 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations six==1.16.0 # via # -r requirements/base.txt @@ -195,7 +214,9 @@ stevedore==5.1.0 # edx-django-utils # edx-opaque-keys text-unidecode==1.3 - # via python-slugify + # via + # -r requirements/base.txt + # python-slugify tomli==2.0.1 # via # coverage