From 72b56bb25685bcd64b8d8045c29ad9bea36f7078 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Thu, 26 Sep 2024 21:52:00 -0700 Subject: [PATCH 01/14] todo --- enterprise_catalog/apps/catalog/models.py | 83 ++++++++++++++++++----- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index ce4d163d..481cf693 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -568,13 +568,11 @@ def bulk_update(self, objs, fields, batch_size=None): super().bulk_update(objs, fields, batch_size=batch_size) -class ContentMetadata(TimeStampedModel): +class BaseContentMetadata(TimeStampedModel): """ - Stores the JSON metadata for a piece of content, such as a course, course run, or program. - The metadata is retrieved from the Discovery service /search/all endpoint. - - .. no_pii: """ + class Meta: + abstract = True content_uuid = models.UUIDField( null=True, @@ -587,7 +585,6 @@ class ContentMetadata(TimeStampedModel): "in the enterprise environment." ) ) - content_key = models.CharField( max_length=255, blank=False, @@ -612,11 +609,7 @@ class ContentMetadata(TimeStampedModel): "The key represents this content's parent. For example for course_runs content their parent course key." ) ) - - # one course can be associated with many programs and one program can contain many courses. - associated_content_metadata = models.ManyToManyField('self', blank=True) - - json_metadata = JSONField( + _json_metadata = JSONField( default={}, blank=True, null=True, @@ -627,17 +620,10 @@ class ContentMetadata(TimeStampedModel): "endpoint results, specified as a JSON object." ) ) - catalog_queries = models.ManyToManyField(CatalogQuery) history = HistoricalRecords() - objects = ContentMetadataManager() - class Meta: - verbose_name = _("Content Metadata") - verbose_name_plural = _("Content Metadata") - app_label = 'catalog' - @property def is_exec_ed_2u_course(self): # pylint: disable=no-member @@ -677,6 +663,67 @@ def __str__(self): ) +class ContentMetadata(BaseContentMetadata): + """ + Stores the JSON metadata for a piece of content, such as a course, course run, or program. + The metadata is retrieved from the Discovery service /search/all endpoint. + + .. no_pii: + """ + class Meta: + verbose_name = _("Content Metadata") + verbose_name_plural = _("Content Metadata") + app_label = 'catalog' + + # one course can be associated with many programs and one program can contain many courses. + associated_content_metadata = models.ManyToManyField('self', blank=True) + + # one course can be part of many CatalogQueries and one CatalogQuery can contain many courses. + catalog_queries = models.ManyToManyField(CatalogQuery) + + @property + def json_metadata(self): + if restricted_metadata_for_catalog_query := getattr(self, 'restricted_metadata_for_catalog_query', None): + return restricted_metadata_for_catalog_query[0]._json_metadata + return self._json_metadata + + +class RestrictedContentMetadata(BaseContentMetadata): + """ + .. no_pii: + """ + class Meta: + verbose_name = _("Restricted Content Metadata") + verbose_name_plural = _("Restricted Content Metadata") + app_label = 'catalog' + unique_together = ('content_key', 'catalog_query') + + content_key = models.CharField( + max_length=255, + blank=False, + null=False, + unique=False, + help_text=_( + "The key that represents a piece of content, such as a course, course run, or program." + ) + ) + unrestricted_parent = models.ForeignKey( + ContentMetadata, + blank=False, + null=True, + related_name='restricted_content_metadata', + on_delete=models.deletion.SET_NULL, + ) + # one restricted course or course run can + catalog_query = models.ForeignKey( + CatalogQuery, + blank=False, + null=True, + related_name='restricted_content_metadata', + on_delete=models.deletion.SET_NULL, + ) + + def content_metadata_with_type_course(): """ Find all ContentMetadata records with a content type of "course". From 8c94294018be98b13e6c63eb845da0aa49ef59cc Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 12:43:43 -0700 Subject: [PATCH 02/14] squash --- .../0040_restrictedcoursemetadata_and_more.py | 58 +++++++++++++++++++ enterprise_catalog/apps/catalog/models.py | 52 ++++++++++++----- 2 files changed, 96 insertions(+), 14 deletions(-) create mode 100644 enterprise_catalog/apps/catalog/migrations/0040_restrictedcoursemetadata_and_more.py diff --git a/enterprise_catalog/apps/catalog/migrations/0040_restrictedcoursemetadata_and_more.py b/enterprise_catalog/apps/catalog/migrations/0040_restrictedcoursemetadata_and_more.py new file mode 100644 index 00000000..270f194e --- /dev/null +++ b/enterprise_catalog/apps/catalog/migrations/0040_restrictedcoursemetadata_and_more.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.16 on 2024-10-01 19:40 + +import collections +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import jsonfield.encoder +import jsonfield.fields +import model_utils.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0039_alter_catalogquery_unique_together_and_more'), + ] + + operations = [ + migrations.CreateModel( + name='RestrictedCourseMetadata', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('content_uuid', models.UUIDField(blank=True, help_text='The UUID that represents a piece of content. This value is usually a secondary identifier to content_key in the enterprise environment.', null=True, verbose_name='Content UUID')), + ('content_type', models.CharField(choices=[('course', 'Course'), ('courserun', 'Course Run'), ('program', 'Program'), ('learnerpathway', 'Learner Pathway')], max_length=255)), + ('parent_content_key', models.CharField(blank=True, db_index=True, help_text="The key represents this content's parent. For example for course_runs content their parent course key.", max_length=255, null=True)), + ('_json_metadata', jsonfield.fields.JSONField(blank=True, default={}, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'indent': 4, 'separators': (',', ':')}, help_text="The metadata about a particular piece content as retrieved from the discovery service's search/all endpoint results, specified as a JSON object.", load_kwargs={'object_pairs_hook': collections.OrderedDict}, null=True)), + ('content_key', models.CharField(help_text='The key that represents a piece of content, such as a course, course run, or program.', max_length=255)), + ], + options={ + 'verbose_name': 'Restricted Content Metadata', + 'verbose_name_plural': 'Restricted Content Metadata', + }, + ), + migrations.RenameField( + model_name='contentmetadata', + old_name='json_metadata', + new_name='_json_metadata', + ), + migrations.AlterField( + model_name='catalogquery', + name='content_filter', + field=jsonfield.fields.JSONField(default=dict, dump_kwargs={'cls': jsonfield.encoder.JSONEncoder, 'ensure_ascii': False, 'indent': 4, 'separators': (',', ':')}, help_text="Query parameters which will be used to filter the discovery service's search/all endpoint results, specified as a JSON object.", load_kwargs={'object_pairs_hook': collections.OrderedDict}), + ), + migrations.DeleteModel( + name='HistoricalContentMetadata', + ), + migrations.AddField( + model_name='restrictedcoursemetadata', + name='catalog_query', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='restricted_content_metadata', to='catalog.catalogquery'), + ), + migrations.AlterUniqueTogether( + name='restrictedcoursemetadata', + unique_together={('content_key', 'catalog_query')}, + ), + ] diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index 481cf693..6834a466 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -256,7 +256,27 @@ def content_metadata(self): """ if not self.catalog_query: return ContentMetadata.objects.none() - return self.catalog_query.contentmetadata_set.all() + return self.catalog_query.contentmetadata_set.filter(restricted_run=False) + # NOTE: json override doesn't need be disabled because + # self.catalog_query.contentmetadata_set is not annotated with + # overrides. + + @property + def content_metadata_with_restricted(self): + """ + Helper to retrieve the content metadata associated with the catalog. + + Returns: + Queryset: The queryset of associated content metadata + """ + if not self.catalog_query: + return ContentMetadata.objects.none() + prefetch_qs = models.Prefetch( + 'restricted_courses', + queryset=RestrictedCourseMetadata.objects.filter(catalog_query=self.catalog_query), + to_attr='restricted_course_metadata_for_catalog_query', + ) + return self.catalog_query.contentmetadata_set.prefetch_related(prefetch_qs) @cached_property def restricted_runs_allowed(self): @@ -324,7 +344,7 @@ def get_catalog_content_diff(self, content_keys): items_not_found = distinct_content_keys - found_content_keys return [{'content_key': item} for item in items_not_found], items_not_included, items_found - def get_matching_content(self, content_keys): + def get_matching_content(self, content_keys, include_restricted=False): """ Returns the set of content contained within this catalog that matches any of the course keys, course run keys, or programs keys specified by @@ -367,7 +387,10 @@ def get_matching_content(self, content_keys): if metadata.parent_content_key } query |= Q(content_key__in=parent_content_keys) - return self.content_metadata.filter(query) + if include_restricted: + return self.content_metadata_with_restricted.filter(query) + else: + return self.content_metadata.filter(query) def contains_content_keys(self, content_keys): """ @@ -683,12 +706,18 @@ class Meta: @property def json_metadata(self): - if restricted_metadata_for_catalog_query := getattr(self, 'restricted_metadata_for_catalog_query', None): - return restricted_metadata_for_catalog_query[0]._json_metadata + restricted_course_metadata_for_catalog_query = getattr( + self, + 'restricted_course_metadata_for_catalog_query', + None, + ) + if restricted_course_metadata_for_catalog_query: + # pylint: disable=protected-access + return restricted_course_metadata_for_catalog_query.first()._json_metadata return self._json_metadata -class RestrictedContentMetadata(BaseContentMetadata): +class RestrictedCourseMetadata(BaseContentMetadata): """ .. no_pii: """ @@ -698,6 +727,9 @@ class Meta: app_label = 'catalog' unique_together = ('content_key', 'catalog_query') + # Overwrite content_key from BaseContentMetadata in order to change unique + # to False. Use unique_together to allow multiple copies of the same course + # (one for each catalog query. content_key = models.CharField( max_length=255, blank=False, @@ -707,14 +739,6 @@ class Meta: "The key that represents a piece of content, such as a course, course run, or program." ) ) - unrestricted_parent = models.ForeignKey( - ContentMetadata, - blank=False, - null=True, - related_name='restricted_content_metadata', - on_delete=models.deletion.SET_NULL, - ) - # one restricted course or course run can catalog_query = models.ForeignKey( CatalogQuery, blank=False, From c3306e2e04146fd843e603a83ea4019af186be0d Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 13:02:04 -0700 Subject: [PATCH 03/14] squash --- .../0041_contentmetadata_is_restricted_run.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 enterprise_catalog/apps/catalog/migrations/0041_contentmetadata_is_restricted_run.py diff --git a/enterprise_catalog/apps/catalog/migrations/0041_contentmetadata_is_restricted_run.py b/enterprise_catalog/apps/catalog/migrations/0041_contentmetadata_is_restricted_run.py new file mode 100644 index 00000000..34c3a720 --- /dev/null +++ b/enterprise_catalog/apps/catalog/migrations/0041_contentmetadata_is_restricted_run.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.16 on 2024-10-01 20:01 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0040_restrictedcoursemetadata_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='contentmetadata', + name='is_restricted_run', + field=models.BooleanField(default=False, help_text='If true, cause this run to be included in various v2 endpoints.'), + ), + ] From 35c4eade2cecb5497dd7173e3b4e0dc5a184165a Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 13:15:04 -0700 Subject: [PATCH 04/14] squash --- enterprise_catalog/apps/catalog/models.py | 10 +++++++++- enterprise_catalog/apps/catalog/tests/factories.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index 6834a466..d8da06e5 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -256,7 +256,7 @@ def content_metadata(self): """ if not self.catalog_query: return ContentMetadata.objects.none() - return self.catalog_query.contentmetadata_set.filter(restricted_run=False) + return self.catalog_query.contentmetadata_set.filter(is_restricted_run=False) # NOTE: json override doesn't need be disabled because # self.catalog_query.contentmetadata_set is not annotated with # overrides. @@ -698,6 +698,14 @@ class Meta: verbose_name_plural = _("Content Metadata") app_label = 'catalog' + is_restricted_run = models.BooleanField( + default=False, + blank=False, + help_text=_( + "If true, cause this run to be included in various v2 endpoints." + ), + ) + # one course can be associated with many programs and one program can contain many courses. associated_content_metadata = models.ManyToManyField('self', blank=True) diff --git a/enterprise_catalog/apps/catalog/tests/factories.py b/enterprise_catalog/apps/catalog/tests/factories.py index edc35d63..6d9b29dd 100644 --- a/enterprise_catalog/apps/catalog/tests/factories.py +++ b/enterprise_catalog/apps/catalog/tests/factories.py @@ -80,7 +80,7 @@ class Meta: parent_content_key = None @factory.lazy_attribute - def json_metadata(self): + def _json_metadata(self): json_metadata = { 'key': self.content_key, 'aggregation_key': f'{self.content_type}:{self.content_key}', From 56ae3aa72114f07e4e828cbc24dbb50e344b1f8d Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 13:34:27 -0700 Subject: [PATCH 05/14] squash --- ...ictedcoursemetadata_unrestricted_parent.py | 19 +++++++++++++++++++ enterprise_catalog/apps/catalog/models.py | 7 +++++++ 2 files changed, 26 insertions(+) create mode 100644 enterprise_catalog/apps/catalog/migrations/0042_restrictedcoursemetadata_unrestricted_parent.py diff --git a/enterprise_catalog/apps/catalog/migrations/0042_restrictedcoursemetadata_unrestricted_parent.py b/enterprise_catalog/apps/catalog/migrations/0042_restrictedcoursemetadata_unrestricted_parent.py new file mode 100644 index 00000000..0202fb84 --- /dev/null +++ b/enterprise_catalog/apps/catalog/migrations/0042_restrictedcoursemetadata_unrestricted_parent.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.16 on 2024-10-01 20:34 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0041_contentmetadata_is_restricted_run'), + ] + + operations = [ + migrations.AddField( + model_name='restrictedcoursemetadata', + name='unrestricted_parent', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='restricted_courses', to='catalog.contentmetadata'), + ), + ] diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index d8da06e5..c0a292f5 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -747,6 +747,13 @@ class Meta: "The key that represents a piece of content, such as a course, course run, or program." ) ) + unrestricted_parent = models.ForeignKey( + ContentMetadata, + blank=False, + null=True, + related_name='restricted_courses', + on_delete=models.deletion.SET_NULL, + ) catalog_query = models.ForeignKey( CatalogQuery, blank=False, From 767ebd9dbbd19422f85b2c62351dd8ab21b8471e Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 13:40:17 -0700 Subject: [PATCH 06/14] squash --- enterprise_catalog/apps/catalog/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index c0a292f5..c3a4e6d9 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -721,7 +721,7 @@ def json_metadata(self): ) if restricted_course_metadata_for_catalog_query: # pylint: disable=protected-access - return restricted_course_metadata_for_catalog_query.first()._json_metadata + return restricted_course_metadata_for_catalog_query[0]._json_metadata return self._json_metadata From 204941d2031a5c7765fe117936d996f4ec490a82 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Tue, 1 Oct 2024 14:01:18 -0700 Subject: [PATCH 07/14] squash --- enterprise_catalog/apps/catalog/models.py | 45 +++++++++++++++++++++++ enterprise_catalog/apps/catalog/tasks.py | 1 + 2 files changed, 46 insertions(+) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index c3a4e6d9..c005be02 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -1283,3 +1283,48 @@ def current_options(cls): 'no_async': current_config.no_async, } return {} + +""" +SPIKE TEST: + +1. First, reset the sqlite database: + +rm enterprise_catalog/default.db && python manage.py migrate && python manage.py shell + +2. Then, paste everything below into the python shell: + +from enterprise_catalog.apps.catalog.models import * +from enterprise_catalog.apps.catalog.tests.factories import * +catalog = EnterpriseCatalogFactory() +catalog_query = catalog.catalog_query + +course_mixed = ContentMetadataFactory(content_key='mixed_course', content_type='course') +course_mixed.catalog_queries.set(CatalogQuery.objects.all()) +course_mixed_run1 = ContentMetadataFactory(content_key='course_mixed_run1', content_type='courserun') +course_mixed_run2 = ContentMetadataFactory(content_key='course_mixed_run2', content_type='courserun', is_restricted_run=True) +course_mixed_run2.catalog_queries.set(CatalogQuery.objects.all()) +# TODO: create restricted course for mixed course + +course_unicorn = ContentMetadataFactory(content_key='unicorn_course', content_type='course') +course_unicorn.catalog_queries.set(CatalogQuery.objects.all()) +course_unicorn_run1 = ContentMetadataFactory(content_key='course_unicorn_run1', content_type='courserun', is_restricted_run=True) +course_unicorn_run1.catalog_queries.set(CatalogQuery.objects.all()) +restricted_course_unicorn, _ = RestrictedCourseMetadata.objects.get_or_create( + content_key = course_unicorn.content_key, + content_type = course_unicorn.content_type, +) +restricted_course_unicorn.catalog_query = CatalogQuery.objects.first() +restricted_course_unicorn.unrestricted_parent = course_unicorn +restricted_course_unicorn.save() + +assert catalog.content_metadata[1].json_metadata +assert not catalog.content_metadata_with_restricted[2].json_metadata + +assert catalog.get_matching_content(['mixed_course'], include_restricted=False)[0].json_metadata +assert catalog.get_matching_content(['mixed_course'], include_restricted=True)[0].json_metadata +assert catalog.get_matching_content(['unicorn_course'], include_restricted=False)[0].json_metadata +assert not catalog.get_matching_content(['unicorn_course'], include_restricted=True)[0].json_metadata + +assert len(catalog.get_matching_content(['course_unicorn_run1'], include_restricted=False)) == 0 +assert len(catalog.get_matching_content(['course_unicorn_run1'], include_restricted=True)) == 1 +""" diff --git a/enterprise_catalog/apps/catalog/tasks.py b/enterprise_catalog/apps/catalog/tasks.py index 82f0e4df..e3785f4f 100644 --- a/enterprise_catalog/apps/catalog/tasks.py +++ b/enterprise_catalog/apps/catalog/tasks.py @@ -17,6 +17,7 @@ @shared_task(base=LoggedTask) def compare_catalog_queries_to_filters_task(): logger.info('compare_catalog_queries_to_filters starting...') + # NOTE: No need to exclude restricted runs because they are already filtered out via content_type. for content_metadata in ContentMetadata.objects.filter(content_type=COURSE): for enterprise_catalog in EnterpriseCatalog.objects.all(): try: From 44f9d70a2510b08a12cd6ef022a717eb8e019613 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Wed, 2 Oct 2024 09:24:50 -0700 Subject: [PATCH 08/14] beginnings of v2 views --- enterprise_catalog/apps/api/urls.py | 2 ++ enterprise_catalog/apps/api/v2/urls.py | 37 ++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 enterprise_catalog/apps/api/v2/urls.py diff --git a/enterprise_catalog/apps/api/urls.py b/enterprise_catalog/apps/api/urls.py index 153b5f93..af816a94 100644 --- a/enterprise_catalog/apps/api/urls.py +++ b/enterprise_catalog/apps/api/urls.py @@ -8,9 +8,11 @@ from django.urls import include, path from enterprise_catalog.apps.api.v1 import urls as v1_urls +from enterprise_catalog.apps.api.v2 import urls as v2_urls app_name = 'api' urlpatterns = [ path('v1/', include(v1_urls)), + path('21/', include(v2_urls)), ] diff --git a/enterprise_catalog/apps/api/v2/urls.py b/enterprise_catalog/apps/api/v2/urls.py new file mode 100644 index 00000000..57ee3f08 --- /dev/null +++ b/enterprise_catalog/apps/api/v2/urls.py @@ -0,0 +1,37 @@ +""" +URL definitions for enterprise catalog API version 1. +""" +from django.urls import path, re_path +from rest_framework.routers import DefaultRouter + +from enterprise_catalog.apps.api.v2.views.enterprise_catalog_contains_content_items import ( + EnterpriseCatalogContainsContentItems, +) +from enterprise_catalog.apps.api.v2.views.enterprise_catalog_get_content_metadata import ( + EnterpriseCatalogGetContentMetadata, +) +from enterprise_catalog.apps.api.v2.views.enterprise_customer import ( + EnterpriseCustomerViewSet, +) + + +app_name = 'v2' + +router = DefaultRouter() +router.register(r'enterprise-catalogs', EnterpriseCatalogContainsContentItems, basename='enterprise-catalog-content-v2') +router.register(r'enterprise-customer', EnterpriseCustomerViewSet, basename='enterprise-customer-v2') + +urlpatterns = [ + re_path( + r'^enterprise-catalogs/(?P[\S]+)/get_content_metadata', + EnterpriseCatalogGetContentMetadata.as_view({'get': 'get'}), + name='get-content-metadata-v2' + ), + path( + 'enterprise-customer//content-metadata//', + EnterpriseCustomerViewSet.as_view({'get': 'content_metadata'}), + name='customer-content-metadata-retrieve-v2' + ), +] + +urlpatterns += router.urls From a44a01dd572f5c001070edcdf4b9e32ade8283eb Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Wed, 2 Oct 2024 13:35:14 -0700 Subject: [PATCH 09/14] squash --- enterprise_catalog/apps/api/urls.py | 2 +- enterprise_catalog/apps/api/v2/__init__.py | 0 enterprise_catalog/apps/api/v2/urls.py | 16 ++++++++-------- enterprise_catalog/apps/api/v2/views/__init__.py | 0 .../enterprise_catalog_contains_content_items.py | 8 ++++++++ .../enterprise_catalog_get_content_metadata.py | 8 ++++++++ .../apps/api/v2/views/enterprise_customer.py | 10 ++++++++++ 7 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 enterprise_catalog/apps/api/v2/__init__.py create mode 100644 enterprise_catalog/apps/api/v2/views/__init__.py create mode 100644 enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py create mode 100644 enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py create mode 100644 enterprise_catalog/apps/api/v2/views/enterprise_customer.py diff --git a/enterprise_catalog/apps/api/urls.py b/enterprise_catalog/apps/api/urls.py index af816a94..097f4eea 100644 --- a/enterprise_catalog/apps/api/urls.py +++ b/enterprise_catalog/apps/api/urls.py @@ -14,5 +14,5 @@ app_name = 'api' urlpatterns = [ path('v1/', include(v1_urls)), - path('21/', include(v2_urls)), + path('v2/', include(v2_urls)), ] diff --git a/enterprise_catalog/apps/api/v2/__init__.py b/enterprise_catalog/apps/api/v2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/enterprise_catalog/apps/api/v2/urls.py b/enterprise_catalog/apps/api/v2/urls.py index 57ee3f08..1ad6f2e0 100644 --- a/enterprise_catalog/apps/api/v2/urls.py +++ b/enterprise_catalog/apps/api/v2/urls.py @@ -1,35 +1,35 @@ """ -URL definitions for enterprise catalog API version 1. +URL definitions for enterprise catalog API version 2. """ from django.urls import path, re_path from rest_framework.routers import DefaultRouter from enterprise_catalog.apps.api.v2.views.enterprise_catalog_contains_content_items import ( - EnterpriseCatalogContainsContentItems, + EnterpriseCatalogContainsContentItemsV2, ) from enterprise_catalog.apps.api.v2.views.enterprise_catalog_get_content_metadata import ( - EnterpriseCatalogGetContentMetadata, + EnterpriseCatalogGetContentMetadataV2, ) from enterprise_catalog.apps.api.v2.views.enterprise_customer import ( - EnterpriseCustomerViewSet, + EnterpriseCustomerViewSetV2, ) app_name = 'v2' router = DefaultRouter() -router.register(r'enterprise-catalogs', EnterpriseCatalogContainsContentItems, basename='enterprise-catalog-content-v2') -router.register(r'enterprise-customer', EnterpriseCustomerViewSet, basename='enterprise-customer-v2') +router.register(r'enterprise-catalogs', EnterpriseCatalogContainsContentItemsV2, basename='enterprise-catalog-content-v2') +router.register(r'enterprise-customer', EnterpriseCustomerViewSetV2, basename='enterprise-customer-v2') urlpatterns = [ re_path( r'^enterprise-catalogs/(?P[\S]+)/get_content_metadata', - EnterpriseCatalogGetContentMetadata.as_view({'get': 'get'}), + EnterpriseCatalogGetContentMetadataV2.as_view({'get': 'get'}), name='get-content-metadata-v2' ), path( 'enterprise-customer//content-metadata//', - EnterpriseCustomerViewSet.as_view({'get': 'content_metadata'}), + EnterpriseCustomerViewSetV2.as_view({'get': 'content_metadata'}), name='customer-content-metadata-retrieve-v2' ), ] diff --git a/enterprise_catalog/apps/api/v2/views/__init__.py b/enterprise_catalog/apps/api/v2/views/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py new file mode 100644 index 00000000..1a14bd1a --- /dev/null +++ b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py @@ -0,0 +1,8 @@ +from enterprise_catalog.apps.api.v1.views.enterprise_catalog_contains_content_items import EnterpriseCatalogContainsContentItems + + +class EnterpriseCatalogContainsContentItemsV2(EnterpriseCatalogContainsContentItems): + """ + View to determine if an enterprise catalog contains certain content + """ + pass diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py new file mode 100644 index 00000000..45334a98 --- /dev/null +++ b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py @@ -0,0 +1,8 @@ +from enterprise_catalog.apps.api.v1.views.enterprise_catalog_get_content_metadata import EnterpriseCatalogGetContentMetadata + + +class EnterpriseCatalogGetContentMetadataV2(EnterpriseCatalogGetContentMetadata): + """ + View for retrieving all the content metadata associated with a catalog. + """ + pass diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_customer.py b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py new file mode 100644 index 00000000..e8e98866 --- /dev/null +++ b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py @@ -0,0 +1,10 @@ +from enterprise_catalog.apps.api.v1.views.enterprise_customer import EnterpriseCustomerViewSet + + +class EnterpriseCustomerViewSetV2(EnterpriseCustomerViewSet): + """ + Viewset for operations on enterprise customers. + + Although we don't have a specific EnterpriseCustomer model, this viewset handles operations that use an enterprise + identifier to perform operations on their associated catalogs, etc. + """ From 304c96709907dc966c08b48e64b3564b7c572592 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Wed, 2 Oct 2024 14:50:47 -0700 Subject: [PATCH 10/14] squash --- .../views/enterprise_catalog_get_content_metadata.py | 12 +++++++++++- enterprise_catalog/apps/catalog/models.py | 5 +++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py index 45334a98..dc3f3ef3 100644 --- a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py +++ b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_get_content_metadata.py @@ -5,4 +5,14 @@ class EnterpriseCatalogGetContentMetadataV2(EnterpriseCatalogGetContentMetadata) """ View for retrieving all the content metadata associated with a catalog. """ - pass + def get_queryset(self, **kwargs): + """ + Returns all of the json of content metadata associated with the catalog. + """ + # Avoids ordering the content metadata by any field on that model to avoid using a temporary table / filesort + queryset = self.enterprise_catalog.content_metadata_with_restricted + content_filter = kwargs.get('content_keys_filter') + if content_filter: + queryset = self.enterprise_catalog.get_matching_content(content_keys=content_filter, include_restricted=True) + + return queryset.order_by('catalog_queries') diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index c005be02..0af12517 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -381,6 +381,11 @@ def get_matching_content(self, content_keys, include_restricted=False): # (if any) to handle the following case: # - catalog contains courses and the specified content_keys are course run ids. searched_metadata = ContentMetadata.objects.filter(content_key__in=content_keys) + if include_restricted: + # TODO: searched_metadata needs to exclude restricted runs that this catalog query is not allowed to see. + pass + else: + searched_metadata = searched_metadata.exclude(is_restricted_run=True) parent_content_keys = { metadata.parent_content_key for metadata in searched_metadata From ff57cb0e2f02905cefdc968137a4109cc9f76256 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Thu, 3 Oct 2024 09:48:14 -0700 Subject: [PATCH 11/14] squash --- enterprise_catalog/apps/catalog/models.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index 0af12517..0db26fce 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -382,9 +382,10 @@ def get_matching_content(self, content_keys, include_restricted=False): # - catalog contains courses and the specified content_keys are course run ids. searched_metadata = ContentMetadata.objects.filter(content_key__in=content_keys) if include_restricted: - # TODO: searched_metadata needs to exclude restricted runs that this catalog query is not allowed to see. - pass + # Only hide restricted runs that are not allowed by the catalog + searched_metadata = searched_metadata.exclude(Q(is_restricted_run=True) & ~Q(catalog_queries=self.catalog_query)) else: + # Hide ALL restricted runs. searched_metadata = searched_metadata.exclude(is_restricted_run=True) parent_content_keys = { metadata.parent_content_key From eaa91988b2d95e00ea4bdba4a42b44713015ae84 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Thu, 3 Oct 2024 15:08:50 -0700 Subject: [PATCH 12/14] squash --- .../apps/api/v2/views/enterprise_customer.py | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_customer.py b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py index e8e98866..0ca8a89b 100644 --- a/enterprise_catalog/apps/api/v2/views/enterprise_customer.py +++ b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py @@ -1,4 +1,14 @@ +import logging +import uuid + +from rest_framework.exceptions import NotFound + +from enterprise_catalog.apps.api.v1.serializers import ContentMetadataSerializer from enterprise_catalog.apps.api.v1.views.enterprise_customer import EnterpriseCustomerViewSet +from enterprise_catalog.apps.catalog.models import EnterpriseCatalog + + +logger = logging.getLogger(__name__) class EnterpriseCustomerViewSetV2(EnterpriseCustomerViewSet): @@ -8,3 +18,39 @@ class EnterpriseCustomerViewSetV2(EnterpriseCustomerViewSet): Although we don't have a specific EnterpriseCustomer model, this viewset handles operations that use an enterprise identifier to perform operations on their associated catalogs, etc. """ + def get_metadata_item_serializer(self): + """ + Gets the first matching serialized ContentMetadata for a requested ``content_identifier`` + associated with any of a requested ``customer_uuid``'s catalogs. + """ + enterprise_catalogs = list(EnterpriseCatalog.objects.filter( + enterprise_uuid=self.kwargs.get('enterprise_uuid') + )) + content_identifier = self.kwargs.get('content_identifier') + serializer_context = { + 'skip_customer_fetch': bool(self.request.query_params.get('skip_customer_fetch', '').lower()), + } + + try: + # Search for matching metadata if the value of the requested + # identifier is a valid UUID. + content_uuid = uuid.UUID(content_identifier) + for catalog in enterprise_catalogs: + content_with_uuid = catalog.content_metadata_with_restricted.filter(content_uuid=content_uuid) + if content_with_uuid: + return ContentMetadataSerializer( + content_with_uuid.first(), + context={'enterprise_catalog': catalog, **serializer_context}, + ) + except ValueError: + # Otherwise, search for matching metadata as a content key + for catalog in enterprise_catalogs: + content_with_key = catalog.get_matching_content(content_keys=[content_identifier], include_restricted=True) + if content_with_key: + return ContentMetadataSerializer( + content_with_key.first(), + context={'enterprise_catalog': catalog, **serializer_context}, + ) + # If we've made it here without finding a matching ContentMetadata record, + # assume no matching record exists and raise a 404. + raise NotFound(detail='No matching content in any catalog for this customer') From 2f8152a41936a1ad6c78e313351f315349bcf149 Mon Sep 17 00:00:00 2001 From: Troy Sankey Date: Fri, 4 Oct 2024 09:35:11 -0700 Subject: [PATCH 13/14] squash --- ...terprise_catalog_contains_content_items.py | 37 ++++++++++++++++++- enterprise_catalog/apps/catalog/models.py | 4 +- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py index 1a14bd1a..89cff73b 100644 --- a/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py +++ b/enterprise_catalog/apps/api/v2/views/enterprise_catalog_contains_content_items.py @@ -1,8 +1,41 @@ -from enterprise_catalog.apps.api.v1.views.enterprise_catalog_contains_content_items import EnterpriseCatalogContainsContentItems +""" +""" +from django.utils.decorators import method_decorator +from django.views.decorators.cache import cache_page +from rest_framework.decorators import action +from rest_framework.response import Response + +from enterprise_catalog.apps.api.constants import ( + CONTAINS_CONTENT_ITEMS_VIEW_CACHE_TIMEOUT_SECONDS, +) +from enterprise_catalog.apps.api.v1.decorators import ( + require_at_least_one_query_parameter, +) +from enterprise_catalog.apps.api.v1.utils import unquote_course_keys +from enterprise_catalog.apps.api.v1.views.enterprise_catalog_contains_content_items import ( + EnterpriseCatalogContainsContentItems, +) class EnterpriseCatalogContainsContentItemsV2(EnterpriseCatalogContainsContentItems): """ View to determine if an enterprise catalog contains certain content """ - pass + @method_decorator(cache_page(CONTAINS_CONTENT_ITEMS_VIEW_CACHE_TIMEOUT_SECONDS)) + @method_decorator(require_at_least_one_query_parameter('course_run_ids', 'program_uuids')) + @action(detail=True) + def contains_content_items(self, request, uuid, course_run_ids, program_uuids, **kwargs): # pylint: disable=unused-argument + """ + Returns whether or not the EnterpriseCatalog contains the specified content. + + Multiple course_run_ids and/or program_uuids query parameters can be sent to this view to check for their + existence in the specified enterprise catalog. + """ + course_run_ids = unquote_course_keys(course_run_ids) + + enterprise_catalog = self.get_object() + contains_content_items = enterprise_catalog.contains_content_keys( + course_run_ids + program_uuids, + include_restricted=True, + ) + return Response({'contains_content_items': contains_content_items}) diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index 0db26fce..beb4c444 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -398,7 +398,7 @@ def get_matching_content(self, content_keys, include_restricted=False): else: return self.content_metadata.filter(query) - def contains_content_keys(self, content_keys): + def contains_content_keys(self, content_keys, include_restricted=False): """ Determines whether the given ``content_keys`` are part of the catalog. @@ -415,7 +415,7 @@ def contains_content_keys(self, content_keys): in the ``content_keys`` list (to handle cases when a catalog contains only courses, but course run keys are provided in the ``content_keys`` argument). """ - included_content = self.get_matching_content(content_keys) + included_content = self.get_matching_content(content_keys, include_restricted=include_restricted) return included_content.exists() def filter_content_keys(self, content_keys): From 1865ace8e70f5b3e2c6c14ba4b2e9374adb692d4 Mon Sep 17 00:00:00 2001 From: Ubuntu Date: Fri, 4 Oct 2024 17:59:54 +0000 Subject: [PATCH 14/14] squash --- .../apps/api/v2/views/enterprise_customer.py | 68 +++++++++++++++++++ enterprise_catalog/apps/catalog/models.py | 8 ++- 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/enterprise_catalog/apps/api/v2/views/enterprise_customer.py b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py index 0ca8a89b..d1192be0 100644 --- a/enterprise_catalog/apps/api/v2/views/enterprise_customer.py +++ b/enterprise_catalog/apps/api/v2/views/enterprise_customer.py @@ -7,6 +7,16 @@ from enterprise_catalog.apps.api.v1.views.enterprise_customer import EnterpriseCustomerViewSet from enterprise_catalog.apps.catalog.models import EnterpriseCatalog +from django.utils.decorators import method_decorator +from rest_framework.decorators import action +from rest_framework.response import Response +from rest_framework.status import HTTP_400_BAD_REQUEST + +from enterprise_catalog.apps.api.v1.decorators import ( + require_at_least_one_query_parameter, +) +from enterprise_catalog.apps.api.v1.utils import unquote_course_keys + logger = logging.getLogger(__name__) @@ -18,6 +28,64 @@ class EnterpriseCustomerViewSetV2(EnterpriseCustomerViewSet): Although we don't have a specific EnterpriseCustomer model, this viewset handles operations that use an enterprise identifier to perform operations on their associated catalogs, etc. """ + @method_decorator(require_at_least_one_query_parameter('course_run_ids', 'program_uuids')) + @action(detail=True) + def contains_content_items(self, request, enterprise_uuid, course_run_ids, program_uuids, **kwargs): + """ + Returns whether or not the specified content is available for the given enterprise. + --- + parameters: + - name: course_run_ids + description: Ids of the course runs to check availability of + paramType: query + - name: program_uuids + description: Uuids of the programs to check availability of + paramType: query + - name: get_catalog_list + description: [Old parameter] Return a list of catalogs in which the course / program is present + paramType: query + - name: get_catalogs_containing_specified_content_ids + description: Return a list of catalogs in which the course / program is present + paramType: query + """ + get_catalogs_containing_specified_content_ids = request.GET.get( + 'get_catalogs_containing_specified_content_ids', False + ) + get_catalog_list = request.GET.get('get_catalog_list', False) + requested_course_or_run_keys = unquote_course_keys(course_run_ids) + + try: + uuid.UUID(enterprise_uuid) + except ValueError as exc: + logger.warning( + f"Could not parse catalogs from provided enterprise uuid: {enterprise_uuid}. " + f"Query failed with exception: {exc}" + ) + return Response( + f'Error: invalid enterprice customer uuid: "{enterprise_uuid}" provided.', + status=HTTP_400_BAD_REQUEST + ) + customer_catalogs = EnterpriseCatalog.objects.filter(enterprise_uuid=enterprise_uuid) + + any_catalog_contains_content_items = False + catalogs_that_contain_course = [] + for catalog in customer_catalogs: + contains_content_items = catalog.contains_content_keys(requested_course_or_run_keys + program_uuids, include_restricted=True) + if contains_content_items: + any_catalog_contains_content_items = True + if not (get_catalogs_containing_specified_content_ids or get_catalog_list): + # Break as soon as we find a catalog that contains the specified content + break + catalogs_that_contain_course.append(catalog.uuid) + + response_data = { + 'contains_content_items': any_catalog_contains_content_items, + } + if (get_catalogs_containing_specified_content_ids or get_catalog_list): + response_data['catalog_list'] = catalogs_that_contain_course + + return Response(response_data) + def get_metadata_item_serializer(self): """ Gets the first matching serialized ContentMetadata for a requested ``content_identifier`` diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index beb4c444..3235552f 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -418,7 +418,7 @@ def contains_content_keys(self, content_keys, include_restricted=False): included_content = self.get_matching_content(content_keys, include_restricted=include_restricted) return included_content.exists() - def filter_content_keys(self, content_keys): + def filter_content_keys(self, content_keys, include_restricted=False): """ Determines whether content_keys are part of the catalog. @@ -448,7 +448,11 @@ def filter_content_keys(self, content_keys): query = Q(content_key__in=content_keys) | Q(parent_content_key__in=content_keys) items_included = set() - for content in self.content_metadata.filter(query).all(): + if include_restricted: + accessible_metadata_qs = self.content_metadata_with_restricted + else: + accessible_metadata_qs = self.content_metadata + for content in accessible_metadata_qs.filter(query).all(): if content.content_key in content_keys: items_included.add(content.content_key) elif content.parent_content_key in content_keys: