feat: take exec ed course data from course run instead of additional_…

…metadata attempt 2

fix: calculate enroll_by correctly

test: fix unit tests pt2

enterprise_catalog/apps/api/tasks.py

@@ -288,17 +288,6 @@ def _update_full_content_metadata_course(content_keys, dry_run=False):
             # Merge the full metadata from discovery's /api/v1/courses into the local metadata object.
             metadata_record.json_metadata.update(course_metadata_dict)
 
-            # Exec ed provides the start/end dates in additional_metadata, so we should copy those over to the keys that
-            # we use (inside the advertised course run).
-            if metadata_record.is_exec_ed_2u_course:
-                json_meta = metadata_record.json_metadata
-                start_date = json_meta.get('additional_metadata', {}).get('start_date')
-                end_date = json_meta.get('additional_metadata', {}).get('end_date')
-                course_run_uuid = json_meta.get('advertised_course_run_uuid')
-                for run in json_meta.get('course_runs', []):
-                    if run.get('uuid') == course_run_uuid:
-                        run.update({'start': start_date, 'end': end_date})
-
             # Perform more steps to normalize and move keys around
             # for more consistency across content types.
             metadata_record.json_metadata['normalized_metadata'] =\

enterprise_catalog/apps/api/tests/test_tasks.py

@@ -450,6 +450,23 @@ def test_update_full_metadata(self, mock_oauth_client, mock_partition_course_key
                 ]
             }],
             'advertised_course_run_uuid': course_run_3_uuid,
+            'advertised_course_run': {
+                'key': f'course-v1:{course_key_3}+1',
+                'uuid': course_run_3_uuid,
+                'start': '2023-03-01T00:00:00Z',
+                'end': '2023-03-01T00:00:00Z',
+                'first_enrollable_paid_seat_price': 90,
+                'seats': [
+                    {
+                        'type': CourseMode.VERIFIED,
+                        'upgrade_deadline': '2023-02-01T00:00:00Z',
+                    },
+                    {
+                        "type": str(CourseMode.PROFESSIONAL),
+                        "upgrade_deadline": '2022-02-01T00:00:00Z',
+                    },
+                ]
+            },
         }
         course_key_4 = 'edX+superDuperFakeX'
         course_data_4 = {'key': course_key_4, 'full_course_only_field': 'test_4', 'programs': []}
@@ -640,8 +657,10 @@ def test_update_full_metadata_exec_ed(self, mock_oauth_client, mock_partition_co
                 'key': course_run_key,
                 'uuid': course_run_uuid,
                 # Use dummy 2022 dates that we will assert are overwritten.
-                'start': '2022-03-01T00:00:00Z',
-                'end': '2022-03-01T00:00:00Z',
+                'start': '2023-03-01T00:00:00Z',
+                'end': '2023-04-09T23:59:59Z',
+                'enrollment_end': '2023-02-01T00:00:00Z',
+                "first_enrollable_paid_seat_price": 2900,
             }],
             'programs': [],
             'additional_metadata': {

enterprise_catalog/apps/api/v1/export_utils.py

@@ -169,67 +169,98 @@ def program_hit_to_row(hit):
     return csv_row
 
 
-# pylint: disable=too-many-statements
-def course_hit_to_row(hit):
-    """
-    Helper function to construct a CSV row according to a single Algolia result course hit.
-    """
-    csv_row = []
-    csv_row.append(hit.get('title'))
-
-    if hit.get('partners'):
-        csv_row.append(hit['partners'][0]['name'])
-    else:
-        csv_row.append(None)
+def _base_csv_row_data(hit):
+    """ Returns the formatted, shared attributes common across all course types. """
+    title = hit.get('title')
+    aggregation_key = hit.get('aggregation_key')
+    language = hit.get('language')
+    transcript_languages = ', '.join(hit.get('transcript_languages', []))
+    marketing_url = hit.get('marketing_url')
+    short_description = strip_tags(hit.get('short_description', ''))
+    subjects = ', '.join(hit.get('subjects', []))
+    skills = ', '.join([skill['name'] for skill in hit.get('skills', [])])
+    outcome = strip_tags(hit.get('outcome', ''))  # What You’ll Learn
+
+    # FIXME: currently ignores partner names when a course has multiple partners
+    partner_name = hit['partners'][0]['name'] if hit.get('partners') else None
 
     empty_advertised_course_run = {}
     advertised_course_run = hit.get('advertised_course_run', empty_advertised_course_run)
+    course_run_key = advertised_course_run.get('key')
+    min_effort = advertised_course_run.get('min_effort')
+    max_effort = advertised_course_run.get('max_effort')
+    weeks_to_complete = advertised_course_run.get('weeks_to_complete')  # Length
+
     if start_date := advertised_course_run.get('start'):
         start_date = parser.parse(start_date).strftime(DATE_FORMAT)
-    csv_row.append(start_date)
 
     if end_date := advertised_course_run.get('end'):
         end_date = parser.parse(end_date).strftime(DATE_FORMAT)
-    csv_row.append(end_date)
-
-    # upgrade_deadline deprecated in favor of enroll_by
-    if upgrade_deadline := advertised_course_run.get('upgrade_deadline'):
-        upgrade_deadline = datetime.datetime.fromtimestamp(upgrade_deadline).strftime(DATE_FORMAT)
-    csv_row.append(upgrade_deadline)
 
     if enroll_by := advertised_course_run.get('enroll_by'):
         enroll_by = datetime.datetime.fromtimestamp(enroll_by).strftime(DATE_FORMAT)
-    csv_row.append(enroll_by)
 
-    pacing_type = advertised_course_run.get('pacing_type')
-    key = advertised_course_run.get('key')
+    return {
+        'title': title,
+        'partner_name': partner_name,
+        'start_date': start_date,
+        'end_date': end_date,
+        'enroll_by': enroll_by,
+        'aggregation_key': aggregation_key,
+        'course_run_key': course_run_key,
+        'language': language,
+        'transcript_languages': transcript_languages,
+        'marketing_url': marketing_url,
+        'short_description': short_description,
+        'subjects': subjects,
+        'skills': skills,
+        'min_effort': min_effort,
+        'max_effort': max_effort,
+        'weeks_to_complete': weeks_to_complete,
+        'outcome': outcome,
+        'advertised_course_run': advertised_course_run,
+    }
 
+
+def course_hit_to_row(hit):
+    """
+    Helper function to construct a CSV row according to a single Algolia result course hit.
+    """
+    row_data = _base_csv_row_data(hit)
+    csv_row = []
+    csv_row.append(row_data.get('title'))
+    csv_row.append(row_data.get('partner_name'))
+
+    advertised_course_run = row_data.get('advertised_course_run')
+
+    csv_row.append(row_data.get('start_date'))
+    csv_row.append(row_data.get('end_date'))
+
+    # upgrade_deadline deprecated in favor of enroll_by
+    if upgrade_deadline := advertised_course_run.get('upgrade_deadline'):
+        upgrade_deadline = datetime.datetime.fromtimestamp(upgrade_deadline).strftime(DATE_FORMAT)
+    csv_row.append(upgrade_deadline)
+    csv_row.append(row_data.get('enroll_by'))
     csv_row.append(', '.join(hit.get('programs', [])))
     csv_row.append(', '.join(hit.get('program_titles', [])))
 
+    pacing_type = advertised_course_run.get('pacing_type')
     csv_row.append(pacing_type)
 
     csv_row.append(hit.get('level_type'))
-
     csv_row.append(hit.get('first_enrollable_paid_seat_price'))
-    csv_row.append(hit.get('language'))
-    csv_row.append(', '.join(hit.get('transcript_languages', [])))
-    csv_row.append(hit.get('marketing_url'))
-    csv_row.append(strip_tags(hit.get('short_description', '')))
-
-    csv_row.append(', '.join(hit.get('subjects', [])))
-    csv_row.append(key)
-    csv_row.append(hit.get('aggregation_key'))
-
-    skills = [skill['name'] for skill in hit.get('skills', [])]
-    csv_row.append(', '.join(skills))
-
-    advertised_course_run = hit.get('advertised_course_run', {})
-    csv_row.append(advertised_course_run.get('min_effort'))
-    csv_row.append(advertised_course_run.get('max_effort'))
-    csv_row.append(advertised_course_run.get('weeks_to_complete'))  # Length
-
-    csv_row.append(strip_tags(hit.get('outcome', '')))  # What You’ll Learn
+    csv_row.append(row_data.get('language'))
+    csv_row.append(row_data.get('transcript_languages'))
+    csv_row.append(row_data.get('marketing_url'))
+    csv_row.append(row_data.get('short_description'))
+    csv_row.append(row_data.get('subjects'))
+    csv_row.append(row_data.get('course_run_key'))
+    csv_row.append(row_data.get('aggregation_key'))
+    csv_row.append(row_data.get('skills'))
+    csv_row.append(row_data.get('min_effort'))
+    csv_row.append(row_data.get('max_effort'))
+    csv_row.append(row_data.get('weeks_to_complete'))
+    csv_row.append(row_data.get('outcome'))
 
     csv_row.append(strip_tags(hit.get('prerequisites_raw', '')))  # Pre-requisites
 
@@ -239,71 +270,33 @@ def course_hit_to_row(hit):
     return csv_row
 
 
-def fetch_and_format_registration_date(obj):
-    enroll_by_date = obj.get('registration_deadline')
-    stripped_enroll_by = enroll_by_date.split("T")[0]
-    formatted_enroll_by = None
-    try:
-        enroll_by_datetime_obj = datetime.datetime.strptime(stripped_enroll_by, '%Y-%m-%d')
-        formatted_enroll_by = enroll_by_datetime_obj.strftime('%m-%d-%Y')
-    except ValueError as exc:
-        logger.info(f"Unable to format registration deadline, failed with error: {exc}")
-    return formatted_enroll_by
-
-
 def exec_ed_course_to_row(hit):
     """
     Helper function to construct a CSV row according to a single executive education course hit.
     """
+    row_data = _base_csv_row_data(hit)
     csv_row = []
-    csv_row.append(hit.get('title'))
+    csv_row.append(row_data.get('title'))
+    csv_row.append(row_data.get('partners'))
 
-    if hit.get('partners'):
-        csv_row.append(hit['partners'][0]['name'])
-    else:
-        csv_row.append(None)
-    if hit.get('additional_metadata'):
-        start_date = None
-        additional_md = hit['additional_metadata']
-        if additional_md.get('start_date'):
-            start_date = parser.parse(additional_md['start_date']).strftime(DATE_FORMAT)
-        csv_row.append(start_date)
-
-        end_date = None
-        if additional_md.get('end_date'):
-            end_date = parser.parse(additional_md['end_date']).strftime(DATE_FORMAT)
-        csv_row.append(end_date)
-        formatted_enroll_by = fetch_and_format_registration_date(additional_md)
-    else:
-        csv_row.append(None)  # no start date
-        csv_row.append(None)  # no end date
-        formatted_enroll_by = None
-
-    csv_row.append(formatted_enroll_by)
-
-    adv_course_run = hit.get('advertised_course_run', {})
-    key = adv_course_run.get('key')
+    csv_row.append(row_data.get('start_date'))
+    csv_row.append(row_data.get('end_date'))
+    csv_row.append(row_data.get('enroll_by'))
 
     price = float(hit['entitlements'][0]['price'])
     csv_row.append(math.trunc(price))
-    csv_row.append(hit.get('language'))
-    csv_row.append(', '.join(hit.get('transcript_languages', [])))
-    csv_row.append(hit.get('marketing_url'))
-    csv_row.append(strip_tags(hit.get('short_description', '')))
-
-    csv_row.append(', '.join(hit.get('subjects', [])))
-    csv_row.append(key)
-    csv_row.append(hit.get('aggregation_key'))
-
-    skills = [skill['name'] for skill in hit.get('skills', [])]
-    csv_row.append(', '.join(skills))
-
-    csv_row.append(adv_course_run.get('min_effort'))
-    csv_row.append(adv_course_run.get('max_effort'))
-    csv_row.append(adv_course_run.get('weeks_to_complete'))  # Length
-
-    csv_row.append(strip_tags(hit.get('outcome', '')))  # What You’ll Learn
-
+    csv_row.append(row_data.get('language'))
+    csv_row.append(row_data.get('transcript_languages'))
+    csv_row.append(row_data.get('marketing_url'))
+    csv_row.append(row_data.get('short_description'))
+    csv_row.append(row_data.get('subjects'))
+    csv_row.append(row_data.get('course_run_key'))
+    csv_row.append(row_data.get('aggregation_key'))
+    csv_row.append(row_data.get('skills'))
+    csv_row.append(row_data.get('min_effort'))
+    csv_row.append(row_data.get('max_effort'))
+    csv_row.append(row_data.get('weeks_to_complete'))
+    csv_row.append(row_data.get('outcome'))
     csv_row.append(strip_tags(hit.get('full_description', '')))
 
     return csv_row

enterprise_catalog/apps/api/v1/tests/test_export_utils.py

@@ -15,16 +15,3 @@ def test_retrieve_available_fields(self):
         """
         # assert that ALGOLIA_ATTRIBUTES_TO_RETRIEVE is a SUBSET of ALGOLIA_FIELDS
         assert set(export_utils.ALGOLIA_ATTRIBUTES_TO_RETRIEVE) <= set(algolia_utils.ALGOLIA_FIELDS)
-
-    def test_fetch_and_format_registration_date(self):
-        """
-        Test the export properly fetches executive education registration dates
-        """
-        # expected hit format from algolia, porperly reformatted for csv download
-        assert export_utils.fetch_and_format_registration_date(
-            {'registration_deadline': '2002-02-15T12:12:200'}
-        ) == '02-15-2002'
-        # some other format from algolia, should return None
-        assert export_utils.fetch_and_format_registration_date(
-            {'registration_deadline': '02-15-2015T12:12:200'}
-        ) is None

enterprise_catalog/apps/catalog/algolia_utils.py

@@ -243,20 +243,18 @@ def _has_enroll_by_deadline_passed(course_json_metadata, advertised_course_run):
     Helper to determine if the enrollment deadline has passed for the given course
     and advertised course run.  For course metadata records with a `course_type` of "course" (e.g. OCM courses),
     this is based on the verified upgrade deadline.
-    For 2u exec ed courses, this is based on the registration deadline.
+    For 2u exec ed courses, this is based on the enrollment_end deadline.
     """
     enroll_by_deadline_timestamp = 0
     if course_json_metadata.get('course_type') == EXEC_ED_2U_COURSE_TYPE:
-        additional_metadata = course_json_metadata.get('additional_metadata') or {}
-        registration_deadline = additional_metadata.get('registration_deadline')
-        if registration_deadline:
+        enrollment_end = advertised_course_run.get('end') or {}
+        if enrollment_end:
             enroll_by_deadline_timestamp = datetime.datetime.strptime(
-                registration_deadline,
+                enrollment_end,
                 '%Y-%m-%dT%H:%M:%S%z',
             ).timestamp()
     else:
         enroll_by_deadline_timestamp = _get_verified_upgrade_deadline(advertised_course_run)
-
     return enroll_by_deadline_timestamp < localized_utcnow().timestamp()
 
 
@@ -1103,6 +1101,7 @@ def _get_course_run(full_course_run):
         'availability': full_course_run.get('availability'),
         'start': full_course_run.get('start'),
         'end': full_course_run.get('end'),
+        'enrollment_end': full_course_run.get('enrollment_end'),
         'min_effort': full_course_run.get('min_effort'),
         'max_effort': full_course_run.get('max_effort'),
         'weeks_to_complete': full_course_run.get('weeks_to_complete'),

enterprise_catalog/apps/catalog/serializers.py

@@ -95,15 +95,8 @@ def advertised_course_run(self):
         advertised_course_run_uuid = self.instance.json_metadata.get('advertised_course_run_uuid')
         return _get_course_run_by_uuid(self.instance.json_metadata, advertised_course_run_uuid)
 
-    @cached_property
-    def additional_metadata(self):
-        return self.instance.json_metadata.get('additional_metadata', {})
-
     @extend_schema_field(serializers.DateTimeField)
-    def get_start_date(self, obj) -> str:
-        if obj.is_exec_ed_2u_course:
-            return self.additional_metadata.get('start_date')
-
+    def get_start_date(self, obj) -> str:  # pylint: disable=unused-argument
         if not self.advertised_course_run:
             return None
 
@@ -113,10 +106,7 @@ def get_start_date(self, obj) -> str:
         return None
 
     @extend_schema_field(serializers.DateTimeField)
-    def get_end_date(self, obj) -> str:
-        if obj.is_exec_ed_2u_course:
-            return self.additional_metadata.get('end_date')
-
+    def get_end_date(self, obj) -> str:  # pylint: disable=unused-argument
         if not self.advertised_course_run:
             return None
 
@@ -131,19 +121,21 @@ def get_enroll_by_date(self, obj) -> str:
             return None
 
         if obj.is_exec_ed_2u_course:
-            return (
-                self.advertised_course_run.get('enrollment_end')
-                or self.additional_metadata.get('registration_deadline')
-            )
-
-        upgrade_deadline = None
+            return self.advertised_course_run.get('enrollment_end')
 
         all_seats = self.advertised_course_run.get('seats', [])
         seat = _find_best_mode_seat(all_seats)
         if seat:
-            upgrade_deadline = seat.get('upgrade_deadline_override') or seat.get('upgrade_deadline')
-
-        return upgrade_deadline or self.advertised_course_run.get('enrollment_end')
+            return seat.get('upgrade_deadline_override') or seat.get('upgrade_deadline')
+
+        if enrollment_end := self.advertised_course_run.get('enrollment_end'):
+            return enrollment_end
+        else:
+            logger.info(
+                f"No Seat Found for course run '{self.advertised_course_run.get('key')}'. "
+                f"Seats: {all_seats}"
+            )
+            return None
 
     @extend_schema_field(serializers.FloatField)
     def get_content_price(self, obj) -> float: