Merge pull request #959 from openedx/iahmad/ENT-9470

feat: Added models and commands for enterprise-jobs association

.annotation_safe_list.yml

@@ -63,3 +63,9 @@ video_catalog.HistoricalVideoTranscriptSummary:
   ".. no_pii:": "This model has no PII"
 video_catalog.HistoricalVideoSkill:
   ".. no_pii:": "This model has no PII"
+jobs.HistoricalJob:
+  ".. no_pii:": "This model has no PII"
+jobs.HistoricalJobEnterprise:
+  ".. no_pii:": "This model has no PII"
+jobs.HistoricalJobSkill:
+  ".. no_pii:": "This model has no PII"

enterprise_catalog/apps/api_client/constants.py

@@ -12,6 +12,7 @@
 DISCOVERY_PROGRAMS_ENDPOINT = urljoin(settings.DISCOVERY_SERVICE_API_URL, 'programs/')
 DISCOVERY_COURSE_REVIEWS_ENDPOINT = urljoin(settings.DISCOVERY_SERVICE_API_URL, 'course_review/')
 DISCOVERY_VIDEO_SKILLS_ENDPOINT = urljoin(settings.DISCOVERY_SERVICE_URL, 'taxonomy/api/v1/xblocks/')
+DISCOVERY_JOBS_SKILLS_ENDPOINT = urljoin(settings.DISCOVERY_SERVICE_URL, 'taxonomy/api/v1/jobs/')
 DISCOVERY_OFFSET_SIZE = 200
 DISCOVERY_CATALOG_QUERY_CACHE_KEY_TPL = 'catalog_query:{id}'
 DISCOVERY_AVERAGE_COURSE_REVIEW_CACHE_KEY = 'average_course_review'

enterprise_catalog/apps/api_client/discovery.py

@@ -21,6 +21,7 @@
 from .constants import (
     DISCOVERY_COURSE_REVIEWS_ENDPOINT,
     DISCOVERY_COURSES_ENDPOINT,
+    DISCOVERY_JOBS_SKILLS_ENDPOINT,
     DISCOVERY_OFFSET_SIZE,
     DISCOVERY_PROGRAMS_ENDPOINT,
     DISCOVERY_SEARCH_ALL_ENDPOINT,
@@ -287,6 +288,69 @@ def get_video_skills(self, video_usage_key):
 
         return video_skills
 
+    def _retrieve_jobs_skills(self, request_params):
+        """
+        Makes a request to discovery's taxonomy/api/v1/jobs paginated endpoint
+        """
+        page = request_params.get('page', 1)
+        LOGGER.info(f'Retrieving video skills from course-discovery for page {page}...')
+        attempts = 0
+        while True:
+            attempts = attempts + 1
+            successful = True
+            exception = None
+            try:
+                response = self.client.get(
+                    DISCOVERY_JOBS_SKILLS_ENDPOINT,
+                    params=request_params,
+                    timeout=self.HTTP_TIMEOUT,
+                )
+                successful = response.status_code < 400
+                elapsed_seconds = response.elapsed.total_seconds()
+                LOGGER.info(
+                    f'Retrieved jobs skills results from course-discovery for page {page} in '
+                    f'retrieve_jobs_skills_seconds={elapsed_seconds} seconds.'
+                )
+            except requests.exceptions.RequestException as err:
+                exception = err
+                LOGGER.exception(f'Error while retrieving jobs skills results from course-discovery for page {page}')
+                successful = False
+            if attempts <= self.MAX_RETRIES and not successful:
+                sleep_seconds = self._calculate_backoff(attempts)
+                LOGGER.warning(
+                    f'failed request detected from {DISCOVERY_JOBS_SKILLS_ENDPOINT}, '
+                    'backing-off before retrying, '
+                    f'sleeping {sleep_seconds} seconds...'
+                )
+                time.sleep(sleep_seconds)
+            else:
+                if exception:
+                    raise exception
+                break
+        try:
+            return response.json()
+        except requests.exceptions.JSONDecodeError as err:
+            LOGGER.exception(
+                f'Invalid JSON while retrieving jobs skills results from course-discovery for page {page}, '
+                f'resonse status code: {response.status_code}, '
+                f'response body: {response.text}'
+            )
+            raise err
+
+    def get_jobs_skills(self, page=1):
+        """
+        Return results from the discovery service's taxonomy/api/v1/jobs endpoint
+        """
+        results = []
+        request_params = {'page': page}
+        try:
+            response = self._retrieve_jobs_skills(request_params)
+            results = response.get('results', [])
+            return results, response.get('next')
+        except Exception as exc:
+            LOGGER.exception(f'Could not retrieve jobs and skills from course-discovery (page {page}) {exc}')
+            raise exc
+
     def get_metadata_by_query(self, catalog_query, extra_query_params=None):
         """
         Return results from the discovery service's search/all endpoint.

enterprise_catalog/apps/jobs/__init__.py

@@ -0,0 +1,3 @@
+"""
+Job app - implementation for enterprise-job relationship.
+"""

enterprise_catalog/apps/jobs/admin.py

@@ -0,0 +1,34 @@
+"""
+Admin for jobs models.
+"""
+from django.contrib import admin
+from simple_history.admin import SimpleHistoryAdmin
+
+from enterprise_catalog.apps.jobs.models import Job, JobEnterprise, JobSkill
+
+
+@admin.register(Job)
+class JobAdmin(admin.ModelAdmin):
+    """
+    Django admin for Jobs.
+    """
+    list_display = ('job_id', 'external_id', 'title', 'description', )
+    search_fields = ('job_id', 'title', 'description', )
+
+
+@admin.register(JobEnterprise)
+class JobEnterpriseAdmin(SimpleHistoryAdmin):
+    """
+    Django admin for Enterprise Jobs.
+    """
+    list_display = ('enterprise_uuid', 'created', 'modified', )
+    search_fields = ('enterprise_uuid', )
+
+
+@admin.register(JobSkill)
+class JobSkillAdmin(SimpleHistoryAdmin):
+    """
+    Django admin for Job Skills.
+    """
+    list_display = ('skill_id', 'name', 'significance', 'created', 'modified',)
+    search_fields = ('name', )

enterprise_catalog/apps/jobs/apps.py

@@ -0,0 +1,9 @@
+"""
+Job app - implementation for enterprise-job relationship.
+"""
+from django.apps import AppConfig
+
+
+class JobConfig(AppConfig):
+    name = 'jobs'
+    default = False

enterprise_catalog/apps/jobs/management/commands/fetch_jobs_skills.py

@@ -0,0 +1,65 @@
+"""
+Management command for fetching jobs skills from taxonomy connector
+"""
+import logging
+
+from django.core.management.base import BaseCommand
+
+from enterprise_catalog.apps.api_client.discovery import DiscoveryApiClient
+from enterprise_catalog.apps.jobs.models import Job, JobSkill
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Management command for fetching job skills from taxonomy connector
+
+    Example Usage:
+    >> python manage.py fetch_jobs_skills
+    """
+    help = (
+        'Fetch the skills associated with jobs from taxonomy connector'
+    )
+
+    def _process_job_skills(self, results):
+        """
+        Process the job skills fetched from taxonomy connector.
+        """
+        for result in results:
+            try:
+                job, _ = Job.objects.update_or_create(
+                    job_id=result.get('id'),
+                    title=result.get('name'),
+                    description=result.get('description'),
+                    external_id=result.get('external_id')
+                )
+                job_skills = result.get('skills')
+                for item in job_skills:
+                    skill = item.get('skill')
+                    JobSkill.objects.update_or_create(
+                        job=job,
+                        skill_id=skill.get('id'),
+                        name=skill.get('name'),
+                        significance=item.get('significance')
+                    )
+            except Exception as exc:  # pylint: disable=broad-exception-caught
+                job_id = result.get('id')
+                logger.exception(f'Could not store job skills. job id {job_id} {exc}')
+
+    def handle(self, *args, **options):
+        """
+        Fetch the skills associated with jobs from taxonomy connector.
+        """
+        page = 1
+        try:
+            results, has_next = DiscoveryApiClient().get_jobs_skills(page=page)
+            self._process_job_skills(results)
+
+            while has_next:
+                page += 1
+                results, has_next = DiscoveryApiClient().get_jobs_skills(page=page)
+                self._process_job_skills(results)
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            logger.exception(f'Could not retrieve job skills for page {page} {exc}')

enterprise_catalog/apps/jobs/management/commands/process_enterprise_jobs.py

@@ -0,0 +1,71 @@
+"""
+Management command for associating jobs with enterprises
+"""
+import logging
+
+from django.core.management.base import BaseCommand
+
+from enterprise_catalog.apps.catalog.algolia_utils import (
+    get_initialized_algolia_client,
+)
+from enterprise_catalog.apps.catalog.models import EnterpriseCatalog
+from enterprise_catalog.apps.jobs.models import Job, JobEnterprise, JobSkill
+
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    """
+    Management command for associating jobs with enterprises via common skills
+
+    Example Usage:
+    >> python manage.py process_enterprise_jobs
+    """
+    help = (
+        'Associate jobs with enterprises via common skills'
+    )
+
+    def handle(self, *args, **options):
+        """
+        Associate jobs with enterprises via common skills
+        """
+        logger.info("Generating enterprise job association...")
+        algolia_client = get_initialized_algolia_client()
+        enterprise_uuids = set()
+        enterprise_catalogs = EnterpriseCatalog.objects.all()
+        for enterprise_catalog in enterprise_catalogs:
+            enterprise_uuids.add(enterprise_catalog.enterprise_uuid)
+
+        jobs = Job.objects.all()
+        associated_jobs = set()
+        for enterprise_uuid in enterprise_uuids:
+            for job in jobs:
+                try:
+                    job_skills = JobSkill.objects.filter(job=job).order_by('-significance')[:3]  # Get top 3 skills
+                    search_query = {
+                        'filters': f'(skill_names:{job_skills[0].name} OR \
+                            skill_names:{job_skills[1].name} OR \
+                            skill_names:{job_skills[2]}.name) AND \
+                            enterprise_customer_uuids:{enterprise_uuid}',
+                        'maxFacetHits': 50
+                    }
+                    response = algolia_client.algolia_index.search_for_facet_values('skill_names', '', search_query)
+                    for hit in response.get('facetHits', []):
+                        if hit.get('count') > 1:
+                            JobEnterprise.objects.update_or_create(
+                                job=job,
+                                enterprise_uuid=enterprise_uuid
+                            )
+                            associated_jobs.add(job.job_id)
+                            break
+                except Exception:  # pylint: disable=broad-exception-caught
+                    logger.error(
+                        '[PROCESS_ENTERPRISE_JOBS] Failure in processing \
+                        enterprise "%s" and job: "%s".',
+                        enterprise_uuid,
+                        job.job_id,
+                        exc_info=True
+                    )
+                else:
+                    JobEnterprise.objects.all().exclude(job_id__in=associated_jobs).delete()