Merge branch 'development' into HYP-280

* development: (25 commits)
  HYP-HOTFIX-091323 - Set uploads to use S3v4 signatures
  HYP-298 - Setup SignedAgreementForms to be blank for instances where users have submitted them via other means
  DBMISVC-99 - Fixed bug in setting reply-to address
  DBMISVC-99 - Fixed comment
  DBMISVC-99 - Set email addresses as dynamic properties; set optionally different FROM and REPLY-TO addresses
  DBMISVC-99 - Setup to enable SES
  DBMISVC-HOTFIX-072723 - Made database name dynamic
  fix(requirements): Updated Python requirements
  HYP-299 - Requirements update; added message to N2C2 NLP research purpose
  fix(requirements): Updated Python requirements
  HYPATIO-HOTFIX-053023 - Fixed navigation groups shown for no visible DataProjects
  HYPATIO-HOTFIX-053023 - Updated base Docker image
  fix(requirements): Updated Python requirements
  HYP-296 - Fixed element ID bug
  HYP-296 - Added HTML field for instructions on ChallengeTask
  HYP-HOTFIX-050823 - Fixed case-sensitivity bug for S3 keys
  HYP-288 - Fixed bug when searching pending participant table
  DBMISVC-HOTFIX-050423 - Reworked migration due to Django issue
  fix(requirements): Updated Python requirements
  fix(requirements): Updated Python requirements
  ...

Dockerfile

@@ -1,4 +1,4 @@
-FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 AS builder
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.1 AS builder
 
 # Install requirements
 RUN apt-get update \
@@ -9,6 +9,7 @@ RUN apt-get update \
         gcc \
         default-libmysqlclient-dev \
         libssl-dev \
+        pkg-config \
     && rm -rf /var/lib/apt/lists/*
 
 # Add requirements
@@ -19,7 +20,7 @@ RUN pip install -U wheel \
     && pip wheel -r /requirements.txt \
         --wheel-dir=/root/wheels
 
-FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0
+FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.1
 
 # Copy Python wheels from builder
 COPY --from=builder /root/wheels /root/wheels

app/contact/views.py

@@ -46,10 +46,10 @@ def contact_form(request, project_key=None):
                 if project.project_supervisors != '' and project.project_supervisors is not None:
                     recipients = project.project_supervisors.split(',')
                 else:
-                    recipients = settings.CONTACT_FORM_RECIPIENTS.split(',')
+                    recipients = settings.CONTACT_FORM_RECIPIENTS
 
             except ObjectDoesNotExist:
-                recipients = settings.CONTACT_FORM_RECIPIENTS.split(',')
+                recipients = settings.CONTACT_FORM_RECIPIENTS
 
             # Send it out.
             success = email_send(subject='DBMI Portal - Contact Inquiry Received',
@@ -116,7 +116,8 @@ def email_send(subject=None, recipients=None, email_template=None, extra=None):
     try:
         msg = EmailMultiAlternatives(subject=subject,
                                      body=msg_plain,
-                                     from_email=settings.DEFAULT_FROM_EMAIL,
+                                     from_email=settings.EMAIL_FROM_ADDRESS,
+                                     reply_to=(settings.EMAIL_REPLY_TO_ADDRESS, ),
                                      to=recipients)
         msg.attach_alternative(msg_html, "text/html")
         msg.send()

app/hypatio/file_services.py

@@ -5,6 +5,8 @@
 from botocore.client import Config
 from django.conf import settings
 
+from projects.models import Bucket
+
 import logging
 logger = logging.getLogger(__name__)
 
@@ -262,41 +264,60 @@ def _s3_client():
     return boto3.client('s3', config=Config(signature_version='s3v4'))
 
 
-def get_download_url(file_name, expires_in=3600):
+def get_download_url(file_uri, expires_in=3600):
     """
     Returns an S3 URL for project related files not tracked by fileservice.
     """
 
-    logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_name))
+    logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_uri))
+
+    # Separate URI
+    provider, bucket, key = Bucket.split_uri(file_uri)
+
+    # Check provider
+    match provider:
+        case Bucket.Provider.S3:
+
+            # Generate the URL to get the file object
+            url = _s3_client().generate_presigned_url(
+                ClientMethod='get_object',
+                Params={
+                    'Bucket': bucket,
+                    'Key': key
+                },
+                ExpiresIn=expires_in
+            )
 
-    # Generate the URL to get the file object
-    url = _s3_client().generate_presigned_url(
-        ClientMethod='get_object',
-        Params={
-            'Bucket': settings.S3_BUCKET,
-            'Key': file_name
-        },
-        ExpiresIn=expires_in
-    )
+        case _:
+            raise NotImplementedError(f"Could not generate download URL for URI: {file_uri}")
 
-    logger.debug('[file_services][get_download_url] Generated URL: {}'.format(url))
+    logger.debug(f'[file_services][get_download_url] Generated URL: {url}')
 
     return url
 
 
-def upload_file(file_name, file, expires_in=3600):
+def upload_file(file, file_uri, expires_in=3600):
     """
     Enables uploading of files directly to the Hypatio S3 bucket without fileservice tracking.
     """
+    logger.debug('[file_services][upload_file] Uploading file to: {}'.format(file_uri))
 
-    logger.error('[file_services][upload_file] Uploading file: {}'.format(file_name))
+    # Separate URI
+    provider, bucket, key = Bucket.split_uri(file_uri)
 
-    # Generate the POST attributes
-    post = _s3_client().generate_presigned_post(
-        Bucket=settings.S3_BUCKET,
-        Key=file_name,
-        ExpiresIn=expires_in
-    )
+    # Check provider
+    match provider:
+        case Bucket.Provider.S3:
+
+            # Generate the POST attributes
+            post = _s3_client().generate_presigned_post(
+                Bucket=bucket,
+                Key=key,
+                ExpiresIn=expires_in
+            )
+
+        case _:
+            raise NotImplementedError(f"Could not generate upload for URI: {file_uri}")
 
     # Perform the request to upload the file
     files = {"file": file}
@@ -315,34 +336,46 @@ def upload_file(file_name, file, expires_in=3600):
         logger.exception(e)
 
 
-def host_file(request, file_uuid, file_location, file_name):
+def host_file(request, file_uuid, file_uri):
     """
     Copies a file from the Fileservice bucket to the Hypatio hosted files bucket
     """
     try:
-        # Make the request.
+        # Get the original file from Fileservice
         file = get(request, '/api/file/{}/'.format(file_uuid))
         logger.debug(file)
 
-        # Perform the request to copy the file
-        source = {'Bucket': settings.FILESERVICE_AWS_BUCKET, 'Key': file['locations'][0]['url'].split('/', 3)[3]}
-        key = os.path.join(file_location, file_name)
+        # Separate destination URI
+        provider, bucket, key = Bucket.split_uri(file_uri)
+
+        # Check provider
+        match provider:
+            case Bucket.Provider.S3:
+
+                # Perform the request to copy the file
+                _, origin_bucket, origin_key = Bucket.split_uri(file['locations'][0]['url'])
+                source = {'Bucket': origin_bucket, 'Key': origin_key}
+                logger.debug(
+                    f'Fileservice: Copying s3://{origin_bucket}/{origin_key}'
+                    f' to {provider.value}://{bucket}/{key}'
+                )
 
-        logger.debug(f'Fileservice: Copying {source["Bucket"]}/{source["Key"]}'
-                     f' to {settings.S3_BUCKET}/{key}')
+                # Generate the URL to get the file object
+                _s3_client().copy_object(
+                    CopySource=source,
+                    Bucket=bucket,
+                    Key=key,
+                )
 
-        # Generate the URL to get the file object
-        _s3_client().copy_object(
-            CopySource=source,
-            Bucket=settings.S3_BUCKET,
-            Key=key,
-        )
+                return True
 
-        return True
+            case _:
+                raise NotImplementedError(f"Could not generate upload for URI: {file_uri}")
 
     except Exception as e:
         logger.exception('[file_services][host_file] Error: {}'.format(e), exc_info=True, extra={
-            'request': request, 'file_uuid': file_uuid, 'file_location': file_location, 'file_name': file_name
+            'request': request, 'bucket': bucket, 'file_uuid': file_uuid,
+            'file_uri': file_uri,
         })
 
     return False

app/hypatio/settings.py

@@ -57,9 +57,13 @@
     'storages',
     'django_jsonfield_backport',
     'django_q',
+<<<<<<< HEAD
     'rest_framework',
     'drf_spectacular',
     'django_filters',
+=======
+    'django_ses',
+>>>>>>> development
 ]
 
 MIDDLEWARE = [
@@ -89,6 +93,7 @@
                 'django.template.context_processors.request',
                 'django.contrib.auth.context_processors.auth',
                 'django.contrib.messages.context_processors.messages',
+                'hypatio.views.navigation_context',
             ],
         },
     },
@@ -103,7 +108,7 @@
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
-        'NAME': 'hypatio',
+        'NAME': environment.get_str("MYSQL_DATABASE", default='hypatio'),
         'USER': environment.get_str("MYSQL_USERNAME", default='hypatio'),
         'PASSWORD': environment.get_str("MYSQL_PASSWORD", required=True),
         'HOST': environment.get_str("MYSQL_HOST", required=True),
@@ -136,8 +141,8 @@
 SSL_SETTING = "https"
 VERIFY_REQUESTS = True
 
-CONTACT_FORM_RECIPIENTS="dbmi_tech_core@hms.harvard.edu"
-DEFAULT_FROM_EMAIL="dbmi_tech_core@hms.harvard.edu"
+# Pass a list of email addresses
+CONTACT_FORM_RECIPIENTS = environment.get_list('CONTACT_FORM_RECIPIENTS', required=True)
 
 RECAPTCHA_KEY = environment.get_str('RECAPTCHA_KEY', required=True)
 RECAPTCHA_CLIENT_ID = environment.get_str('RECAPTCHA_CLIENT_ID', required=True)
@@ -147,6 +152,7 @@
 S3_BUCKET = environment.get_str('S3_BUCKET', required=True)
 
 DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+AWS_S3_SIGNATURE_VERSION = 's3v4'
 AWS_STORAGE_BUCKET_NAME = environment.get_str('S3_BUCKET', required=True)
 AWS_LOCATION = 'upload'
 
@@ -239,12 +245,31 @@
 # Email Configurations
 #####################################################################################
 
-EMAIL_BACKEND = environment.get_str("EMAIL_BACKEND", "django_smtp_ssl.SSLEmailBackend")
-EMAIL_USE_SSL = EMAIL_BACKEND == 'django_smtp_ssl.SSLEmailBackend'
-EMAIL_HOST = environment.get_str("EMAIL_HOST", required=True)
-EMAIL_HOST_USER = environment.get_str("EMAIL_HOST_USER", required=not DEBUG)
-EMAIL_HOST_PASSWORD = environment.get_str("EMAIL_HOST_PASSWORD", required=EMAIL_HOST_USER is not None)
-EMAIL_PORT = environment.get_str("EMAIL_PORT", required=True)
+# Determine email backend
+EMAIL_BACKEND = environment.get_str("EMAIL_BACKEND", required=True)
+if EMAIL_BACKEND == "django.core.mail.backends.smtp.EmailBackend":
+
+    # SMTP Email configuration
+    EMAIL_USE_SSL = environment.get_bool("EMAIL_USE_SSL", default=True)
+    EMAIL_HOST = environment.get_str("EMAIL_HOST", required=True)
+    EMAIL_HOST_USER = environment.get_str("EMAIL_HOST_USER", required=False)
+    EMAIL_HOST_PASSWORD = environment.get_str("EMAIL_HOST_PASSWORD", required=False)
+    EMAIL_PORT = environment.get_str("EMAIL_PORT", required=True)
+
+elif EMAIL_BACKEND == "django_ses.SESBackend":
+
+    # AWS SES Email configuration
+    AWS_SES_SOURCE_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True)
+    AWS_SES_FROM_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True)
+    AWS_SES_RETURN_PATH_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True)
+    USE_SES_V2 = True
+
+else:
+    raise SystemError(f"Email backend '{EMAIL_BACKEND}' is not supported for this application")
+
+# Set default from address
+EMAIL_FROM_ADDRESS = environment.get_str("EMAIL_FROM_ADDRESS", required=True)
+EMAIL_REPLY_TO_ADDRESS = environment.get_str("EMAIL_REPLY_TO_ADDRESS", default=EMAIL_FROM_ADDRESS)
 
 #####################################################################################
 

app/hypatio/urls.py

@@ -6,6 +6,7 @@
 from projects.views import list_data_projects
 from projects.views import list_data_challenges
 from projects.views import list_software_projects
+from projects.views import GroupView
 
 
 urlpatterns = [
@@ -19,5 +20,6 @@
     re_path(r'^data-challenges/$', list_data_challenges, name='data-challenges'),
     re_path(r'^software-projects/$', list_software_projects, name='software-projects'),
     re_path(r'^healthcheck/?', include('health_check.urls')),
+    re_path(r'^groups/(?P<group_key>[^/]+)/?', GroupView.as_view(), name="group"),
     re_path(r'^', index, name='index'),
 ]

app/hypatio/views.py

@@ -1,7 +1,9 @@
+import os
 from django.shortcuts import render
+from django.utils.functional import SimpleLazyObject
 
 from hypatio.auth0authenticate import public_user_auth_and_jwt
-
+from projects.models import Group, DataProject
 
 @public_user_auth_and_jwt
 def index(request, template_name='index.html'):
@@ -12,3 +14,35 @@ def index(request, template_name='index.html'):
     context = {}
 
     return render(request, template_name, context=context)
+
+def navigation_context(request):
+    """
+    Includes global navigation context in all requests.
+
+    This method is enabled by including it in settings.TEMPLATES as
+    a context processor.
+
+    :param request: The current HttpRequest
+    :type request: HttpRequest
+    :return: The context that should be included in the response's context
+    :rtype: dict
+    """
+    def group_context():
+
+        # Check for an active project and determine its group
+        groups = Group.objects.filter(dataproject__isnull=False, dataproject__visible=True).distinct()
+        active_group = None
+        project = DataProject.objects.filter(project_key=os.path.basename(os.path.normpath(request.path))).first()
+        if project:
+
+            # Check for group
+            active_group = next((g for g in groups if project in g.dataproject_set.all()), None)
+
+        return {
+            "groups": groups,
+            "active_group": active_group,
+        }
+
+    return {
+        "navigation": SimpleLazyObject(group_context)
+    }

app/manage/api.py

@@ -1018,14 +1018,18 @@ def host_submission(request, fileservice_uuid):
             return HttpResponse(host_submission_form.errors.as_json(), status=400)
 
         try:
+            # Build the destination URI
+            file_uri = f"{project.bucket.uri}/" \
+                       f"{host_submission_form.cleaned_data['file_location']}/" \
+                       f"{host_submission_form.cleaned_data['file_name']}"
+
             # Do the copy
-            logger.debug(f'[HYPATIO][DEBUG][host_submission] Copying submission "{submission}" '
-                         f'to hosted location "{host_submission_form.cleaned_data["file_location"]}/'
-                         f'{host_submission_form.cleaned_data["file_name"]}"')
+            logger.debug(
+                f'[HYPATIO][DEBUG][host_submission] Copying submission '
+                f'"{submission}" to hosted location "{file_uri}"'
+            )
 
-            if host_file(request=request, file_uuid=fileservice_uuid,
-                         file_location=host_submission_form.cleaned_data['file_location'],
-                         file_name=host_submission_form.cleaned_data['file_name']):
+            if host_file(request=request, file_uuid=fileservice_uuid, file_uri=file_uri):
                 logger.debug(f'[HYPATIO][DEBUG][host_submission] File was copied successfully')
                 host_submission_form.save()
 

app/manage/forms.py

@@ -79,3 +79,7 @@ def __init__(self, *args, **kwargs):
             # Limit agreement form choices to those related to the passed project
             if project_key:
                 self.fields['agreement_form'].queryset = DataProject.objects.get(project_key=project_key).agreement_forms.all()
+
+
+class UploadSignedAgreementFormFileForm(forms.Form):
+    file = forms.FileField(label="Signed Agreement Form PDF", required=True)

app/manage/urls.py

@@ -8,6 +8,7 @@
 from manage.views import ProjectPendingParticipants
 from manage.views import team_notification
 from manage.views import UploadSignedAgreementFormView
+from manage.views import UploadSignedAgreementFormFileView
 
 from manage.api import set_dataproject_details
 from manage.api import set_dataproject_registration_status
@@ -62,6 +63,7 @@
     re_path(r'^get-project-participants/(?P<project_key>[^/]+)/$', ProjectParticipants.as_view(), name='get-project-participants'),
     re_path(r'^get-project-pending-participants/(?P<project_key>[^/]+)/$', ProjectPendingParticipants.as_view(), name='get-project-pending-participants'),
     re_path(r'^upload-signed-agreement-form/(?P<project_key>[^/]+)/(?P<user_email>[^/]+)/$', UploadSignedAgreementFormView.as_view(), name='upload-signed-agreement-form'),
+    re_path(r'^upload-signed-agreement-form-file/(?P<signed_agreement_form_id>[^/]+)/$', UploadSignedAgreementFormFileView.as_view(), name='upload-signed-agreement-form-file'),
     re_path(r'^(?P<project_key>[^/]+)/$', DataProjectManageView.as_view(), name='manage-project'),
     re_path(r'^(?P<project_key>[^/]+)/(?P<team_leader>[^/]+)/$', manage_team, name='manage-team'),
 ]