diff --git a/Dockerfile b/Dockerfile index ec9dfa65..c31b3081 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 AS builder +FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.1 AS builder # Install requirements RUN apt-get update \ @@ -9,6 +9,7 @@ RUN apt-get update \ gcc \ default-libmysqlclient-dev \ libssl-dev \ + pkg-config \ && rm -rf /var/lib/apt/lists/* # Add requirements @@ -19,7 +20,7 @@ RUN pip install -U wheel \ && pip wheel -r /requirements.txt \ --wheel-dir=/root/wheels -FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 +FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.1 # Copy Python wheels from builder COPY --from=builder /root/wheels /root/wheels diff --git a/app/contact/views.py b/app/contact/views.py index d36cd60f..fe208965 100644 --- a/app/contact/views.py +++ b/app/contact/views.py @@ -46,10 +46,10 @@ def contact_form(request, project_key=None): if project.project_supervisors != '' and project.project_supervisors is not None: recipients = project.project_supervisors.split(',') else: - recipients = settings.CONTACT_FORM_RECIPIENTS.split(',') + recipients = settings.CONTACT_FORM_RECIPIENTS except ObjectDoesNotExist: - recipients = settings.CONTACT_FORM_RECIPIENTS.split(',') + recipients = settings.CONTACT_FORM_RECIPIENTS # Send it out. success = email_send(subject='DBMI Portal - Contact Inquiry Received', @@ -116,7 +116,8 @@ def email_send(subject=None, recipients=None, email_template=None, extra=None): try: msg = EmailMultiAlternatives(subject=subject, body=msg_plain, - from_email=settings.DEFAULT_FROM_EMAIL, + from_email=settings.EMAIL_FROM_ADDRESS, + reply_to=(settings.EMAIL_REPLY_TO_ADDRESS, ), to=recipients) msg.attach_alternative(msg_html, "text/html") msg.send() diff --git a/app/hypatio/file_services.py b/app/hypatio/file_services.py index e7347288..70695e2a 100644 --- a/app/hypatio/file_services.py +++ b/app/hypatio/file_services.py @@ -5,6 +5,8 @@ from botocore.client import Config from django.conf import settings +from projects.models import Bucket + import logging logger = logging.getLogger(__name__) @@ -262,41 +264,60 @@ def _s3_client(): return boto3.client('s3', config=Config(signature_version='s3v4')) -def get_download_url(file_name, expires_in=3600): +def get_download_url(file_uri, expires_in=3600): """ Returns an S3 URL for project related files not tracked by fileservice. """ - logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_name)) + logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_uri)) + + # Separate URI + provider, bucket, key = Bucket.split_uri(file_uri) + + # Check provider + match provider: + case Bucket.Provider.S3: + + # Generate the URL to get the file object + url = _s3_client().generate_presigned_url( + ClientMethod='get_object', + Params={ + 'Bucket': bucket, + 'Key': key + }, + ExpiresIn=expires_in + ) - # Generate the URL to get the file object - url = _s3_client().generate_presigned_url( - ClientMethod='get_object', - Params={ - 'Bucket': settings.S3_BUCKET, - 'Key': file_name - }, - ExpiresIn=expires_in - ) + case _: + raise NotImplementedError(f"Could not generate download URL for URI: {file_uri}") - logger.debug('[file_services][get_download_url] Generated URL: {}'.format(url)) + logger.debug(f'[file_services][get_download_url] Generated URL: {url}') return url -def upload_file(file_name, file, expires_in=3600): +def upload_file(file, file_uri, expires_in=3600): """ Enables uploading of files directly to the Hypatio S3 bucket without fileservice tracking. """ + logger.debug('[file_services][upload_file] Uploading file to: {}'.format(file_uri)) - logger.error('[file_services][upload_file] Uploading file: {}'.format(file_name)) + # Separate URI + provider, bucket, key = Bucket.split_uri(file_uri) - # Generate the POST attributes - post = _s3_client().generate_presigned_post( - Bucket=settings.S3_BUCKET, - Key=file_name, - ExpiresIn=expires_in - ) + # Check provider + match provider: + case Bucket.Provider.S3: + + # Generate the POST attributes + post = _s3_client().generate_presigned_post( + Bucket=bucket, + Key=key, + ExpiresIn=expires_in + ) + + case _: + raise NotImplementedError(f"Could not generate upload for URI: {file_uri}") # Perform the request to upload the file files = {"file": file} @@ -315,34 +336,46 @@ def upload_file(file_name, file, expires_in=3600): logger.exception(e) -def host_file(request, file_uuid, file_location, file_name): +def host_file(request, file_uuid, file_uri): """ Copies a file from the Fileservice bucket to the Hypatio hosted files bucket """ try: - # Make the request. + # Get the original file from Fileservice file = get(request, '/api/file/{}/'.format(file_uuid)) logger.debug(file) - # Perform the request to copy the file - source = {'Bucket': settings.FILESERVICE_AWS_BUCKET, 'Key': file['locations'][0]['url'].split('/', 3)[3]} - key = os.path.join(file_location, file_name) + # Separate destination URI + provider, bucket, key = Bucket.split_uri(file_uri) + + # Check provider + match provider: + case Bucket.Provider.S3: + + # Perform the request to copy the file + _, origin_bucket, origin_key = Bucket.split_uri(file['locations'][0]['url']) + source = {'Bucket': origin_bucket, 'Key': origin_key} + logger.debug( + f'Fileservice: Copying s3://{origin_bucket}/{origin_key}' + f' to {provider.value}://{bucket}/{key}' + ) - logger.debug(f'Fileservice: Copying {source["Bucket"]}/{source["Key"]}' - f' to {settings.S3_BUCKET}/{key}') + # Generate the URL to get the file object + _s3_client().copy_object( + CopySource=source, + Bucket=bucket, + Key=key, + ) - # Generate the URL to get the file object - _s3_client().copy_object( - CopySource=source, - Bucket=settings.S3_BUCKET, - Key=key, - ) + return True - return True + case _: + raise NotImplementedError(f"Could not generate upload for URI: {file_uri}") except Exception as e: logger.exception('[file_services][host_file] Error: {}'.format(e), exc_info=True, extra={ - 'request': request, 'file_uuid': file_uuid, 'file_location': file_location, 'file_name': file_name + 'request': request, 'bucket': bucket, 'file_uuid': file_uuid, + 'file_uri': file_uri, }) return False diff --git a/app/hypatio/settings.py b/app/hypatio/settings.py index 04c5ed8a..55535c4b 100644 --- a/app/hypatio/settings.py +++ b/app/hypatio/settings.py @@ -57,9 +57,13 @@ 'storages', 'django_jsonfield_backport', 'django_q', +<<<<<<< HEAD 'rest_framework', 'drf_spectacular', 'django_filters', +======= + 'django_ses', +>>>>>>> development ] MIDDLEWARE = [ @@ -89,6 +93,7 @@ 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', + 'hypatio.views.navigation_context', ], }, }, @@ -103,7 +108,7 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.mysql', - 'NAME': 'hypatio', + 'NAME': environment.get_str("MYSQL_DATABASE", default='hypatio'), 'USER': environment.get_str("MYSQL_USERNAME", default='hypatio'), 'PASSWORD': environment.get_str("MYSQL_PASSWORD", required=True), 'HOST': environment.get_str("MYSQL_HOST", required=True), @@ -136,8 +141,8 @@ SSL_SETTING = "https" VERIFY_REQUESTS = True -CONTACT_FORM_RECIPIENTS="dbmi_tech_core@hms.harvard.edu" -DEFAULT_FROM_EMAIL="dbmi_tech_core@hms.harvard.edu" +# Pass a list of email addresses +CONTACT_FORM_RECIPIENTS = environment.get_list('CONTACT_FORM_RECIPIENTS', required=True) RECAPTCHA_KEY = environment.get_str('RECAPTCHA_KEY', required=True) RECAPTCHA_CLIENT_ID = environment.get_str('RECAPTCHA_CLIENT_ID', required=True) @@ -147,6 +152,7 @@ S3_BUCKET = environment.get_str('S3_BUCKET', required=True) DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +AWS_S3_SIGNATURE_VERSION = 's3v4' AWS_STORAGE_BUCKET_NAME = environment.get_str('S3_BUCKET', required=True) AWS_LOCATION = 'upload' @@ -239,12 +245,31 @@ # Email Configurations ##################################################################################### -EMAIL_BACKEND = environment.get_str("EMAIL_BACKEND", "django_smtp_ssl.SSLEmailBackend") -EMAIL_USE_SSL = EMAIL_BACKEND == 'django_smtp_ssl.SSLEmailBackend' -EMAIL_HOST = environment.get_str("EMAIL_HOST", required=True) -EMAIL_HOST_USER = environment.get_str("EMAIL_HOST_USER", required=not DEBUG) -EMAIL_HOST_PASSWORD = environment.get_str("EMAIL_HOST_PASSWORD", required=EMAIL_HOST_USER is not None) -EMAIL_PORT = environment.get_str("EMAIL_PORT", required=True) +# Determine email backend +EMAIL_BACKEND = environment.get_str("EMAIL_BACKEND", required=True) +if EMAIL_BACKEND == "django.core.mail.backends.smtp.EmailBackend": + + # SMTP Email configuration + EMAIL_USE_SSL = environment.get_bool("EMAIL_USE_SSL", default=True) + EMAIL_HOST = environment.get_str("EMAIL_HOST", required=True) + EMAIL_HOST_USER = environment.get_str("EMAIL_HOST_USER", required=False) + EMAIL_HOST_PASSWORD = environment.get_str("EMAIL_HOST_PASSWORD", required=False) + EMAIL_PORT = environment.get_str("EMAIL_PORT", required=True) + +elif EMAIL_BACKEND == "django_ses.SESBackend": + + # AWS SES Email configuration + AWS_SES_SOURCE_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True) + AWS_SES_FROM_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True) + AWS_SES_RETURN_PATH_ARN = environment.get_str("DBMI_SES_IDENTITY", required=True) + USE_SES_V2 = True + +else: + raise SystemError(f"Email backend '{EMAIL_BACKEND}' is not supported for this application") + +# Set default from address +EMAIL_FROM_ADDRESS = environment.get_str("EMAIL_FROM_ADDRESS", required=True) +EMAIL_REPLY_TO_ADDRESS = environment.get_str("EMAIL_REPLY_TO_ADDRESS", default=EMAIL_FROM_ADDRESS) ##################################################################################### diff --git a/app/hypatio/urls.py b/app/hypatio/urls.py index f97fb4cc..bf4844df 100644 --- a/app/hypatio/urls.py +++ b/app/hypatio/urls.py @@ -6,6 +6,7 @@ from projects.views import list_data_projects from projects.views import list_data_challenges from projects.views import list_software_projects +from projects.views import GroupView urlpatterns = [ @@ -19,5 +20,6 @@ re_path(r'^data-challenges/$', list_data_challenges, name='data-challenges'), re_path(r'^software-projects/$', list_software_projects, name='software-projects'), re_path(r'^healthcheck/?', include('health_check.urls')), + re_path(r'^groups/(?P[^/]+)/?', GroupView.as_view(), name="group"), re_path(r'^', index, name='index'), ] diff --git a/app/hypatio/views.py b/app/hypatio/views.py index 6da9267e..bd1c0295 100644 --- a/app/hypatio/views.py +++ b/app/hypatio/views.py @@ -1,7 +1,9 @@ +import os from django.shortcuts import render +from django.utils.functional import SimpleLazyObject from hypatio.auth0authenticate import public_user_auth_and_jwt - +from projects.models import Group, DataProject @public_user_auth_and_jwt def index(request, template_name='index.html'): @@ -12,3 +14,35 @@ def index(request, template_name='index.html'): context = {} return render(request, template_name, context=context) + +def navigation_context(request): + """ + Includes global navigation context in all requests. + + This method is enabled by including it in settings.TEMPLATES as + a context processor. + + :param request: The current HttpRequest + :type request: HttpRequest + :return: The context that should be included in the response's context + :rtype: dict + """ + def group_context(): + + # Check for an active project and determine its group + groups = Group.objects.filter(dataproject__isnull=False, dataproject__visible=True).distinct() + active_group = None + project = DataProject.objects.filter(project_key=os.path.basename(os.path.normpath(request.path))).first() + if project: + + # Check for group + active_group = next((g for g in groups if project in g.dataproject_set.all()), None) + + return { + "groups": groups, + "active_group": active_group, + } + + return { + "navigation": SimpleLazyObject(group_context) + } diff --git a/app/manage/api.py b/app/manage/api.py index 9c2e7277..bdbd7ef8 100644 --- a/app/manage/api.py +++ b/app/manage/api.py @@ -1018,14 +1018,18 @@ def host_submission(request, fileservice_uuid): return HttpResponse(host_submission_form.errors.as_json(), status=400) try: + # Build the destination URI + file_uri = f"{project.bucket.uri}/" \ + f"{host_submission_form.cleaned_data['file_location']}/" \ + f"{host_submission_form.cleaned_data['file_name']}" + # Do the copy - logger.debug(f'[HYPATIO][DEBUG][host_submission] Copying submission "{submission}" ' - f'to hosted location "{host_submission_form.cleaned_data["file_location"]}/' - f'{host_submission_form.cleaned_data["file_name"]}"') + logger.debug( + f'[HYPATIO][DEBUG][host_submission] Copying submission ' + f'"{submission}" to hosted location "{file_uri}"' + ) - if host_file(request=request, file_uuid=fileservice_uuid, - file_location=host_submission_form.cleaned_data['file_location'], - file_name=host_submission_form.cleaned_data['file_name']): + if host_file(request=request, file_uuid=fileservice_uuid, file_uri=file_uri): logger.debug(f'[HYPATIO][DEBUG][host_submission] File was copied successfully') host_submission_form.save() diff --git a/app/manage/forms.py b/app/manage/forms.py index 93464f6b..809415c4 100644 --- a/app/manage/forms.py +++ b/app/manage/forms.py @@ -79,3 +79,7 @@ def __init__(self, *args, **kwargs): # Limit agreement form choices to those related to the passed project if project_key: self.fields['agreement_form'].queryset = DataProject.objects.get(project_key=project_key).agreement_forms.all() + + +class UploadSignedAgreementFormFileForm(forms.Form): + file = forms.FileField(label="Signed Agreement Form PDF", required=True) diff --git a/app/manage/urls.py b/app/manage/urls.py index 8821522c..82b579ca 100644 --- a/app/manage/urls.py +++ b/app/manage/urls.py @@ -8,6 +8,7 @@ from manage.views import ProjectPendingParticipants from manage.views import team_notification from manage.views import UploadSignedAgreementFormView +from manage.views import UploadSignedAgreementFormFileView from manage.api import set_dataproject_details from manage.api import set_dataproject_registration_status @@ -62,6 +63,7 @@ re_path(r'^get-project-participants/(?P[^/]+)/$', ProjectParticipants.as_view(), name='get-project-participants'), re_path(r'^get-project-pending-participants/(?P[^/]+)/$', ProjectPendingParticipants.as_view(), name='get-project-pending-participants'), re_path(r'^upload-signed-agreement-form/(?P[^/]+)/(?P[^/]+)/$', UploadSignedAgreementFormView.as_view(), name='upload-signed-agreement-form'), + re_path(r'^upload-signed-agreement-form-file/(?P[^/]+)/$', UploadSignedAgreementFormFileView.as_view(), name='upload-signed-agreement-form-file'), re_path(r'^(?P[^/]+)/$', DataProjectManageView.as_view(), name='manage-project'), re_path(r'^(?P[^/]+)/(?P[^/]+)/$', manage_team, name='manage-team'), ] diff --git a/app/manage/views.py b/app/manage/views.py index 38ebceaa..c531ae1a 100644 --- a/app/manage/views.py +++ b/app/manage/views.py @@ -19,6 +19,7 @@ from django.core.mail import EmailMultiAlternatives from django.template.loader import render_to_string from dbmi_client import fileservice +from django.shortcuts import get_object_or_404 from hypatio.sciauthz_services import SciAuthZ from hypatio.scireg_services import get_user_profile, get_distinct_countries_participating @@ -26,6 +27,7 @@ from manage.forms import NotificationForm from manage.models import ChallengeTaskSubmissionExport from manage.forms import UploadSignedAgreementFormForm +from manage.forms import UploadSignedAgreementFormFileForm from projects.models import AgreementForm, ChallengeTaskSubmission from projects.models import DataProject from projects.models import Participant @@ -469,6 +471,11 @@ def get(self, request, project_key, *args, **kwargs): ) ) + # Add search if necessary + if search: + participants_waiting_access = participants_waiting_access.filter(user__email__icontains=search) + participants_awaiting_approval = participants_awaiting_approval.filter(user__email__icontains=search) + # We only want distinct Participants belonging to the users query # Django won't sort on a related field after this union so we annotate each queryset with the user's email to sort on query_set = participants_waiting_access.annotate(email=F("user__email")) \ @@ -477,8 +484,8 @@ def get(self, request, project_key, *args, **kwargs): # Setup paginator paginator = Paginator( - query_set.filter(user__email__icontains=search) if search else query_set, - length + query_set, + length, ) # Determine page index (1-index) from DT parameters @@ -598,7 +605,8 @@ def team_notification(request, project_key=None): try: msg = EmailMultiAlternatives(subject=subject, body=msg_plain, - from_email=settings.DEFAULT_FROM_EMAIL, + from_email=settings.EMAIL_FROM_ADDRESS, + reply_to=(settings.EMAIL_REPLY_TO_ADDRESS, ), to=[team.team_leader.email]) msg.attach_alternative(msg_html, "text/html") msg.send() @@ -903,3 +911,87 @@ def post(self, request, project_key, user_email, *args, **kwargs): response['X-IC-Script'] += "$('#page-modal').modal('hide');" return response + + +@method_decorator([user_auth_and_jwt], name='dispatch') +class UploadSignedAgreementFormFileView(View): + """ + View to upload signed agreement form files for participants. + + * Requires token authentication. + * Only admin users are able to access this view. + """ + def get(self, request, signed_agreement_form_id, *args, **kwargs): + """ + Return the upload form template + """ + user = request.user + user_jwt = request.COOKIES.get("DBMI_JWT", None) + + signed_agreement_form = get_object_or_404(SignedAgreementForm, id=signed_agreement_form_id) + + sciauthz = SciAuthZ(user_jwt, user.email) + is_manager = sciauthz.user_has_manage_permission(signed_agreement_form.project.project_key) + + if not is_manager: + logger.debug('User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=signed_agreement_form.project.project_key + )) + return HttpResponse(403) + + # Return file upload form + form = UploadSignedAgreementFormFileForm() + + # Set context + context = { + "form": form, + "signed_agreement_form_id": signed_agreement_form_id, + } + + # Render html + return render(request, "manage/upload-signed-agreement-form-file.html", context) + + def post(self, request, signed_agreement_form_id, *args, **kwargs): + """ + Process the form + """ + user = request.user + user_jwt = request.COOKIES.get("DBMI_JWT", None) + + signed_agreement_form = get_object_or_404(SignedAgreementForm, id=signed_agreement_form_id) + + sciauthz = SciAuthZ(user_jwt, user.email) + is_manager = sciauthz.user_has_manage_permission(signed_agreement_form.project.project_key) + + if not is_manager: + logger.debug('User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=signed_agreement_form.project.project_key + )) + return HttpResponse(403) + + # Assembles the form and run validation. + form = UploadSignedAgreementFormFileForm(data=request.POST, files=request.FILES) + if not form.is_valid(): + logger.warning('Form failed: {}'.format(form.errors.as_json())) + return HttpResponse(status=400) + + logger.debug(f"[upload_signed_agreement_form_file] Data -> {form.cleaned_data}") + + # Set the file and save + signed_agreement_form.upload = form.cleaned_data['file'] + signed_agreement_form.save() + + # Create the response. + response = HttpResponse(status=201) + + # Setup the script run. + response['X-IC-Script'] = "notify('{}', '{}', 'glyphicon glyphicon-{}');".format( + "success", "Signed agreement form file successfully uploaded", "thumbs-up" + ) + + # Close the modal + response['X-IC-Script'] += "$('#page-modal').modal('hide');" + + return response diff --git a/app/projects/admin.py b/app/projects/admin.py index 42a66ef4..cea9fcf3 100644 --- a/app/projects/admin.py +++ b/app/projects/admin.py @@ -2,6 +2,7 @@ from django.urls import reverse from django.utils.html import escape, mark_safe +from projects.models import Group from projects.models import DataProject from projects.models import AgreementForm from projects.models import SignedAgreementForm @@ -14,12 +15,18 @@ from projects.models import ChallengeTask from projects.models import ChallengeTaskSubmission from projects.models import ChallengeTaskSubmissionDownload -from projects.models import NLPDUASignedAgreementFormFields -from projects.models import NLPWHYSignedAgreementFormFields -from projects.models import DUASignedAgreementFormFields -from projects.models import ROCSignedAgreementFormFields -from projects.models import MAYOSignedAgreementFormFields -from projects.models import MIMIC3SignedAgreementFormFields +from projects.models import Bucket + + +class GroupAdmin(admin.ModelAdmin): + list_display = ('title', 'key', 'created', 'modified', ) + readonly_fields = ('created', 'modified', ) + + +class BucketAdmin(admin.ModelAdmin): + list_display = ('name', 'provider', 'created', 'modified', ) + readonly_fields = ('created', 'modified', ) + class DataProjectAdmin(admin.ModelAdmin): list_display = ('name', 'project_key', 'informational_only', 'registration_open', 'requires_authorization', 'is_challenge', 'order', 'created', 'modified', ) @@ -80,66 +87,8 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin): search_fields = ('user__email', ) -class SignedAgreementFormFieldsAdmin(admin.ModelAdmin): - def get_user(self, obj): - return obj.signed_agreement_form.user.email - get_user.short_description = 'User' - get_user.admin_order_field = 'signed_agreement_form__user__email' - - def get_status(self, obj): - return obj.signed_agreement_form.status - get_status.short_description = 'Status' - get_status.admin_order_field = 'signed_agreement_form__status' - - def signed_agreement_form_link(self, obj): - link = reverse("admin:projects_signedagreementform_change", args=[obj.signed_agreement_form.id]) - return mark_safe(f'{escape(obj.signed_agreement_form.__str__())}') - - signed_agreement_form_link.short_description = 'Signed Agreement Form' - signed_agreement_form_link.admin_order_field = 'signed agreement form' - - list_display = ( - 'get_user', - 'get_status', - 'signed_agreement_form_link' - ) - search_fields = ( - 'signed_agreement_form__user__email', - 'signed_agreement_form__agreement_form__project', - 'signed_agreement_form__agreement_form__short_name', - 'signed_agreement_form', - ) - readonly_fields = ( - 'signed_agreement_form', - 'created', - 'modified' - ) - - -class NLPDUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class NLPWHYSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class DUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class ROCSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class MAYOSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - +admin.site.register(Group, GroupAdmin) +admin.site.register(Bucket, BucketAdmin) admin.site.register(DataProject, DataProjectAdmin) admin.site.register(AgreementForm, AgreementformAdmin) admin.site.register(SignedAgreementForm, SignedagreementformAdmin) @@ -152,11 +101,3 @@ class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): admin.site.register(ChallengeTask, ChallengeTaskAdmin) admin.site.register(ChallengeTaskSubmission, ChallengeTaskSubmissionAdmin) admin.site.register(ChallengeTaskSubmissionDownload, ChallengeTaskSubmissionDownloadAdmin) - - -admin.site.register(NLPDUASignedAgreementFormFields, NLPDUASignedAgreementFormFieldsAdmin) -admin.site.register(NLPWHYSignedAgreementFormFields, NLPWHYSignedAgreementFormFieldsAdmin) -admin.site.register(DUASignedAgreementFormFields, DUASignedAgreementFormFieldsAdmin) -admin.site.register(ROCSignedAgreementFormFields, ROCSignedAgreementFormFieldsAdmin) -admin.site.register(MAYOSignedAgreementFormFields, MAYOSignedAgreementFormFieldsAdmin) -admin.site.register(MIMIC3SignedAgreementFormFields, MIMIC3SignedAgreementFormFieldsAdmin) diff --git a/app/projects/api.py b/app/projects/api.py index 024342c6..f3f3dd11 100644 --- a/app/projects/api.py +++ b/app/projects/api.py @@ -360,10 +360,10 @@ def download_dataset(request): # Save a record of this person downloading this file. HostedFileDownload.objects.create(user=request.user, hosted_file=file_to_download) - s3_filename = file_to_download.file_location + "/" + file_to_download.file_name - logger.debug("[download_dataset] - User " + request.user.email + " is downloading file " + s3_filename + " from bucket " + settings.S3_BUCKET + ".") + file_uri = f"{file_to_download.project.bucket.uri}/{file_to_download.file_location}/{file_to_download.file_name}" + logger.debug(f"[download_dataset] - User {request.user.email} is downloading file {file_uri}.") - download_url = get_download_url(s3_filename) + download_url = get_download_url(file_uri) response = redirect(download_url) response['Content-Disposition'] = 'attachment' diff --git a/app/projects/migrations/0097_participant_created_participant_modified.py b/app/projects/migrations/0097_participant_created_participant_modified.py index e09b65ee..69502161 100644 --- a/app/projects/migrations/0097_participant_created_participant_modified.py +++ b/app/projects/migrations/0097_participant_created_participant_modified.py @@ -11,29 +11,10 @@ def migrate_participants_model(apps, schema_editor): been created. This is calculated by fetching the date of the last signed SignedAgreementForm relevant to the DataProjects. """ - for participant in Participant.objects.all(): - - # Fetch signed agreement forms - signed_agreement_form = SignedAgreementForm.objects.filter(user=participant.user, project=participant.project).last() - if signed_agreement_form: - - # Set the dates - participant.created = signed_agreement_form.date_signed - participant.modified = signed_agreement_form.date_signed - - # Do the update - Participant.objects.filter(pk=participant.pk).update( - created=signed_agreement_form.date_signed, - modified=signed_agreement_form.date_signed - ) - - else: - - # Do the update - Participant.objects.filter(pk=participant.pk).update( - created="2018-02-20T00:00:00Z", - modified="2018-02-20T00:00:00Z", - ) + # Do nothing due to issue with Django accessing properties not added + # until later migration + # This migration was moved to 0100 + pass class Migration(migrations.Migration): diff --git a/app/projects/migrations/0098_group_dataproject_group.py b/app/projects/migrations/0098_group_dataproject_group.py new file mode 100755 index 00000000..947cb370 --- /dev/null +++ b/app/projects/migrations/0098_group_dataproject_group.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.7 on 2023-03-24 20:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0097_participant_created_participant_modified'), + ] + + operations = [ + migrations.CreateModel( + name='Group', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(max_length=100, unique=True)), + ('title', models.CharField(max_length=255)), + ('description', models.TextField(blank=True)), + ('navigation_title', models.CharField(blank=True, max_length=20, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + migrations.AddField( + model_name='dataproject', + name='group', + field=models.ForeignKey(blank=True, help_text='Set this to manage where this project is shown in the navigation and interface.', null=True, on_delete=django.db.models.deletion.PROTECT, to='projects.group'), + ), + ] diff --git a/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py b/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py new file mode 100755 index 00000000..1f4173fc --- /dev/null +++ b/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py @@ -0,0 +1,41 @@ +# Generated by Django 4.1.7 on 2023-04-04 18:33 + +from django.db import migrations, models +import django.db.models.deletion +import projects.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0098_group_dataproject_group'), + ] + + operations = [ + migrations.CreateModel( + name='Bucket', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('default', models.BooleanField(default=False)), + ('provider', models.CharField(choices=[('s3', 'AWS S3')], default='s3', max_length=255)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + migrations.AddField( + model_name='dataproject', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text="Set this to a specific bucket where this project's files should be stored.", on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + migrations.AddField( + model_name='hostedfile', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text='Set this to a specific bucket where this file is stored.', on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + migrations.AddField( + model_name='hostedfileset', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text="Set this to a specific bucket where this set's files are stored.", on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + ] diff --git a/app/projects/migrations/0100_participant_created_participant_modified.py b/app/projects/migrations/0100_participant_created_participant_modified.py new file mode 100644 index 00000000..0872fea7 --- /dev/null +++ b/app/projects/migrations/0100_participant_created_participant_modified.py @@ -0,0 +1,47 @@ +# Generated by Django 4.1.6 on 2023-02-14 17:07 + +from django.db import migrations, models + +from projects.models import Participant, SignedAgreementForm + + +def migrate_participants_model(apps, schema_editor): + """ + Attempts to set a roughly accurate date of when each object would have + been created. This is calculated by fetching the date of the last + signed SignedAgreementForm relevant to the DataProjects. + """ + for participant in Participant.objects.all(): + + # Fetch signed agreement forms + signed_agreement_form = SignedAgreementForm.objects.filter(user=participant.user, project=participant.project).last() + if signed_agreement_form: + + # Set the dates + participant.created = signed_agreement_form.date_signed + participant.modified = signed_agreement_form.date_signed + + # Do the update + Participant.objects.filter(pk=participant.pk).update( + created=signed_agreement_form.date_signed, + modified=signed_agreement_form.date_signed + ) + + else: + + # Do the update + Participant.objects.filter(pk=participant.pk).update( + created="2018-02-20T00:00:00Z", + modified="2018-02-20T00:00:00Z", + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0099_bucket_dataproject_bucket_hostedfile_bucket_and_more'), + ] + + operations = [ + migrations.RunPython(migrate_participants_model), + ] diff --git a/app/projects/migrations/0101_challengetask_submission_instructions.py b/app/projects/migrations/0101_challengetask_submission_instructions.py new file mode 100755 index 00000000..5e649639 --- /dev/null +++ b/app/projects/migrations/0101_challengetask_submission_instructions.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-05-19 21:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0100_participant_created_participant_modified'), + ] + + operations = [ + migrations.AddField( + model_name='challengetask', + name='submission_instructions', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/app/projects/migrations/0102_agreementform_skippable.py b/app/projects/migrations/0102_agreementform_skippable.py new file mode 100644 index 00000000..04124026 --- /dev/null +++ b/app/projects/migrations/0102_agreementform_skippable.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.4 on 2023-09-12 16:20 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0101_challengetask_submission_instructions'), + ] + + operations = [ + migrations.AddField( + model_name='agreementform', + name='skippable', + field=models.BooleanField(default=False, help_text='Allow participants to skip this step in instances where they have submitted the agreement form via email or some other means. They will be required to include the name and contact information of the person who they submitted their signed agreement form to.'), + ), + ] diff --git a/app/projects/models.py b/app/projects/models.py index f096aaa8..d8318852 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -1,12 +1,19 @@ import uuid +import re from datetime import datetime +import boto3 +from botocore.exceptions import ClientError from django.conf import settings from django.db import models from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.db.models import JSONField from django.core.files.uploadedfile import UploadedFile +from django.utils.translation import gettext_lazy as _ + +import logging +logger = logging.getLogger(__name__) TEAM_PENDING = 'Pending' @@ -56,6 +63,8 @@ FILE_TYPE_PDF: ["application/pdf", "application/x-pdf"], } + + def get_agreement_form_upload_path(instance, filename): form_directory = 'agreementforms/' @@ -78,6 +87,137 @@ def get_institution_logo_upload_path(instance, filename): return '%s/%s.%s' % (form_directory, file_name, file_extension) +class Bucket(models.Model): + """ + An object store for project files. + """ + + class Provider(models.TextChoices): + S3 = 's3', _('AWS S3') + + name = models.CharField(max_length=255, blank=False, null=False) + default = models.BooleanField(default=False) + provider = models.CharField( + max_length=255, + blank=False, + null=False, + choices=Provider.choices, + default=Provider.S3, + ) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + def clean(self): + + # Check for multiple defaults + if self.default and Bucket.objects.filter(default=True): + raise ValidationError('Only one bucket may be configured as the default at one time.') + + # Check bucket for needed permissions + try: + match self.provider: + case Bucket.Provider.S3: + try: + # Get the s3 client + s3 = boto3.client("s3") + + # Download the test file + s3.list_objects_v2( + Bucket=self.name, + ) + + # Create a test file + key = f"test.{uuid.uuid4()}.txt" + s3.put_object( + Body="This is a test file.", + Bucket=self.name, + Key=key, + ) + + # Download the test file + s3.get_object( + Bucket=self.name, + Key=key, + ) + + # Delete the test file + s3.delete_object( + Bucket=self.name, + Key=key, + ) + + except ClientError as e: + logger.exception(f"Bucket permissions error: {e.response}") + raise ValidationError('This application has not been granted sufficient permissions on the bucket. Check logs for more info.') + + case _: + raise ValidationError('This application has not implemented validation for the specified bucket provider.') + + except Exception as e: + logger.exception(f"Bucket check error: {e}", exc_info=True) + raise ValidationError('This application could not verify sufficient bucket permissions. Check logs for more info.') + + @property + def uri(self): + return f"{self.provider}://{self.name}" + + @classmethod + def get_default_pk(cls): + """ + Returns the primary key of the default bucket. If this bucket does not + exist, it is created using the S3_BUCKET parameter in settings. + + :return: The primary key of the default bucket + :rtype: int + """ + bucket, created = cls.objects.get_or_create( + name=settings.S3_BUCKET, + default=True, + provider=Bucket.Provider.S3, + ) + + # Log if created + if created: + logger.info(f"Default bucket '{bucket.provider}://{bucket.name}' was created") + + return bucket.pk + + @classmethod + def split_uri(cls, uri): + """ + Accepts a bucket object's URI and splits it into three components: + provider, bucket and key. These three values are returned as a tuple. + The provider is converted to an instance of Bucket.Provider. If the + passed URI contains an unsupported provider, an exception is raised. + + :param uri: The URI of the object + :type uri: str + :raises ValueError: Raises an error if the URI is invalid + :raises ValueError: Raises an error if the URI's provider is unsupported + :return: Returns a tuple of the URI's components + :rtype: Bucket.Provider, str, str + """ + provider = None + try: + # Separate URI + pattern = r"(\w+):\/\/([^:\/\/]+?)\/(.+)" + provider, bucket, key = re.match(pattern, uri).groups() + + return Bucket.Provider(provider.lower()), bucket, key + + except ValueError: + raise ValueError(f"Unsupported bucket provider: '{provider}'") + + except Exception as e: + logger.exception(f"Invalid file URI '{uri}': {e}") + raise ValueError(f"Invalid file URI: '{uri}'") + + class Institution(models.Model): """ This represents an institution such as a university that might be co-sponsoring a challenge. @@ -113,6 +253,12 @@ class AgreementForm(models.Model): order = models.IntegerField(default=50, help_text="Indicate an order (lowest number = first listing) for how the Agreement Forms should be listed during registration workflows.") content = models.TextField(blank=True, null=True, help_text="If Agreement Form type is set to 'MODEL', the HTML set here will be rendered for the user") internal = models.BooleanField(default=False, help_text="Internal agreement forms are never presented to participants and are only submitted by administrators on behalf of participants") + skippable = models.BooleanField( + default=False, + help_text="Allow participants to skip this step in instances where they have submitted the agreement form via" + " email or some other means. They will be required to include the name and contact information of" + " the person who they submitted their signed agreement form to." + ) # Meta created = models.DateTimeField(auto_now_add=True) @@ -187,6 +333,22 @@ class DataProject(models.Model): order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for how the DataProjects should be listed.") + group = models.ForeignKey( + to="Group", + on_delete=models.PROTECT, + blank=True, + null=True, + help_text="Set this to manage where this project is shown in the navigation and interface." + ) + + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this project's files should be stored." + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -248,179 +410,6 @@ class Meta: verbose_name = 'Signed Agreement Form' verbose_name_plural = 'Signed Agreement Forms' - -class MIMIC3SignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - email = models.CharField(max_length=255) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - - class Meta: - verbose_name = 'MIMIC3 Signed Agreement Form Fields' - verbose_name_plural = 'MIMIC3 Signed Agreement Forms Fields' - - -class ROCSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # N2C2-t1 ROC - e_signature = models.CharField(max_length=255, null=True, blank=True) - organization = models.CharField(max_length=255, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'ROC signed agreement form fields' - verbose_name_plural = 'ROC signed agreement form fields' - -class DUASignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # N2C2-T1 DUA - person_name = models.CharField(max_length=1024, null=True, blank=True) - institution = models.CharField(max_length=255, null=True, blank=True) - address = models.TextField(null=True, blank=True) - city = models.CharField(max_length=255, null=True, blank=True) - state = models.CharField(max_length=255, null=True, blank=True) - zip = models.CharField(max_length=255, null=True, blank=True) - country = models.CharField(max_length=255, null=True, blank=True) - person_phone = models.CharField(max_length=255, null=True, blank=True) - person_email = models.CharField(max_length=255, null=True, blank=True) - place_of_business = models.CharField(max_length=255, null=True, blank=True) - contact_name = models.CharField(max_length=1024, null=True, blank=True) - business_phone = models.CharField(max_length=255, null=True, blank=True) - business_email = models.CharField(max_length=255, null=True, blank=True) - electronic_signature = models.CharField(max_length=255, null=True, blank=True) - professional_title = models.CharField(max_length=255, null=True, blank=True) - date = models.CharField(max_length=255, null=True, blank=True) - i_agree = models.CharField(max_length=10, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'DUA signed agreement form fields' - verbose_name_plural = 'DUA signed agreement form fields' - -class MAYOSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # Mayo DUA - institution = models.CharField(max_length=255, null=True, blank=True) - pi_name = models.CharField(max_length=1024, null=True, blank=True) - i_agree = models.CharField(max_length=3, null=True, blank=True) - recipient_institution = models.CharField(max_length=1024, null=True, blank=True) - recipient_by = models.CharField(max_length=255, null=True, blank=True) - recipient_its = models.CharField(max_length=255, null=True, blank=True) - recipient_attn = models.CharField(max_length=255, null=True, blank=True) - recipient_phone = models.CharField(max_length=255, null=True, blank=True) - recipient_fax = models.CharField(max_length=1024, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'Mayo DUA signed agreement form fields' - verbose_name_plural = 'Mayo DUA signed agreement form fields' - -class NLPWHYSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # NLP Research Purpose - research_use = models.TextField(null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'NLP Research Purpose signed agreement form fields' - verbose_name_plural = 'NLP Research Purpose signed agreement form fields' - -class NLPDUASignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # NLP DUA - form_type = models.CharField(max_length=255, null=True, blank=True) - data_user = models.CharField(max_length=255, null=True, blank=True) - individual_name = models.CharField(max_length=255, null=True, blank=True) - individual_professional_title = models.CharField(max_length=255, null=True, blank=True) - individual_address_1 = models.TextField(null=True, blank=True) - individual_address_2 = models.TextField(null=True, blank=True) - individual_address_city = models.CharField(max_length=255, null=True, blank=True) - individual_address_state = models.CharField(max_length=255, null=True, blank=True) - individual_address_zip = models.CharField(max_length=255, null=True, blank=True) - individual_address_country = models.CharField(max_length=255, null=True, blank=True) - individual_phone = models.CharField(max_length=255, null=True, blank=True) - individual_fax = models.CharField(max_length=255, null=True, blank=True) - individual_email = models.CharField(max_length=255, null=True, blank=True) - corporation_place_of_business = models.CharField(max_length=255, null=True, blank=True) - corporation_contact_name = models.CharField(max_length=255, null=True, blank=True) - corporation_phone = models.CharField(max_length=255, null=True, blank=True) - corporation_fax = models.CharField(max_length=255, null=True, blank=True) - corporation_email = models.CharField(max_length=255, null=True, blank=True) - research_team_person_1 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_2 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_3 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_4 = models.CharField(max_length=1024, null=True, blank=True) - data_user_signature = models.CharField(max_length=255, null=True, blank=True) - data_user_name = models.CharField(max_length=255, null=True, blank=True) - data_user_title = models.CharField(max_length=255, null=True, blank=True) - data_user_address_1 = models.TextField(null=True, blank=True) - data_user_address_2 = models.TextField(null=True, blank=True) - data_user_address_city = models.CharField(max_length=255, null=True, blank=True) - data_user_address_state = models.CharField(max_length=255, null=True, blank=True) - data_user_address_zip = models.CharField(max_length=255, null=True, blank=True) - data_user_address_country = models.CharField(max_length=255, null=True, blank=True) - data_user_date = models.CharField(max_length=255, null=True, blank=True) - registrant_is = models.CharField(max_length=255, null=True, blank=True) - commercial_registrant_is = models.CharField(max_length=255, null=True, blank=True) - data_user_acknowledge = models.CharField(max_length=3, null=True, blank=True) - partners_name = models.CharField(max_length=255, null=True, blank=True) - partners_title = models.CharField(max_length=255, null=True, blank=True) - partners_address = models.TextField(null=True, blank=True) - partners_date = models.CharField(max_length=255, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'NLP DUA signed agreement form fields' - verbose_name_plural = 'NLP DUA signed agreement form fields' - class Team(models.Model): """ This model describes a team of participants that are competing in a data challenge. @@ -514,6 +503,14 @@ class HostedFileSet(models.Model): project = models.ForeignKey(DataProject, on_delete=models.CASCADE) order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for file sets to appear within a DataProject.") + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this set's files are stored.", + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -550,6 +547,14 @@ class HostedFile(models.Model): order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for files to appear within a DataProject.") + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this file is stored.", + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -601,6 +606,9 @@ class ChallengeTask(models.Model): # Optional path to an html file that contains a form that should be completed when uploading a task solution submission_form_file_path = models.CharField(max_length=300, blank=True, null=True) + # Optional HTML content to be displayed along with submission upload for instructions on upload preparation + submission_instructions = models.TextField(blank=True, null=True) + # If blank, allow infinite submissions max_submissions = models.IntegerField(default=1, blank=True, null=True, help_text="Leave blank if you want there to be no cap.") @@ -672,3 +680,199 @@ class ChallengeTaskSubmissionDownload(models.Model): user = models.ForeignKey(User, on_delete=models.PROTECT) submission = models.ForeignKey(ChallengeTaskSubmission, on_delete=models.PROTECT) download_date = models.DateTimeField(auto_now_add=True) + + +class Group(models.Model): + """ + An optional grouping for projects. + """ + + key = models.CharField(max_length=100, blank=False, null=False, unique=True) + title = models.CharField(max_length=255, blank=False, null=False) + description = models.TextField(blank=True) + navigation_title = models.CharField(max_length=20, blank=True, null=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.title + + +################################################################################ +# Deprecated models +################################################################################ + + +class MIMIC3SignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + email = models.CharField(max_length=255) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = 'MIMIC3 Signed Agreement Form Fields' + verbose_name_plural = 'MIMIC3 Signed Agreement Forms Fields' + + +class ROCSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-t1 ROC + e_signature = models.CharField(max_length=255, null=True, blank=True) + organization = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'ROC signed agreement form fields' + verbose_name_plural = 'ROC signed agreement form fields' + +class DUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-T1 DUA + person_name = models.CharField(max_length=1024, null=True, blank=True) + institution = models.CharField(max_length=255, null=True, blank=True) + address = models.TextField(null=True, blank=True) + city = models.CharField(max_length=255, null=True, blank=True) + state = models.CharField(max_length=255, null=True, blank=True) + zip = models.CharField(max_length=255, null=True, blank=True) + country = models.CharField(max_length=255, null=True, blank=True) + person_phone = models.CharField(max_length=255, null=True, blank=True) + person_email = models.CharField(max_length=255, null=True, blank=True) + place_of_business = models.CharField(max_length=255, null=True, blank=True) + contact_name = models.CharField(max_length=1024, null=True, blank=True) + business_phone = models.CharField(max_length=255, null=True, blank=True) + business_email = models.CharField(max_length=255, null=True, blank=True) + electronic_signature = models.CharField(max_length=255, null=True, blank=True) + professional_title = models.CharField(max_length=255, null=True, blank=True) + date = models.CharField(max_length=255, null=True, blank=True) + i_agree = models.CharField(max_length=10, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'DUA signed agreement form fields' + verbose_name_plural = 'DUA signed agreement form fields' + +class MAYOSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # Mayo DUA + institution = models.CharField(max_length=255, null=True, blank=True) + pi_name = models.CharField(max_length=1024, null=True, blank=True) + i_agree = models.CharField(max_length=3, null=True, blank=True) + recipient_institution = models.CharField(max_length=1024, null=True, blank=True) + recipient_by = models.CharField(max_length=255, null=True, blank=True) + recipient_its = models.CharField(max_length=255, null=True, blank=True) + recipient_attn = models.CharField(max_length=255, null=True, blank=True) + recipient_phone = models.CharField(max_length=255, null=True, blank=True) + recipient_fax = models.CharField(max_length=1024, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'Mayo DUA signed agreement form fields' + verbose_name_plural = 'Mayo DUA signed agreement form fields' + +class NLPWHYSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP Research Purpose + research_use = models.TextField(null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP Research Purpose signed agreement form fields' + verbose_name_plural = 'NLP Research Purpose signed agreement form fields' + +class NLPDUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP DUA + form_type = models.CharField(max_length=255, null=True, blank=True) + data_user = models.CharField(max_length=255, null=True, blank=True) + individual_name = models.CharField(max_length=255, null=True, blank=True) + individual_professional_title = models.CharField(max_length=255, null=True, blank=True) + individual_address_1 = models.TextField(null=True, blank=True) + individual_address_2 = models.TextField(null=True, blank=True) + individual_address_city = models.CharField(max_length=255, null=True, blank=True) + individual_address_state = models.CharField(max_length=255, null=True, blank=True) + individual_address_zip = models.CharField(max_length=255, null=True, blank=True) + individual_address_country = models.CharField(max_length=255, null=True, blank=True) + individual_phone = models.CharField(max_length=255, null=True, blank=True) + individual_fax = models.CharField(max_length=255, null=True, blank=True) + individual_email = models.CharField(max_length=255, null=True, blank=True) + corporation_place_of_business = models.CharField(max_length=255, null=True, blank=True) + corporation_contact_name = models.CharField(max_length=255, null=True, blank=True) + corporation_phone = models.CharField(max_length=255, null=True, blank=True) + corporation_fax = models.CharField(max_length=255, null=True, blank=True) + corporation_email = models.CharField(max_length=255, null=True, blank=True) + research_team_person_1 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_2 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_3 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_4 = models.CharField(max_length=1024, null=True, blank=True) + data_user_signature = models.CharField(max_length=255, null=True, blank=True) + data_user_name = models.CharField(max_length=255, null=True, blank=True) + data_user_title = models.CharField(max_length=255, null=True, blank=True) + data_user_address_1 = models.TextField(null=True, blank=True) + data_user_address_2 = models.TextField(null=True, blank=True) + data_user_address_city = models.CharField(max_length=255, null=True, blank=True) + data_user_address_state = models.CharField(max_length=255, null=True, blank=True) + data_user_address_zip = models.CharField(max_length=255, null=True, blank=True) + data_user_address_country = models.CharField(max_length=255, null=True, blank=True) + data_user_date = models.CharField(max_length=255, null=True, blank=True) + registrant_is = models.CharField(max_length=255, null=True, blank=True) + commercial_registrant_is = models.CharField(max_length=255, null=True, blank=True) + data_user_acknowledge = models.CharField(max_length=3, null=True, blank=True) + partners_name = models.CharField(max_length=255, null=True, blank=True) + partners_title = models.CharField(max_length=255, null=True, blank=True) + partners_address = models.TextField(null=True, blank=True) + partners_date = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP DUA signed agreement form fields' + verbose_name_plural = 'NLP DUA signed agreement form fields' diff --git a/app/projects/views.py b/app/projects/views.py index ed536bef..0d0a708b 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -27,6 +27,7 @@ from projects.models import HostedFile from projects.models import Participant from projects.models import SignedAgreementForm +from projects.models import Group from projects.panels import SIGNUP_STEP_COMPLETED_STATUS from projects.panels import SIGNUP_STEP_CURRENT_STATUS from projects.panels import SIGNUP_STEP_FUTURE_STATUS @@ -113,6 +114,47 @@ def list_software_projects(request, template_name='projects/list-software-projec return render(request, template_name, context=context) +@method_decorator(public_user_auth_and_jwt, name='dispatch') +class GroupView(TemplateView): + """ + Builds and renders screens related to Groups. + """ + + group = None + template_name = 'projects/group.html' + + def dispatch(self, request, *args, **kwargs): + """ + Sets up the instance. + """ + + # Get the project key from the URL. + group_key = self.kwargs['group_key'] + + # If this project does not exist, display a 404 Error. + try: + self.group = Group.objects.get(key=group_key) + except ObjectDoesNotExist: + error_message = "The group you searched for does not exist." + return render(request, '404.html', {'error_message': error_message}) + + return super(GroupView, self).dispatch(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + """ + Dynamically builds the context for rendering the view based on information + about the user and the Group. + """ + # Get super's context. This is the dictionary of variables for the base template being rendered. + context = super(GroupView, self).get_context_data(**kwargs) + + # Add the project to the context. + context['group'] = self.group + context['projects'] = DataProject.objects.filter(group=self.group, visible=True).order_by(F('order').asc(nulls_last=True)) + + return context + + @method_decorator(public_user_auth_and_jwt, name='dispatch') class DataProjectView(TemplateView): """ diff --git a/app/static/agreementforms/4ce-research-purpose.html b/app/static/agreementforms/4ce-research-purpose.html new file mode 100644 index 00000000..2ba163c1 --- /dev/null +++ b/app/static/agreementforms/4ce-research-purpose.html @@ -0,0 +1,6 @@ +
+
+ + +
+
diff --git a/app/static/agreementforms/nlp_purpose.html b/app/static/agreementforms/nlp_purpose.html index 2ba163c1..9e84e2f7 100644 --- a/app/static/agreementforms/nlp_purpose.html +++ b/app/static/agreementforms/nlp_purpose.html @@ -1,4 +1,8 @@
+
+

  Training Model Notification

+ The n2c2 data cannot be redistributed in any way. If you will be using/training a large language model in your intended research, please confirm that the LLM will not integrate the data such that a user of the model will have access to the n2c2 data. Once you have done so, please indicate this in the research purpose of your application. Thank you! +
diff --git a/app/static/css/portal.css b/app/static/css/portal.css index d46194fc..d19a1e14 100644 --- a/app/static/css/portal.css +++ b/app/static/css/portal.css @@ -293,6 +293,41 @@ footer { color: #5bc0de; } + +.icon-col { + padding-top: 20px; + margin-top: 20px; + padding-right: 30px; +} + +.icon-col h1 { + font-size: 3.5em; +} + +.icon-primary { + color: #428bca; +} + +.icon-primary { + color: #428bca; +} + +.icon-success { + color: #5cb85c; +} + +.icon-danger { + color: #d9534f; +} + +.icon-warning { + color: #f0ad4e; +} + +.icon-info { + color: #5bc0de; +} + /* Bootstrap accordian panels styling */ .panel-group { diff --git a/app/static/institutionlogos/4ce_logo.png b/app/static/institutionlogos/4ce_logo.png new file mode 100644 index 00000000..bf454408 Binary files /dev/null and b/app/static/institutionlogos/4ce_logo.png differ diff --git a/app/static/institutionlogos/4ce_logo@0.5x.png b/app/static/institutionlogos/4ce_logo@0.5x.png new file mode 100644 index 00000000..9a4a3e23 Binary files /dev/null and b/app/static/institutionlogos/4ce_logo@0.5x.png differ diff --git a/app/static/js/portal.js b/app/static/js/portal.js new file mode 100644 index 00000000..c017ed2f --- /dev/null +++ b/app/static/js/portal.js @@ -0,0 +1,12 @@ + +/** + * Finds all 'button' elements contained in a form and toggles their 'disabled' proeprty. + * @param {String} formSelector The jQuery selector of the form to disable buttons for. + */ +function toggleFormButtons(formSelector) { + + // Toggle disabled state of all buttons in form + $(formSelector).find("button").each(function() { + $(this).prop("disabled", !$(this).prop("disabled")); + }); +} diff --git a/app/static/submissionforms/4ce-obesity-submissions.html b/app/static/submissionforms/4ce-obesity-submissions.html new file mode 100644 index 00000000..79a847b7 --- /dev/null +++ b/app/static/submissionforms/4ce-obesity-submissions.html @@ -0,0 +1,47 @@ +
+ +
+ + +
+ +
+ + +
+ +
+ +
+ + +
+
+ +
+ +
+ +
diff --git a/app/templates/base.html b/app/templates/base.html index fb545e7e..82380f13 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -47,6 +47,9 @@ + + + {% block tab_name %}DBMI Portal{% endblock %} + diff --git a/app/templates/manage/upload-signed-agreement-form-file.html b/app/templates/manage/upload-signed-agreement-form-file.html new file mode 100644 index 00000000..c22828ee --- /dev/null +++ b/app/templates/manage/upload-signed-agreement-form-file.html @@ -0,0 +1,23 @@ +{% load bootstrap3 %} + +
+
+ +

Signed Agreement Form File Upload

+
+
+ {% csrf_token %} + {% bootstrap_form form inline=True %} +
+
+ {% buttons %} + + + {% endbuttons %} +
+
diff --git a/app/templates/manage/upload-signed-agreement-form.html b/app/templates/manage/upload-signed-agreement-form.html index 795486a1..05860752 100644 --- a/app/templates/manage/upload-signed-agreement-form.html +++ b/app/templates/manage/upload-signed-agreement-form.html @@ -1,6 +1,9 @@ {% load bootstrap3 %} -
+

Signed Agreement Form Upload

@@ -13,7 +16,7 @@

Signed Agreement Form Upload

{% buttons %} {% endbuttons %}
diff --git a/app/templates/projects/group.html b/app/templates/projects/group.html new file mode 100644 index 00000000..255cf457 --- /dev/null +++ b/app/templates/projects/group.html @@ -0,0 +1,22 @@ +{% extends 'sub-base.html' %} + +{% block headscripts %} +{% endblock %} + +{% block title %}{{ group.title }}{% endblock %} +{% block subtitle %} +{{ group.description|safe }} +{% endblock %} + +{% block subcontent %} +
+{% for project in projects %} +
+
+ {% include "projects/project-blurb.html" with project_counter=forloop.counter0 %} +
+
+
+{% endfor %} +
+{% endblock %} diff --git a/app/templates/projects/participate/complete-tasks.html b/app/templates/projects/participate/complete-tasks.html index c14c20c2..2e77436e 100644 --- a/app/templates/projects/participate/complete-tasks.html +++ b/app/templates/projects/participate/complete-tasks.html @@ -40,6 +40,12 @@

{% endif %} + {% if task_detail.task.submission_instructions %} + + {% endif %} +
{# Check type #} - {% if signed_form.agreement_form.type == "FILE" %} + {% if signed_form.agreement_form.type == "FILE" or signed_form.upload %} {# Handle instances where a DUA was submitted when it was not required to upload a file #} {% if not signed_form.upload %} @@ -156,4 +163,12 @@

Actions

}); }); + +{# Add a placeholder for any modal dialogs #} +
+
+
+
+
+
{% endblock %} diff --git a/app/templates/projects/project.html b/app/templates/projects/project.html index 37f0a354..5d4327cd 100644 --- a/app/templates/projects/project.html +++ b/app/templates/projects/project.html @@ -108,12 +108,107 @@

+ + + {% endblock %} {% block footerscripts %}