From 8c5d8187a2b31f444b68c85e71b77b4152ea45a4 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Fri, 24 Mar 2023 23:04:14 -0600 Subject: [PATCH 01/25] HYP-292 - Added 4CE challenge; refactored navigation; added Groups to organize DataProjects --- app/hypatio/settings.py | 1 + app/hypatio/urls.py | 2 + app/hypatio/views.py | 36 +++++++++++++- app/projects/admin.py | 8 ++++ .../0098_group_dataproject_group.py | 31 ++++++++++++ app/projects/models.py | 26 ++++++++++ app/projects/views.py | 41 ++++++++++++++++ .../agreementforms/4ce-research-purpose.html | 6 +++ .../4ce-obesity-submissions.html | 47 +++++++++++++++++++ app/templates/base.html | 5 ++ app/templates/projects/group.html | 22 +++++++++ 11 files changed, 224 insertions(+), 1 deletion(-) create mode 100755 app/projects/migrations/0098_group_dataproject_group.py create mode 100644 app/static/agreementforms/4ce-research-purpose.html create mode 100644 app/static/submissionforms/4ce-obesity-submissions.html create mode 100644 app/templates/projects/group.html diff --git a/app/hypatio/settings.py b/app/hypatio/settings.py index fdd870c4..e2aaf35a 100644 --- a/app/hypatio/settings.py +++ b/app/hypatio/settings.py @@ -86,6 +86,7 @@ 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', + 'hypatio.views.navigation_context', ], }, }, diff --git a/app/hypatio/urls.py b/app/hypatio/urls.py index 28b9ffef..17ac072a 100644 --- a/app/hypatio/urls.py +++ b/app/hypatio/urls.py @@ -6,6 +6,7 @@ from projects.views import list_data_projects from projects.views import list_data_challenges from projects.views import list_software_projects +from projects.views import GroupView urlpatterns = [ @@ -18,5 +19,6 @@ re_path(r'^data-challenges/$', list_data_challenges, name='data-challenges'), re_path(r'^software-projects/$', list_software_projects, name='software-projects'), re_path(r'^healthcheck/?', include('health_check.urls')), + re_path(r'^groups/(?P[^/]+)/?', GroupView.as_view(), name="group"), re_path(r'^', index, name='index'), ] diff --git a/app/hypatio/views.py b/app/hypatio/views.py index 6da9267e..ebbd2f30 100644 --- a/app/hypatio/views.py +++ b/app/hypatio/views.py @@ -1,7 +1,9 @@ +import os from django.shortcuts import render +from django.utils.functional import SimpleLazyObject from hypatio.auth0authenticate import public_user_auth_and_jwt - +from projects.models import Group, DataProject @public_user_auth_and_jwt def index(request, template_name='index.html'): @@ -12,3 +14,35 @@ def index(request, template_name='index.html'): context = {} return render(request, template_name, context=context) + +def navigation_context(request): + """ + Includes global navigation context in all requests. + + This method is enabled by including it in settings.TEMPLATES as + a context processor. + + :param request: The current HttpRequest + :type request: HttpRequest + :return: The context that should be included in the response's context + :rtype: dict + """ + def group_context(): + + # Check for an active project and determine its group + groups = Group.objects.all() + active_group = None + project = DataProject.objects.filter(project_key=os.path.basename(os.path.normpath(request.path))).first() + if project: + + # Check for group + active_group = next((g for g in groups if project in g.dataproject_set.all()), None) + + return { + "groups": groups, + "active_group": active_group, + } + + return { + "navigation": SimpleLazyObject(group_context) + } diff --git a/app/projects/admin.py b/app/projects/admin.py index 42a66ef4..1beec8cf 100644 --- a/app/projects/admin.py +++ b/app/projects/admin.py @@ -2,6 +2,7 @@ from django.urls import reverse from django.utils.html import escape, mark_safe +from projects.models import Group from projects.models import DataProject from projects.models import AgreementForm from projects.models import SignedAgreementForm @@ -21,6 +22,12 @@ from projects.models import MAYOSignedAgreementFormFields from projects.models import MIMIC3SignedAgreementFormFields + +class GroupAdmin(admin.ModelAdmin): + list_display = ('title', 'key', 'created', 'modified', ) + readonly_fields = ('created', 'modified', ) + + class DataProjectAdmin(admin.ModelAdmin): list_display = ('name', 'project_key', 'informational_only', 'registration_open', 'requires_authorization', 'is_challenge', 'order', 'created', 'modified', ) list_filter = ('informational_only', 'registration_open', 'requires_authorization', 'is_challenge') @@ -140,6 +147,7 @@ class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): pass +admin.site.register(Group, GroupAdmin) admin.site.register(DataProject, DataProjectAdmin) admin.site.register(AgreementForm, AgreementformAdmin) admin.site.register(SignedAgreementForm, SignedagreementformAdmin) diff --git a/app/projects/migrations/0098_group_dataproject_group.py b/app/projects/migrations/0098_group_dataproject_group.py new file mode 100755 index 00000000..947cb370 --- /dev/null +++ b/app/projects/migrations/0098_group_dataproject_group.py @@ -0,0 +1,31 @@ +# Generated by Django 4.1.7 on 2023-03-24 20:28 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0097_participant_created_participant_modified'), + ] + + operations = [ + migrations.CreateModel( + name='Group', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(max_length=100, unique=True)), + ('title', models.CharField(max_length=255)), + ('description', models.TextField(blank=True)), + ('navigation_title', models.CharField(blank=True, max_length=20, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + migrations.AddField( + model_name='dataproject', + name='group', + field=models.ForeignKey(blank=True, help_text='Set this to manage where this project is shown in the navigation and interface.', null=True, on_delete=django.db.models.deletion.PROTECT, to='projects.group'), + ), + ] diff --git a/app/projects/models.py b/app/projects/models.py index f096aaa8..b5c5107b 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -187,6 +187,14 @@ class DataProject(models.Model): order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for how the DataProjects should be listed.") + group = models.ForeignKey( + to="Group", + on_delete=models.PROTECT, + blank=True, + null=True, + help_text="Set this to manage where this project is shown in the navigation and interface." + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -672,3 +680,21 @@ class ChallengeTaskSubmissionDownload(models.Model): user = models.ForeignKey(User, on_delete=models.PROTECT) submission = models.ForeignKey(ChallengeTaskSubmission, on_delete=models.PROTECT) download_date = models.DateTimeField(auto_now_add=True) + + +class Group(models.Model): + """ + An optional grouping for projects. + """ + + key = models.CharField(max_length=100, blank=False, null=False, unique=True) + title = models.CharField(max_length=255, blank=False, null=False) + description = models.TextField(blank=True) + navigation_title = models.CharField(max_length=20, blank=True, null=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.title diff --git a/app/projects/views.py b/app/projects/views.py index ed536bef..7640f190 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -27,6 +27,7 @@ from projects.models import HostedFile from projects.models import Participant from projects.models import SignedAgreementForm +from projects.models import Group from projects.panels import SIGNUP_STEP_COMPLETED_STATUS from projects.panels import SIGNUP_STEP_CURRENT_STATUS from projects.panels import SIGNUP_STEP_FUTURE_STATUS @@ -113,6 +114,46 @@ def list_software_projects(request, template_name='projects/list-software-projec return render(request, template_name, context=context) +@method_decorator(public_user_auth_and_jwt, name='dispatch') +class GroupView(TemplateView): + """ + Builds and renders screens related to Groups. + """ + + group = None + template_name = 'projects/group.html' + + def dispatch(self, request, *args, **kwargs): + """ + Sets up the instance. + """ + + # Get the project key from the URL. + group_key = self.kwargs['group_key'] + + # If this project does not exist, display a 404 Error. + try: + self.group = Group.objects.get(key=group_key) + except ObjectDoesNotExist: + error_message = "The group you searched for does not exist." + return render(request, '404.html', {'error_message': error_message}) + + return super(GroupView, self).dispatch(request, *args, **kwargs) + + def get_context_data(self, **kwargs): + """ + Dynamically builds the context for rendering the view based on information + about the user and the Group. + """ + # Get super's context. This is the dictionary of variables for the base template being rendered. + context = super(GroupView, self).get_context_data(**kwargs) + + # Add the project to the context. + context['group'] = self.group + + return context + + @method_decorator(public_user_auth_and_jwt, name='dispatch') class DataProjectView(TemplateView): """ diff --git a/app/static/agreementforms/4ce-research-purpose.html b/app/static/agreementforms/4ce-research-purpose.html new file mode 100644 index 00000000..2ba163c1 --- /dev/null +++ b/app/static/agreementforms/4ce-research-purpose.html @@ -0,0 +1,6 @@ +
+
+ + +
+
diff --git a/app/static/submissionforms/4ce-obesity-submissions.html b/app/static/submissionforms/4ce-obesity-submissions.html new file mode 100644 index 00000000..79a847b7 --- /dev/null +++ b/app/static/submissionforms/4ce-obesity-submissions.html @@ -0,0 +1,47 @@ +
+ +
+ + +
+ +
+ + +
+ +
+ +
+ + +
+
+ +
+ +
+ +
diff --git a/app/templates/base.html b/app/templates/base.html index fb545e7e..b392ff55 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -212,6 +212,11 @@ {% url 'data-challenges' as data_challenges_url %}
  • n2c2 Challenges
    • + {% for group in navigation.groups %} + {% url 'group' group.key as group_url %} +
  • {{ group.navigation_title|default:group.title }}
    • + {% endfor %} +
  • Contact
      diff --git a/app/templates/projects/group.html b/app/templates/projects/group.html new file mode 100644 index 00000000..83d31bf0 --- /dev/null +++ b/app/templates/projects/group.html @@ -0,0 +1,22 @@ +{% extends 'sub-base.html' %} + +{% block headscripts %} +{% endblock %} + +{% block title %}{{ group.title }}{% endblock %} +{% block subtitle %} +{{ group.description }} +{% endblock %} + +{% block subcontent %} +
      +{% for project in group.dataproject_set.all %} +
      +
      + {% include "projects/project-blurb.html" with project_counter=forloop.counter0 %} +
      +
      +
      +{% endfor %} +
      +{% endblock %} From d6f9b2ddcfad7ed36f648e0f32a31aa3fc8475b5 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Sat, 25 Mar 2023 10:25:09 -0600 Subject: [PATCH 02/25] HYP-292 - Fixed DataProject query for Group views; Groups now determine navigation menu --- app/projects/views.py | 1 + app/templates/base.html | 6 ------ app/templates/projects/group.html | 4 ++-- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/app/projects/views.py b/app/projects/views.py index 7640f190..0d0a708b 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -150,6 +150,7 @@ def get_context_data(self, **kwargs): # Add the project to the context. context['group'] = self.group + context['projects'] = DataProject.objects.filter(group=self.group, visible=True).order_by(F('order').asc(nulls_last=True)) return context diff --git a/app/templates/base.html b/app/templates/base.html index b392ff55..31a6b5e1 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -206,12 +206,6 @@ {% url 'index' as index_url %}
    • Home
      • - {% url 'projects:view-project' 'n2c2-nlp' as data_sets_url %} -
    • n2c2 Data Sets
      • - - {% url 'data-challenges' as data_challenges_url %} -
    • n2c2 Challenges
      • - {% for group in navigation.groups %} {% url 'group' group.key as group_url %}
    • {{ group.navigation_title|default:group.title }}
      • diff --git a/app/templates/projects/group.html b/app/templates/projects/group.html index 83d31bf0..255cf457 100644 --- a/app/templates/projects/group.html +++ b/app/templates/projects/group.html @@ -5,12 +5,12 @@ {% block title %}{{ group.title }}{% endblock %} {% block subtitle %} -{{ group.description }} +{{ group.description|safe }} {% endblock %} {% block subcontent %}
      -{% for project in group.dataproject_set.all %} +{% for project in projects %}
      {% include "projects/project-blurb.html" with project_counter=forloop.counter0 %} From d5ae155248c4de22b8ddd53cdb9e8a4a02665aa4 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Tue, 4 Apr 2023 13:24:09 -0600 Subject: [PATCH 03/25] HYP-293 - Setup Buckets via model to allow project-specific buckets --- app/hypatio/file_services.py | 103 ++-- app/manage/api.py | 16 +- app/projects/admin.py | 75 +-- app/projects/api.py | 6 +- ...oject_bucket_hostedfile_bucket_and_more.py | 41 ++ app/projects/models.py | 515 ++++++++++++------ app/static/institutionlogos/4ce_logo.png | Bin 0 -> 3413 bytes app/static/institutionlogos/4ce_logo@0.5x.png | Bin 0 -> 2539 bytes requirements.txt | 70 +-- 9 files changed, 506 insertions(+), 320 deletions(-) create mode 100755 app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py create mode 100644 app/static/institutionlogos/4ce_logo.png create mode 100644 app/static/institutionlogos/4ce_logo@0.5x.png diff --git a/app/hypatio/file_services.py b/app/hypatio/file_services.py index e7347288..70695e2a 100644 --- a/app/hypatio/file_services.py +++ b/app/hypatio/file_services.py @@ -5,6 +5,8 @@ from botocore.client import Config from django.conf import settings +from projects.models import Bucket + import logging logger = logging.getLogger(__name__) @@ -262,41 +264,60 @@ def _s3_client(): return boto3.client('s3', config=Config(signature_version='s3v4')) -def get_download_url(file_name, expires_in=3600): +def get_download_url(file_uri, expires_in=3600): """ Returns an S3 URL for project related files not tracked by fileservice. """ - logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_name)) + logger.debug('[file_services][get_download_url] Generating URL for {}'.format(file_uri)) + + # Separate URI + provider, bucket, key = Bucket.split_uri(file_uri) + + # Check provider + match provider: + case Bucket.Provider.S3: + + # Generate the URL to get the file object + url = _s3_client().generate_presigned_url( + ClientMethod='get_object', + Params={ + 'Bucket': bucket, + 'Key': key + }, + ExpiresIn=expires_in + ) - # Generate the URL to get the file object - url = _s3_client().generate_presigned_url( - ClientMethod='get_object', - Params={ - 'Bucket': settings.S3_BUCKET, - 'Key': file_name - }, - ExpiresIn=expires_in - ) + case _: + raise NotImplementedError(f"Could not generate download URL for URI: {file_uri}") - logger.debug('[file_services][get_download_url] Generated URL: {}'.format(url)) + logger.debug(f'[file_services][get_download_url] Generated URL: {url}') return url -def upload_file(file_name, file, expires_in=3600): +def upload_file(file, file_uri, expires_in=3600): """ Enables uploading of files directly to the Hypatio S3 bucket without fileservice tracking. """ + logger.debug('[file_services][upload_file] Uploading file to: {}'.format(file_uri)) - logger.error('[file_services][upload_file] Uploading file: {}'.format(file_name)) + # Separate URI + provider, bucket, key = Bucket.split_uri(file_uri) - # Generate the POST attributes - post = _s3_client().generate_presigned_post( - Bucket=settings.S3_BUCKET, - Key=file_name, - ExpiresIn=expires_in - ) + # Check provider + match provider: + case Bucket.Provider.S3: + + # Generate the POST attributes + post = _s3_client().generate_presigned_post( + Bucket=bucket, + Key=key, + ExpiresIn=expires_in + ) + + case _: + raise NotImplementedError(f"Could not generate upload for URI: {file_uri}") # Perform the request to upload the file files = {"file": file} @@ -315,34 +336,46 @@ def upload_file(file_name, file, expires_in=3600): logger.exception(e) -def host_file(request, file_uuid, file_location, file_name): +def host_file(request, file_uuid, file_uri): """ Copies a file from the Fileservice bucket to the Hypatio hosted files bucket """ try: - # Make the request. + # Get the original file from Fileservice file = get(request, '/api/file/{}/'.format(file_uuid)) logger.debug(file) - # Perform the request to copy the file - source = {'Bucket': settings.FILESERVICE_AWS_BUCKET, 'Key': file['locations'][0]['url'].split('/', 3)[3]} - key = os.path.join(file_location, file_name) + # Separate destination URI + provider, bucket, key = Bucket.split_uri(file_uri) + + # Check provider + match provider: + case Bucket.Provider.S3: + + # Perform the request to copy the file + _, origin_bucket, origin_key = Bucket.split_uri(file['locations'][0]['url']) + source = {'Bucket': origin_bucket, 'Key': origin_key} + logger.debug( + f'Fileservice: Copying s3://{origin_bucket}/{origin_key}' + f' to {provider.value}://{bucket}/{key}' + ) - logger.debug(f'Fileservice: Copying {source["Bucket"]}/{source["Key"]}' - f' to {settings.S3_BUCKET}/{key}') + # Generate the URL to get the file object + _s3_client().copy_object( + CopySource=source, + Bucket=bucket, + Key=key, + ) - # Generate the URL to get the file object - _s3_client().copy_object( - CopySource=source, - Bucket=settings.S3_BUCKET, - Key=key, - ) + return True - return True + case _: + raise NotImplementedError(f"Could not generate upload for URI: {file_uri}") except Exception as e: logger.exception('[file_services][host_file] Error: {}'.format(e), exc_info=True, extra={ - 'request': request, 'file_uuid': file_uuid, 'file_location': file_location, 'file_name': file_name + 'request': request, 'bucket': bucket, 'file_uuid': file_uuid, + 'file_uri': file_uri, }) return False diff --git a/app/manage/api.py b/app/manage/api.py index 9c2e7277..bdbd7ef8 100644 --- a/app/manage/api.py +++ b/app/manage/api.py @@ -1018,14 +1018,18 @@ def host_submission(request, fileservice_uuid): return HttpResponse(host_submission_form.errors.as_json(), status=400) try: + # Build the destination URI + file_uri = f"{project.bucket.uri}/" \ + f"{host_submission_form.cleaned_data['file_location']}/" \ + f"{host_submission_form.cleaned_data['file_name']}" + # Do the copy - logger.debug(f'[HYPATIO][DEBUG][host_submission] Copying submission "{submission}" ' - f'to hosted location "{host_submission_form.cleaned_data["file_location"]}/' - f'{host_submission_form.cleaned_data["file_name"]}"') + logger.debug( + f'[HYPATIO][DEBUG][host_submission] Copying submission ' + f'"{submission}" to hosted location "{file_uri}"' + ) - if host_file(request=request, file_uuid=fileservice_uuid, - file_location=host_submission_form.cleaned_data['file_location'], - file_name=host_submission_form.cleaned_data['file_name']): + if host_file(request=request, file_uuid=fileservice_uuid, file_uri=file_uri): logger.debug(f'[HYPATIO][DEBUG][host_submission] File was copied successfully') host_submission_form.save() diff --git a/app/projects/admin.py b/app/projects/admin.py index 1beec8cf..94a4fc9d 100644 --- a/app/projects/admin.py +++ b/app/projects/admin.py @@ -15,6 +15,7 @@ from projects.models import ChallengeTask from projects.models import ChallengeTaskSubmission from projects.models import ChallengeTaskSubmissionDownload +from projects.models import Bucket from projects.models import NLPDUASignedAgreementFormFields from projects.models import NLPWHYSignedAgreementFormFields from projects.models import DUASignedAgreementFormFields @@ -28,6 +29,11 @@ class GroupAdmin(admin.ModelAdmin): readonly_fields = ('created', 'modified', ) +class BucketAdmin(admin.ModelAdmin): + list_display = ('name', 'provider', 'created', 'modified', ) + readonly_fields = ('created', 'modified', ) + + class DataProjectAdmin(admin.ModelAdmin): list_display = ('name', 'project_key', 'informational_only', 'registration_open', 'requires_authorization', 'is_challenge', 'order', 'created', 'modified', ) list_filter = ('informational_only', 'registration_open', 'requires_authorization', 'is_challenge') @@ -87,67 +93,8 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin): search_fields = ('user__email', ) -class SignedAgreementFormFieldsAdmin(admin.ModelAdmin): - def get_user(self, obj): - return obj.signed_agreement_form.user.email - get_user.short_description = 'User' - get_user.admin_order_field = 'signed_agreement_form__user__email' - - def get_status(self, obj): - return obj.signed_agreement_form.status - get_status.short_description = 'Status' - get_status.admin_order_field = 'signed_agreement_form__status' - - def signed_agreement_form_link(self, obj): - link = reverse("admin:projects_signedagreementform_change", args=[obj.signed_agreement_form.id]) - return mark_safe(f'{escape(obj.signed_agreement_form.__str__())}') - - signed_agreement_form_link.short_description = 'Signed Agreement Form' - signed_agreement_form_link.admin_order_field = 'signed agreement form' - - list_display = ( - 'get_user', - 'get_status', - 'signed_agreement_form_link' - ) - search_fields = ( - 'signed_agreement_form__user__email', - 'signed_agreement_form__agreement_form__project', - 'signed_agreement_form__agreement_form__short_name', - 'signed_agreement_form', - ) - readonly_fields = ( - 'signed_agreement_form', - 'created', - 'modified' - ) - - -class NLPDUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class NLPWHYSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class DUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class ROCSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class MAYOSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - -class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): - pass - - admin.site.register(Group, GroupAdmin) +admin.site.register(Bucket, BucketAdmin) admin.site.register(DataProject, DataProjectAdmin) admin.site.register(AgreementForm, AgreementformAdmin) admin.site.register(SignedAgreementForm, SignedagreementformAdmin) @@ -160,11 +107,3 @@ class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): admin.site.register(ChallengeTask, ChallengeTaskAdmin) admin.site.register(ChallengeTaskSubmission, ChallengeTaskSubmissionAdmin) admin.site.register(ChallengeTaskSubmissionDownload, ChallengeTaskSubmissionDownloadAdmin) - - -admin.site.register(NLPDUASignedAgreementFormFields, NLPDUASignedAgreementFormFieldsAdmin) -admin.site.register(NLPWHYSignedAgreementFormFields, NLPWHYSignedAgreementFormFieldsAdmin) -admin.site.register(DUASignedAgreementFormFields, DUASignedAgreementFormFieldsAdmin) -admin.site.register(ROCSignedAgreementFormFields, ROCSignedAgreementFormFieldsAdmin) -admin.site.register(MAYOSignedAgreementFormFields, MAYOSignedAgreementFormFieldsAdmin) -admin.site.register(MIMIC3SignedAgreementFormFields, MIMIC3SignedAgreementFormFieldsAdmin) diff --git a/app/projects/api.py b/app/projects/api.py index 024342c6..f3f3dd11 100644 --- a/app/projects/api.py +++ b/app/projects/api.py @@ -360,10 +360,10 @@ def download_dataset(request): # Save a record of this person downloading this file. HostedFileDownload.objects.create(user=request.user, hosted_file=file_to_download) - s3_filename = file_to_download.file_location + "/" + file_to_download.file_name - logger.debug("[download_dataset] - User " + request.user.email + " is downloading file " + s3_filename + " from bucket " + settings.S3_BUCKET + ".") + file_uri = f"{file_to_download.project.bucket.uri}/{file_to_download.file_location}/{file_to_download.file_name}" + logger.debug(f"[download_dataset] - User {request.user.email} is downloading file {file_uri}.") - download_url = get_download_url(s3_filename) + download_url = get_download_url(file_uri) response = redirect(download_url) response['Content-Disposition'] = 'attachment' diff --git a/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py b/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py new file mode 100755 index 00000000..1f4173fc --- /dev/null +++ b/app/projects/migrations/0099_bucket_dataproject_bucket_hostedfile_bucket_and_more.py @@ -0,0 +1,41 @@ +# Generated by Django 4.1.7 on 2023-04-04 18:33 + +from django.db import migrations, models +import django.db.models.deletion +import projects.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0098_group_dataproject_group'), + ] + + operations = [ + migrations.CreateModel( + name='Bucket', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255)), + ('default', models.BooleanField(default=False)), + ('provider', models.CharField(choices=[('s3', 'AWS S3')], default='s3', max_length=255)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ], + ), + migrations.AddField( + model_name='dataproject', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text="Set this to a specific bucket where this project's files should be stored.", on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + migrations.AddField( + model_name='hostedfile', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text='Set this to a specific bucket where this file is stored.', on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + migrations.AddField( + model_name='hostedfileset', + name='bucket', + field=models.ForeignKey(default=projects.models.Bucket.get_default_pk, help_text="Set this to a specific bucket where this set's files are stored.", on_delete=django.db.models.deletion.SET_DEFAULT, to='projects.bucket'), + ), + ] diff --git a/app/projects/models.py b/app/projects/models.py index b5c5107b..d524df3d 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -1,12 +1,19 @@ import uuid +import re from datetime import datetime +import boto3 +from botocore.exceptions import ClientError from django.conf import settings from django.db import models from django.contrib.auth.models import User from django.core.exceptions import ValidationError from django.db.models import JSONField from django.core.files.uploadedfile import UploadedFile +from django.utils.translation import gettext_lazy as _ + +import logging +logger = logging.getLogger(__name__) TEAM_PENDING = 'Pending' @@ -56,6 +63,8 @@ FILE_TYPE_PDF: ["application/pdf", "application/x-pdf"], } + + def get_agreement_form_upload_path(instance, filename): form_directory = 'agreementforms/' @@ -78,6 +87,137 @@ def get_institution_logo_upload_path(instance, filename): return '%s/%s.%s' % (form_directory, file_name, file_extension) +class Bucket(models.Model): + """ + An object store for project files. + """ + + class Provider(models.TextChoices): + S3 = 's3', _('AWS S3') + + name = models.CharField(max_length=255, blank=False, null=False) + default = models.BooleanField(default=False) + provider = models.CharField( + max_length=255, + blank=False, + null=False, + choices=Provider.choices, + default=Provider.S3, + ) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + def __str__(self): + return self.name + + def clean(self): + + # Check for multiple defaults + if self.objects.filter(default=True): + raise ValidationError('Only one bucket may be configured as the default at one time.') + + # Check bucket for needed permissions + try: + match self.provider: + case Bucket.Provider.S3: + try: + # Get the s3 client + s3 = boto3.client("s3") + + # Download the test file + s3.list_objects_v2( + Bucket=self.name, + ) + + # Create a test file + key = f"test.{uuid.uuid4()}.txt" + s3.put_object( + Body="This is a test file.", + Bucket=self.name, + Key=key, + ) + + # Download the test file + s3.get_object( + Bucket=self.name, + Key=key, + ) + + # Delete the test file + s3.delete_object( + Bucket=self.name, + Key=key, + ) + + except ClientError as e: + logger.exception(f"Bucket permissions error: {e.response}") + raise ValidationError('This application has not been granted sufficient permissions on the bucket. Check logs for more info.') + + case _: + raise ValidationError('This application has not implemented validation for the specified bucket provider.') + + except Exception as e: + logger.exception(f"Bucket check error: {e}", exc_info=True) + raise ValidationError('This application could not verify sufficient bucket permissions. Check logs for more info.') + + @property + def uri(self): + return f"{self.provider}://{self.name}" + + @classmethod + def get_default_pk(cls): + """ + Returns the primary key of the default bucket. If this bucket does not + exist, it is created using the S3_BUCKET parameter in settings. + + :return: The primary key of the default bucket + :rtype: int + """ + bucket, created = cls.objects.get_or_create( + name=settings.S3_BUCKET, + default=True, + provider=Bucket.Provider.S3, + ) + + # Log if created + if created: + logger.info(f"Default bucket '{bucket.provider}://{bucket.name}' was created") + + return bucket.pk + + @classmethod + def split_uri(cls, uri): + """ + Accepts a bucket object's URI and splits it into three components: + provider, bucket and key. These three values are returned as a tuple. + The provider is converted to an instance of Bucket.Provider. If the + passed URI contains an unsupported provider, an exception is raised. + + :param uri: The URI of the object + :type uri: str + :raises ValueError: Raises an error if the URI is invalid + :raises ValueError: Raises an error if the URI's provider is unsupported + :return: Returns a tuple of the URI's components + :rtype: Bucket.Provider, str, str + """ + provider = None + try: + # Separate URI + pattern = r"(\w+):\/\/([^:\/\/]+?)\/(.+)" + provider, bucket, key = re.match(pattern, uri.lower()).groups() + + return Bucket.Provider(provider), bucket, key + + except ValueError: + raise ValueError(f"Unsupported bucket provider: '{provider}'") + + except Exception as e: + logger.exception(f"Invalid file URI '{uri}': {e}") + raise ValueError(f"Invalid file URI: '{uri}'") + + class Institution(models.Model): """ This represents an institution such as a university that might be co-sponsoring a challenge. @@ -195,6 +335,14 @@ class DataProject(models.Model): help_text="Set this to manage where this project is shown in the navigation and interface." ) + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this project's files should be stored." + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -256,179 +404,6 @@ class Meta: verbose_name = 'Signed Agreement Form' verbose_name_plural = 'Signed Agreement Forms' - -class MIMIC3SignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - email = models.CharField(max_length=255) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - - class Meta: - verbose_name = 'MIMIC3 Signed Agreement Form Fields' - verbose_name_plural = 'MIMIC3 Signed Agreement Forms Fields' - - -class ROCSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # N2C2-t1 ROC - e_signature = models.CharField(max_length=255, null=True, blank=True) - organization = models.CharField(max_length=255, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'ROC signed agreement form fields' - verbose_name_plural = 'ROC signed agreement form fields' - -class DUASignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # N2C2-T1 DUA - person_name = models.CharField(max_length=1024, null=True, blank=True) - institution = models.CharField(max_length=255, null=True, blank=True) - address = models.TextField(null=True, blank=True) - city = models.CharField(max_length=255, null=True, blank=True) - state = models.CharField(max_length=255, null=True, blank=True) - zip = models.CharField(max_length=255, null=True, blank=True) - country = models.CharField(max_length=255, null=True, blank=True) - person_phone = models.CharField(max_length=255, null=True, blank=True) - person_email = models.CharField(max_length=255, null=True, blank=True) - place_of_business = models.CharField(max_length=255, null=True, blank=True) - contact_name = models.CharField(max_length=1024, null=True, blank=True) - business_phone = models.CharField(max_length=255, null=True, blank=True) - business_email = models.CharField(max_length=255, null=True, blank=True) - electronic_signature = models.CharField(max_length=255, null=True, blank=True) - professional_title = models.CharField(max_length=255, null=True, blank=True) - date = models.CharField(max_length=255, null=True, blank=True) - i_agree = models.CharField(max_length=10, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'DUA signed agreement form fields' - verbose_name_plural = 'DUA signed agreement form fields' - -class MAYOSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # Mayo DUA - institution = models.CharField(max_length=255, null=True, blank=True) - pi_name = models.CharField(max_length=1024, null=True, blank=True) - i_agree = models.CharField(max_length=3, null=True, blank=True) - recipient_institution = models.CharField(max_length=1024, null=True, blank=True) - recipient_by = models.CharField(max_length=255, null=True, blank=True) - recipient_its = models.CharField(max_length=255, null=True, blank=True) - recipient_attn = models.CharField(max_length=255, null=True, blank=True) - recipient_phone = models.CharField(max_length=255, null=True, blank=True) - recipient_fax = models.CharField(max_length=1024, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'Mayo DUA signed agreement form fields' - verbose_name_plural = 'Mayo DUA signed agreement form fields' - -class NLPWHYSignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # NLP Research Purpose - research_use = models.TextField(null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'NLP Research Purpose signed agreement form fields' - verbose_name_plural = 'NLP Research Purpose signed agreement form fields' - -class NLPDUASignedAgreementFormFields(models.Model): - - signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) - - # All DUAs - day = models.CharField(max_length=2, null=True, blank=True) - month = models.CharField(max_length=20, null=True, blank=True) - year = models.CharField(max_length=4, null=True, blank=True) - - # NLP DUA - form_type = models.CharField(max_length=255, null=True, blank=True) - data_user = models.CharField(max_length=255, null=True, blank=True) - individual_name = models.CharField(max_length=255, null=True, blank=True) - individual_professional_title = models.CharField(max_length=255, null=True, blank=True) - individual_address_1 = models.TextField(null=True, blank=True) - individual_address_2 = models.TextField(null=True, blank=True) - individual_address_city = models.CharField(max_length=255, null=True, blank=True) - individual_address_state = models.CharField(max_length=255, null=True, blank=True) - individual_address_zip = models.CharField(max_length=255, null=True, blank=True) - individual_address_country = models.CharField(max_length=255, null=True, blank=True) - individual_phone = models.CharField(max_length=255, null=True, blank=True) - individual_fax = models.CharField(max_length=255, null=True, blank=True) - individual_email = models.CharField(max_length=255, null=True, blank=True) - corporation_place_of_business = models.CharField(max_length=255, null=True, blank=True) - corporation_contact_name = models.CharField(max_length=255, null=True, blank=True) - corporation_phone = models.CharField(max_length=255, null=True, blank=True) - corporation_fax = models.CharField(max_length=255, null=True, blank=True) - corporation_email = models.CharField(max_length=255, null=True, blank=True) - research_team_person_1 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_2 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_3 = models.CharField(max_length=1024, null=True, blank=True) - research_team_person_4 = models.CharField(max_length=1024, null=True, blank=True) - data_user_signature = models.CharField(max_length=255, null=True, blank=True) - data_user_name = models.CharField(max_length=255, null=True, blank=True) - data_user_title = models.CharField(max_length=255, null=True, blank=True) - data_user_address_1 = models.TextField(null=True, blank=True) - data_user_address_2 = models.TextField(null=True, blank=True) - data_user_address_city = models.CharField(max_length=255, null=True, blank=True) - data_user_address_state = models.CharField(max_length=255, null=True, blank=True) - data_user_address_zip = models.CharField(max_length=255, null=True, blank=True) - data_user_address_country = models.CharField(max_length=255, null=True, blank=True) - data_user_date = models.CharField(max_length=255, null=True, blank=True) - registrant_is = models.CharField(max_length=255, null=True, blank=True) - commercial_registrant_is = models.CharField(max_length=255, null=True, blank=True) - data_user_acknowledge = models.CharField(max_length=3, null=True, blank=True) - partners_name = models.CharField(max_length=255, null=True, blank=True) - partners_title = models.CharField(max_length=255, null=True, blank=True) - partners_address = models.TextField(null=True, blank=True) - partners_date = models.CharField(max_length=255, null=True, blank=True) - - # Meta - created = models.DateTimeField(auto_now_add=True) - modified = models.DateTimeField(auto_now=True) - class Meta: - verbose_name = 'NLP DUA signed agreement form fields' - verbose_name_plural = 'NLP DUA signed agreement form fields' - class Team(models.Model): """ This model describes a team of participants that are competing in a data challenge. @@ -522,6 +497,14 @@ class HostedFileSet(models.Model): project = models.ForeignKey(DataProject, on_delete=models.CASCADE) order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for file sets to appear within a DataProject.") + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this set's files are stored.", + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -558,6 +541,14 @@ class HostedFile(models.Model): order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for files to appear within a DataProject.") + # Set the optional bucket to use for storage + bucket = models.ForeignKey( + to="Bucket", + on_delete=models.SET_DEFAULT, + default=Bucket.get_default_pk, + help_text="Set this to a specific bucket where this file is stored.", + ) + # Meta created = models.DateTimeField(auto_now_add=True) modified = models.DateTimeField(auto_now=True) @@ -698,3 +689,181 @@ class Group(models.Model): def __str__(self): return self.title + + +################################################################################ +# Deprecated models +################################################################################ + + +class MIMIC3SignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + email = models.CharField(max_length=255) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = 'MIMIC3 Signed Agreement Form Fields' + verbose_name_plural = 'MIMIC3 Signed Agreement Forms Fields' + + +class ROCSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-t1 ROC + e_signature = models.CharField(max_length=255, null=True, blank=True) + organization = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'ROC signed agreement form fields' + verbose_name_plural = 'ROC signed agreement form fields' + +class DUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-T1 DUA + person_name = models.CharField(max_length=1024, null=True, blank=True) + institution = models.CharField(max_length=255, null=True, blank=True) + address = models.TextField(null=True, blank=True) + city = models.CharField(max_length=255, null=True, blank=True) + state = models.CharField(max_length=255, null=True, blank=True) + zip = models.CharField(max_length=255, null=True, blank=True) + country = models.CharField(max_length=255, null=True, blank=True) + person_phone = models.CharField(max_length=255, null=True, blank=True) + person_email = models.CharField(max_length=255, null=True, blank=True) + place_of_business = models.CharField(max_length=255, null=True, blank=True) + contact_name = models.CharField(max_length=1024, null=True, blank=True) + business_phone = models.CharField(max_length=255, null=True, blank=True) + business_email = models.CharField(max_length=255, null=True, blank=True) + electronic_signature = models.CharField(max_length=255, null=True, blank=True) + professional_title = models.CharField(max_length=255, null=True, blank=True) + date = models.CharField(max_length=255, null=True, blank=True) + i_agree = models.CharField(max_length=10, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'DUA signed agreement form fields' + verbose_name_plural = 'DUA signed agreement form fields' + +class MAYOSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # Mayo DUA + institution = models.CharField(max_length=255, null=True, blank=True) + pi_name = models.CharField(max_length=1024, null=True, blank=True) + i_agree = models.CharField(max_length=3, null=True, blank=True) + recipient_institution = models.CharField(max_length=1024, null=True, blank=True) + recipient_by = models.CharField(max_length=255, null=True, blank=True) + recipient_its = models.CharField(max_length=255, null=True, blank=True) + recipient_attn = models.CharField(max_length=255, null=True, blank=True) + recipient_phone = models.CharField(max_length=255, null=True, blank=True) + recipient_fax = models.CharField(max_length=1024, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'Mayo DUA signed agreement form fields' + verbose_name_plural = 'Mayo DUA signed agreement form fields' + +class NLPWHYSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP Research Purpose + research_use = models.TextField(null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP Research Purpose signed agreement form fields' + verbose_name_plural = 'NLP Research Purpose signed agreement form fields' + +class NLPDUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP DUA + form_type = models.CharField(max_length=255, null=True, blank=True) + data_user = models.CharField(max_length=255, null=True, blank=True) + individual_name = models.CharField(max_length=255, null=True, blank=True) + individual_professional_title = models.CharField(max_length=255, null=True, blank=True) + individual_address_1 = models.TextField(null=True, blank=True) + individual_address_2 = models.TextField(null=True, blank=True) + individual_address_city = models.CharField(max_length=255, null=True, blank=True) + individual_address_state = models.CharField(max_length=255, null=True, blank=True) + individual_address_zip = models.CharField(max_length=255, null=True, blank=True) + individual_address_country = models.CharField(max_length=255, null=True, blank=True) + individual_phone = models.CharField(max_length=255, null=True, blank=True) + individual_fax = models.CharField(max_length=255, null=True, blank=True) + individual_email = models.CharField(max_length=255, null=True, blank=True) + corporation_place_of_business = models.CharField(max_length=255, null=True, blank=True) + corporation_contact_name = models.CharField(max_length=255, null=True, blank=True) + corporation_phone = models.CharField(max_length=255, null=True, blank=True) + corporation_fax = models.CharField(max_length=255, null=True, blank=True) + corporation_email = models.CharField(max_length=255, null=True, blank=True) + research_team_person_1 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_2 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_3 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_4 = models.CharField(max_length=1024, null=True, blank=True) + data_user_signature = models.CharField(max_length=255, null=True, blank=True) + data_user_name = models.CharField(max_length=255, null=True, blank=True) + data_user_title = models.CharField(max_length=255, null=True, blank=True) + data_user_address_1 = models.TextField(null=True, blank=True) + data_user_address_2 = models.TextField(null=True, blank=True) + data_user_address_city = models.CharField(max_length=255, null=True, blank=True) + data_user_address_state = models.CharField(max_length=255, null=True, blank=True) + data_user_address_zip = models.CharField(max_length=255, null=True, blank=True) + data_user_address_country = models.CharField(max_length=255, null=True, blank=True) + data_user_date = models.CharField(max_length=255, null=True, blank=True) + registrant_is = models.CharField(max_length=255, null=True, blank=True) + commercial_registrant_is = models.CharField(max_length=255, null=True, blank=True) + data_user_acknowledge = models.CharField(max_length=3, null=True, blank=True) + partners_name = models.CharField(max_length=255, null=True, blank=True) + partners_title = models.CharField(max_length=255, null=True, blank=True) + partners_address = models.TextField(null=True, blank=True) + partners_date = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP DUA signed agreement form fields' + verbose_name_plural = 'NLP DUA signed agreement form fields' diff --git a/app/static/institutionlogos/4ce_logo.png b/app/static/institutionlogos/4ce_logo.png new file mode 100644 index 0000000000000000000000000000000000000000..bf45440865b1e5b87f502953f88def6e802f32ba GIT binary patch literal 3413 zcmV-b4XW~qP) zYj9I#8pr=9_cWI_Nt>h%rAZs9r3JK5XbZ@tAPVieyRhJj7gih*bw$B(XV%dVx-;sC zEVAmZu&y&YxXwE4t}6)aD9dFXMdTu43)oU<3vJSrHVsYMG|AZywJt)=CCPbvQaHa) zOwY;lzAaDQ^Ssadf1X1j%~9u<6*;Hh`yK_*$$UPRr1*JyRF6+lsePWN z+2^lF#rR0zn)SeDttEe|*z=G`Ph?_GfDK#Koh!AL{JCP!Ln3`iTL;$UmK5YxEPh$+ zc}S!;>FU60_oB6brzn|8?1@OEH)-j>1-C!>j8bPV7keHO=}+1@ux`w>iKZJWpAvf> z5~G83b6|PFycJt0ic*R_5sA@3S~;*|&V!GsGp$m}OoBIQlLyw=id{JqZhcM!t?FylPFedgi)0(t#VCVeRZ&DPcBVqd|Q|zcM zw{zaNufd0{2iizX0RW(sQ>Yqe#`Iz{COI>aZBVn`AERlU>kQ&(vmXaecHx_rUcu*7 zJ>G_@8@c~BB-paE9=)N#lx(Dh1KTTqd7svjKc9qcTc#Qh);OV2$vN+>KOZ151?V+$ zEU9#$Zn7Pj8T`Y08AZWm)xl-e;pfxx(AE{g2Vb1Wu47(AV=0~xe@~%36L}UQ>jo>~ z!6*VLIxmn04y@PPtk#Of8%Wt-v#^kt^O9oW%{DwxQvjnj^?m6Wixy8Ta%1tdJZ${9 z3CGW+z9QRA+LUc1gQq8ys>lRD8f@b7YS(U(%P2Dm+jC2^G1X1PG$(qE9M3N+!P;84 zAkKR^W3w}`{+=?dn4J$Q1v@ed$jKDPfjhc`g4hvt1gyW5EfR=cfzDokulDKS+B$^_f8<{yq7b#EC=gq-AR4cCQ|qS03F>SLF|aO0vFDB z=ta3wGlqoiNd-nMm}DhkniZ9T!td@Jk35rx_h$y9F|_(Z@CCvMM`F+@Ww2(d;V^3< zm+|{{-#W>LCQl!>9c)i%zIM~de97W5szV6V_?nqUSNSRuwpDTpPb_kSBDJ)$@X&l0 z#^vfc@1toN2OGPw<8TL#H2X0Sx%&B^p_XH6kqLEE?3h%L$!T-tTqh1S^`OPae`T|Y zr2)Zu~2x3bN2Ua_#-@TavMI?M#J~JOpyBleH(kq zE;tay-g+PQ*85O3&V;qKZq|@UrI6u~1upz%Yd!Bd+q3xno%+KOwEFsZ+vURN7bbiF zL~!7O+n;s#pG@~fsiA8QG8Ftkke?PW-dRDLlfXkUvnhlqY{V?F07C)Z-TSLOmJB-I4 z#&6%NN8dn%^}e50bu12J~0?c2bLGiUh!uNWRzxglkF(YH^#k2(=;}H ze1@2=;1^ZQt3CkugM)Z}+bIl2Q~UC?UF^qS_MBtAr)r!T8YTCCYa$02ZJok}4?;6y z4(zB|wMK2UR*pQFWE1C{zxquOQkqRK zHe<$(*^Jk>>^Upg>KBiSBKESqj}CW0=&<{%rXI|jz?|o{=(!JY_~wAU0_AH-LEM4u zm3OWnr8LV>$?@oIg^brvwD|Gy5#d(9c;gpZnZK7bO=Iu3KEcO5de)C+8xJM@c`AVn zgr`Tilottzh7fwa&1$Py@(L;YE9Vv<#~5EzI~a{&!;U6F6i-aJ?K;RPdgH}(+7xv~-FsN+%b&?JmBVu;vDt-gSO z4op&M1^}yXb-~E6Y3dJ0@#@YdQbRKm>}Nk>HE7bWL)NvlfdhndV0pnTa!Ru)g;|(e zmcw}co&7E7=n+QYp9tMS_Qzt5apY_AxDaCnCbfYB1j++D=B!$y&LpEWQ!8ayv#^L! zEZ*303EK`{ATcy2UXCvf*FlFE@IAyh=Lulf;TA#0ywF8Xj+wFp5A`%7r!>27Rz7kW ziX@{k8XLAZB~?-`1g%}{kH+FWZdKSwLQ#N~XEn-MZvdyy`w7n|by5x?NcJ>y+jUqx zjae-Iudmv1x?Q~E+QyDP#8{+e1=(6y4CxaG4`m?ubty`B71}T7kck(V&0y@1~AEeDKB9VO1g#SCrwPMM7At!Uy7z`ry zVT7(eHtAGxD}UKabPhFjvsPr(P07VS4z&xa39c@-;Fc0*4RmkteB7Vg6A{=MT?pV%o&tKp(tP;hrQzrN9lZzkOQGGf*i%8=|WNt zoLzoLo!&-9X*SyUuA>i49qe)VgEdZMW(XtvTQJcIhne}wGcXv%;WIsPukj5byae;M zLX)@y6Ygm?8i^NgUXDqx!qY!1hJt6SC?;~V)L1p&$$HO$hOW4!`xF4{HByCfISD(k zlVDG?(aN3^ojBXczH2Wp%)+xvilI(YhhR0T@WNfCFtD_0rD>WmCvG;V8AVW6;^`S3 z83-W3O`NH8-i$0KnVx3xsBa6PZ$Q|J*vW;NP{`stk=H+f<~WWRS1i|XV54x)onuEWIM{n#&6hiIhXn)_|w17GG0$a11$7$ zb8vuMTLaz>Oj2oft?>HpW)$ZcP-JI6*P$p1)x{Q67h4dC#?aaoLRTP+!6^Mhd(~W% zhOuPoN@$wK@3%LilcfSgtX{y;7>!PXF*Pp-*14p&XYyen62sH)*W=X{6JXDq?KmZGQFV)p8jh+HtuLdASCRm*0-&_c=)ZavGbUh*ZpF3Vh6}p zV*ng6FiEA^^#F~bZj$>Ny0D}&4|U~s#;OuNoVpOe`=6c1fs^dwRcs$nkQa zkW-?xNTq3@(|@&!Pe|hCX>?Yrt!UXB-%4T~vSei_;jn06(W{W5mP1BS7>vdc2u0xS z9Y9-GKf-Zx#1j7}?h5RfyKa-@yyWQ#4x)#nu1ye9g>ELT}HMkRZjd1FId=mv084w`WR7kU|DXN;8Ski%e3Q rNsML`IVHEO4tdWU7keTS63OsC;ZN}~hMO5<00000NkvXXu0mjf$J~O` literal 0 HcmV?d00001 diff --git a/app/static/institutionlogos/4ce_logo@0.5x.png b/app/static/institutionlogos/4ce_logo@0.5x.png new file mode 100644 index 0000000000000000000000000000000000000000..9a4a3e235b3c321814bbc270458907fe74122c9a GIT binary patch literal 2539 zcmVX>UZ-(l$6LINXP>+AFpll9ow;`sg9PL+`C(?hd+t5| zbI$kuzyGyh6VbQ-rAD=}`R9^iJWwOdt38U(&qDYF_`vhKzV^?;C%|3vSqbt?K`=P@ ztXgVndt3=O91xNm->h-}+aRy`EN#-bB757;A=!+*@c&HiM%WaBdxu|c)I#wWgsk*! z#(4irxb74&IQXnujHbS&XvV`Jd>2epBqD}_5G&tVjz^|o<9Wf-s}Nvn3UN~*5z#PI z3CHnR$XhJrEwV)$+wmVu2ncBkihOPRtZX?HEvIJ8brYQH@4H<$V&6x|F~3+@5`|So04gMZGmh z{I+J3$A;S2)@)XNzmT_B#rK9#WWk+Xafa`0XGcdALt9%;Drk=D?ARJ*&-O;%_;8Xp zKbmH-u<^nfnoLhij4cUs!{2McvRz!ySMyv4!NbFHG}^siR*gpyV&mDaLReRLWUz(K z#!%h2Qe?sI{uEE$-$`2{RFz&F*TZo=gb=7pt0YT;j#P+)U+E+f)_CQgV`Mit9Z^%m z)YZD52cT@b%x6pf&wfp#1HIhcuNcwigd`{ZVkIHCySIVex2BLKpRKwELcorWDBs@G zNqcf-aTW_UpIuz!)Od!4oQ0-HY)yq2*cPWf8A6r>rY^H@U<*^}5^o%v#CEIqTQZ`f zsnz+{^T2Uk-<_E&TK_(R`ycs9R81tmCn@@!LE>8zItPYYiG?b=Wjb9VY$$|v-;Gb) zRC#QujgBR9J>bGjo>$%(^mSPeKY5_71er z*IFqgnS#xqk4^I6&L+Z5wR4{U2D;tl5!e+vV(JmfycM zN;0f+dsju0ccenx)f4B!Y<_t$6j>q}(~;JksEadM4xgN&>_c!+)U8kY&gk8m+3>t1 zDb0SdLckqeF&@0F5mgqruE*iiv%GtD4kv&lGc=i@-UJb&vJV{Bk3*W-^t7bt z_fXkjHIa&FJi4omRJ1aYi*p5DKQ_5x!Py9LQ|0E?$nxjKyu~}GW+_=$mtIiGqen&- zP~=hcr5rn*F5#|PR9%&6SVOR7yUb-u1eD`}P!GW(V)T4fHlj}oA#}fBRjz2>+q){q zyHIv`E7HWU;AqiNYr3#zfRWQ>%-tuF5<#&`~q}6w3~;9zMsZv4ueQ2}E%3-XG~|G>q3)yj1y5j%RrD<7rBE?ZUJX zidIDhT!E>{$g+T7HS(&^6p2K1eHkg1otoYSz~o$>vQLp+jhY14>y(+g%s209p=V1N zsCW#o9hqQiAy{Eri}|cY*>*9P8j|L?j-krrOV^i&byX&@MnRNqmqVY9b9Ovk^V(d| z@@+y^qb9-iIAs9aJEDALRXdz5*t91@>>q5cii;S^a#sh?9Mjo5&_c27aBe!s@rxN; z&trPA%tGE;Rvj%3I_*h=xvYN&m5iv|(G^FPB|Km*TjtXXiGWgD(O5C50OmK2M zgX=&#Z*g`aOZQU87dKVz>rZm#QnqHw5dwyKlYIAq?j;>uX;4m#Rme+3bu^_~K@=^A z#k~K*)r*af{qV3H-O{~Z*31JzoU$trqPqK#gg}+0b$^O1RmF)_ajVUsV%gz?&*zsL z6Il}M+1|+DO${|oqqc;>gFBmun+k?1Aqm0f6S-w`JQ>let+H-ailbT^ymOOwnNZ7$9ha9odxg_TaqqoSWb9!+y!U#eob$8?^) zzY9Z^IXbd|+0$h5#J%n8*t%k(kIv*ddTs&Ft0;&>)L5q=bXDQ5-bUXdH@#5ci>XXi z@vVh*1lQ}7t;W%jc}`qh^o?tbY5er@TWD>l=;V>99KZa_2$_OS$#ScNemZaQ$M+}b zY&6-L3L${*CX?qL?y1<7molVt7Md!vH5KB{?%1+j2^CxO^`nzqny(aEO{qwk|N|id03S<#;UQtPS5hcpe-dTjURKU*M_x zyJ$@q2q0!E+}EGvo|_Zcu7@PVx|5SB*u46|CEh-HdHH^tDw7DWCs+~!$>-$Rj!QaM zuJSbeP(^Ux8NFMJ2Xo5)ldv6+f1H{nTXYz{r;Y7fqsz{iBnh-N$K7^4#%2q=esqGr zeR{b{)ss&(cdzLq-+?*MpMnGBID|qtP^nPtS4Ta(><0WlfQoNf)k3F=PuCmlx|J z_zIC?$H8Acq{N$lBgx8oUA4)OBta~sEa%B+u!^q9mE)VYN#~Yc+h3&! zsfvW5Dc7`vT+bt4wChR{0-$gBXYEF+>({bo?hRt*+(2+h6{oD}psX2zoU$9J50aGH zHl&8)&mg4w|2g0We4$|3Puk^Nh%#5V|8L-$@h`8vji(c`m3IID002ovPDHLkV1ico B@f!dD literal 0 HcmV?d00001 diff --git a/requirements.txt b/requirements.txt index d8b1fb1e..1ad8fda3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,21 +14,21 @@ asgiref==3.6.0 \ # via # django # django-countries -awscli==1.27.98 \ - --hash=sha256:7d4165a1d0f16d36d08a2aef8d13f9f5de6cc3d9ebd02c045270269de9582c84 \ - --hash=sha256:f6edc2509644d79246287c066f52db00b8f29f4e3e5ee886b3c986cdd9248b5d +awscli==1.27.105 \ + --hash=sha256:6393acc01a369548341e658a3287612b091ff3e3507e363e3b8b826eb4c3a207 \ + --hash=sha256:c617ac3ea7ed6357cb058a384f650fd7b6c9008e4896787a60abbb19be854b7d # via -r requirements.in blessed==1.20.0 \ --hash=sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058 \ --hash=sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680 # via django-q -boto3==1.26.98 \ - --hash=sha256:983ec9e539431c29b5265e435b91af7c0d77a75809e173427798edb4ede1d69c \ - --hash=sha256:f35a42c6d0130a75e58485efa94383256d9b8c72c3a31ad872807873a8800363 +boto3==1.26.105 \ + --hash=sha256:2914776e0138530ec6464d0e2f05b4aa18e9212ac920c48472f8a93650feaed2 \ + --hash=sha256:f4951f8162905b96fd045e32853ba8cf707042faac846a23910817c508ef27d7 # via -r requirements.in -botocore==1.29.98 \ - --hash=sha256:ae906c1feb56063a38ffd2280232fa44d634057825470d3beed274925088cb42 \ - --hash=sha256:b74283ff71eb4e57edfa5cf6dc36d959b1eec618a2b1e5e781643184857dd1c4 +botocore==1.29.105 \ + --hash=sha256:06a2838daad3f346cba5460d0d3deb198225b556ff9ca729798d787fadbdebde \ + --hash=sha256:17c82391dfd6aaa8f96fbbb08cad2c2431ef3cda0ece89e6e6ba444c5eed45c2 # via # awscli # boto3 @@ -184,30 +184,30 @@ colorama==0.4.4 \ --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 # via awscli -cryptography==40.0.0 \ - --hash=sha256:14da8c26755ffa5c7863ffa5e8b87cb9596a21b6c34852cb19e0f48c226c64fb \ - --hash=sha256:168ded448fb5d82dfa911156ab8b13b1716de65bd50ff977f4657643f998fa05 \ - --hash=sha256:22e63fb48e2615cfab5a9c4bb457d35e7ae03ea8593996bfbe257e78244d12d0 \ - --hash=sha256:23c42c59c2b5b9ddc6a85b5c46b8fabc4d63a1714f4dbea4bf20d25690bf2365 \ - --hash=sha256:34f502619964210939bb7ee7cd5df53178534eb08d3526f941695a8f7aa0efe4 \ - --hash=sha256:43089be365c0ca4235c6e4e781f3bc125bc1fff576c9dd22cdfb585309b9bb9d \ - --hash=sha256:6b36e2864e04c82634879c7e7aad48824b1847fdb06b64cd410d2ec5e51d1b31 \ - --hash=sha256:7162ae4530958114ca2eee30a56eca46527def33493f622f059dc2e825fd0913 \ - --hash=sha256:71cb346b9dd1537102e7466a2d629385b01847f8d96cd7405f0e717d91cebc8e \ - --hash=sha256:722cfddae79684166840be2cbbae154f44a455519e644b60bf274a50ccb834db \ - --hash=sha256:754dc5ab648113dc54197f242db43234a04e4d61193fb5d3ebb42bd569dca571 \ - --hash=sha256:7cc9fc3ffcb766c313ed0515d77d0deabb4f36bdcff3a9f115c43e5ec611b82a \ - --hash=sha256:b05c9f25a1ea42e427085230815bbdebe15a53bb6163c4c06022e5630645046b \ - --hash=sha256:e5855a80c77565fe2464e88e0095764e25d8ddb2d24df2b1d31773e80be94435 \ - --hash=sha256:e917a07094217edeefe8f6ea960b45d7aab650b982e4209da078332cc9d3ac3a \ - --hash=sha256:f2c4134d29cdce0735c16abf48fa8435f001a7b0031e68dd9a9ee1c80a29374a \ - --hash=sha256:f421f6777592eb199ca8abac7c20b9ecef27c50ad63546e6c614b29771b46d0d \ - --hash=sha256:fafa997b9e6818db333ded4b379f5b7679b48bd88ac878428cea2a1aa6e79fd8 \ - --hash=sha256:fba36ec552794a06a07ac8bdc5ad83a587f6959d98547f373d401975d55c7c9e +cryptography==40.0.1 \ + --hash=sha256:0a4e3406cfed6b1f6d6e87ed243363652b2586b2d917b0609ca4f97072994405 \ + --hash=sha256:1e0af458515d5e4028aad75f3bb3fe7a31e46ad920648cd59b64d3da842e4356 \ + --hash=sha256:2803f2f8b1e95f614419926c7e6f55d828afc614ca5ed61543877ae668cc3472 \ + --hash=sha256:28d63d75bf7ae4045b10de5413fb1d6338616e79015999ad9cf6fc538f772d41 \ + --hash=sha256:32057d3d0ab7d4453778367ca43e99ddb711770477c4f072a51b3ca69602780a \ + --hash=sha256:3a4805a4ca729d65570a1b7cac84eac1e431085d40387b7d3bbaa47e39890b88 \ + --hash=sha256:63dac2d25c47f12a7b8aa60e528bfb3c51c5a6c5a9f7c86987909c6c79765554 \ + --hash=sha256:650883cc064297ef3676b1db1b7b1df6081794c4ada96fa457253c4cc40f97db \ + --hash=sha256:6f2bbd72f717ce33100e6467572abaedc61f1acb87b8d546001328d7f466b778 \ + --hash=sha256:7c872413353c70e0263a9368c4993710070e70ab3e5318d85510cc91cce77e7c \ + --hash=sha256:918cb89086c7d98b1b86b9fdb70c712e5a9325ba6f7d7cfb509e784e0cfc6917 \ + --hash=sha256:9618a87212cb5200500e304e43691111570e1f10ec3f35569fdfcd17e28fd797 \ + --hash=sha256:a805a7bce4a77d51696410005b3e85ae2839bad9aa38894afc0aa99d8e0c3160 \ + --hash=sha256:cc3a621076d824d75ab1e1e530e66e7e8564e357dd723f2533225d40fe35c60c \ + --hash=sha256:cd033d74067d8928ef00a6b1327c8ea0452523967ca4463666eeba65ca350d4c \ + --hash=sha256:cf91e428c51ef692b82ce786583e214f58392399cf65c341bc7301d096fa3ba2 \ + --hash=sha256:d36bbeb99704aabefdca5aee4eba04455d7a27ceabd16f3b3ba9bdcc31da86c4 \ + --hash=sha256:d8aa3609d337ad85e4eb9bb0f8bcf6e4409bfb86e706efa9a027912169e89122 \ + --hash=sha256:f5d7b79fa56bc29580faafc2ff736ce05ba31feaa9d4735048b0de7d9ceb2b94 # via django-dbmi-client -django==4.1.7 \ - --hash=sha256:44f714b81c5f190d9d2ddad01a532fe502fa01c4cb8faf1d081f4264ed15dcd8 \ - --hash=sha256:f2f431e75adc40039ace496ad3b9f17227022e8b11566f4b363da44c7e44761e +django==4.2 \ + --hash=sha256:ad33ed68db9398f5dfb33282704925bce044bef4261cd4fb59e4e7f9ae505a78 \ + --hash=sha256:c36e2ab12824e2ac36afa8b2515a70c53c7742f0d6eaefa7311ec379558db997 # via # -r requirements.in # django-bootstrap-datepicker-plus @@ -373,9 +373,9 @@ python-magic==0.4.27 \ --hash=sha256:c1ba14b08e4a5f5c31a302b7721239695b2f0f058d125bd5ce1ee36b9d9d3c3b \ --hash=sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3 # via -r requirements.in -pytz==2022.7.1 \ - --hash=sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0 \ - --hash=sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a +pytz==2023.3 \ + --hash=sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588 \ + --hash=sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb # via djangorestframework pyyaml==5.4.1 \ --hash=sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf \ From 36d4d1d045baf4b0c3abc69208b155a7bab395c2 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Tue, 4 Apr 2023 13:26:41 -0600 Subject: [PATCH 04/25] HYP-293 - Removed unecessary imports --- app/projects/admin.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/projects/admin.py b/app/projects/admin.py index 94a4fc9d..cea9fcf3 100644 --- a/app/projects/admin.py +++ b/app/projects/admin.py @@ -16,12 +16,6 @@ from projects.models import ChallengeTaskSubmission from projects.models import ChallengeTaskSubmissionDownload from projects.models import Bucket -from projects.models import NLPDUASignedAgreementFormFields -from projects.models import NLPWHYSignedAgreementFormFields -from projects.models import DUASignedAgreementFormFields -from projects.models import ROCSignedAgreementFormFields -from projects.models import MAYOSignedAgreementFormFields -from projects.models import MIMIC3SignedAgreementFormFields class GroupAdmin(admin.ModelAdmin): From 8a03150c5197d8df8bc19a753f93988879f5c641 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Tue, 4 Apr 2023 13:55:12 -0600 Subject: [PATCH 05/25] HYP-293 - Fixed Bucket validation bug --- app/projects/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/projects/models.py b/app/projects/models.py index d524df3d..5e753fbb 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -115,7 +115,7 @@ def __str__(self): def clean(self): # Check for multiple defaults - if self.objects.filter(default=True): + if self.default and Bucket.objects.filter(default=True): raise ValidationError('Only one bucket may be configured as the default at one time.') # Check bucket for needed permissions From fed0c03c83a311d6eaef66df92bb979839d18324 Mon Sep 17 00:00:00 2001 From: b32147 Date: Mon, 24 Apr 2023 12:07:20 +0000 Subject: [PATCH 06/25] fix(requirements): Updated Python requirements --- requirements.txt | 70 ++++++++++++++++++++++++------------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1ad8fda3..c5edc0f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,21 +14,21 @@ asgiref==3.6.0 \ # via # django # django-countries -awscli==1.27.105 \ - --hash=sha256:6393acc01a369548341e658a3287612b091ff3e3507e363e3b8b826eb4c3a207 \ - --hash=sha256:c617ac3ea7ed6357cb058a384f650fd7b6c9008e4896787a60abbb19be854b7d +awscli==1.27.118 \ + --hash=sha256:590fcdd7c49194b09e575e2e1594c5838a2320d9c8216eda15fdc659c45866fd \ + --hash=sha256:a41d6d97398cc0ef7a54f788a1909fe98a69694acb1b3ae2a74d1a551abf02e0 # via -r requirements.in blessed==1.20.0 \ --hash=sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058 \ --hash=sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680 # via django-q -boto3==1.26.105 \ - --hash=sha256:2914776e0138530ec6464d0e2f05b4aa18e9212ac920c48472f8a93650feaed2 \ - --hash=sha256:f4951f8162905b96fd045e32853ba8cf707042faac846a23910817c508ef27d7 +boto3==1.26.118 \ + --hash=sha256:1ff703152553f4d5fc9774071d114dbf06ec661eb1b29b6051f6b1f9d0c24873 \ + --hash=sha256:d0ed43228952b55c9f44d1c733f74656418c39c55dbe36bc37feeef6aa583ded # via -r requirements.in -botocore==1.29.105 \ - --hash=sha256:06a2838daad3f346cba5460d0d3deb198225b556ff9ca729798d787fadbdebde \ - --hash=sha256:17c82391dfd6aaa8f96fbbb08cad2c2431ef3cda0ece89e6e6ba444c5eed45c2 +botocore==1.29.118 \ + --hash=sha256:44cb088a73b02dd716c5c5715143a64d5f10388957285246e11f3cc893eebf9d \ + --hash=sha256:b51fc5d50cbc43edaf58b3ec4fa933b82755801c453bf8908c8d3e70ae1142c1 # via # awscli # boto3 @@ -184,26 +184,26 @@ colorama==0.4.4 \ --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b \ --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 # via awscli -cryptography==40.0.1 \ - --hash=sha256:0a4e3406cfed6b1f6d6e87ed243363652b2586b2d917b0609ca4f97072994405 \ - --hash=sha256:1e0af458515d5e4028aad75f3bb3fe7a31e46ad920648cd59b64d3da842e4356 \ - --hash=sha256:2803f2f8b1e95f614419926c7e6f55d828afc614ca5ed61543877ae668cc3472 \ - --hash=sha256:28d63d75bf7ae4045b10de5413fb1d6338616e79015999ad9cf6fc538f772d41 \ - --hash=sha256:32057d3d0ab7d4453778367ca43e99ddb711770477c4f072a51b3ca69602780a \ - --hash=sha256:3a4805a4ca729d65570a1b7cac84eac1e431085d40387b7d3bbaa47e39890b88 \ - --hash=sha256:63dac2d25c47f12a7b8aa60e528bfb3c51c5a6c5a9f7c86987909c6c79765554 \ - --hash=sha256:650883cc064297ef3676b1db1b7b1df6081794c4ada96fa457253c4cc40f97db \ - --hash=sha256:6f2bbd72f717ce33100e6467572abaedc61f1acb87b8d546001328d7f466b778 \ - --hash=sha256:7c872413353c70e0263a9368c4993710070e70ab3e5318d85510cc91cce77e7c \ - --hash=sha256:918cb89086c7d98b1b86b9fdb70c712e5a9325ba6f7d7cfb509e784e0cfc6917 \ - --hash=sha256:9618a87212cb5200500e304e43691111570e1f10ec3f35569fdfcd17e28fd797 \ - --hash=sha256:a805a7bce4a77d51696410005b3e85ae2839bad9aa38894afc0aa99d8e0c3160 \ - --hash=sha256:cc3a621076d824d75ab1e1e530e66e7e8564e357dd723f2533225d40fe35c60c \ - --hash=sha256:cd033d74067d8928ef00a6b1327c8ea0452523967ca4463666eeba65ca350d4c \ - --hash=sha256:cf91e428c51ef692b82ce786583e214f58392399cf65c341bc7301d096fa3ba2 \ - --hash=sha256:d36bbeb99704aabefdca5aee4eba04455d7a27ceabd16f3b3ba9bdcc31da86c4 \ - --hash=sha256:d8aa3609d337ad85e4eb9bb0f8bcf6e4409bfb86e706efa9a027912169e89122 \ - --hash=sha256:f5d7b79fa56bc29580faafc2ff736ce05ba31feaa9d4735048b0de7d9ceb2b94 +cryptography==40.0.2 \ + --hash=sha256:05dc219433b14046c476f6f09d7636b92a1c3e5808b9a6536adf4932b3b2c440 \ + --hash=sha256:0dcca15d3a19a66e63662dc8d30f8036b07be851a8680eda92d079868f106288 \ + --hash=sha256:142bae539ef28a1c76794cca7f49729e7c54423f615cfd9b0b1fa90ebe53244b \ + --hash=sha256:3daf9b114213f8ba460b829a02896789751626a2a4e7a43a28ee77c04b5e4958 \ + --hash=sha256:48f388d0d153350f378c7f7b41497a54ff1513c816bcbbcafe5b829e59b9ce5b \ + --hash=sha256:4df2af28d7bedc84fe45bd49bc35d710aede676e2a4cb7fc6d103a2adc8afe4d \ + --hash=sha256:4f01c9863da784558165f5d4d916093737a75203a5c5286fde60e503e4276c7a \ + --hash=sha256:7a38250f433cd41df7fcb763caa3ee9362777fdb4dc642b9a349721d2bf47404 \ + --hash=sha256:8f79b5ff5ad9d3218afb1e7e20ea74da5f76943ee5edb7f76e56ec5161ec782b \ + --hash=sha256:956ba8701b4ffe91ba59665ed170a2ebbdc6fc0e40de5f6059195d9f2b33ca0e \ + --hash=sha256:a04386fb7bc85fab9cd51b6308633a3c271e3d0d3eae917eebab2fac6219b6d2 \ + --hash=sha256:a95f4802d49faa6a674242e25bfeea6fc2acd915b5e5e29ac90a32b1139cae1c \ + --hash=sha256:adc0d980fd2760c9e5de537c28935cc32b9353baaf28e0814df417619c6c8c3b \ + --hash=sha256:aecbb1592b0188e030cb01f82d12556cf72e218280f621deed7d806afd2113f9 \ + --hash=sha256:b12794f01d4cacfbd3177b9042198f3af1c856eedd0a98f10f141385c809a14b \ + --hash=sha256:c0764e72b36a3dc065c155e5b22f93df465da9c39af65516fe04ed3c68c92636 \ + --hash=sha256:c33c0d32b8594fa647d2e01dbccc303478e16fdd7cf98652d5b3ed11aa5e5c99 \ + --hash=sha256:cbaba590180cba88cb99a5f76f90808a624f18b169b90a4abb40c1fd8c19420e \ + --hash=sha256:d5a1bd0e9e2031465761dfa920c16b0065ad77321d8a8c1f5ee331021fda65e9 # via django-dbmi-client django==4.2 \ --hash=sha256:ad33ed68db9398f5dfb33282704925bce044bef4261cd4fb59e4e7f9ae505a78 \ @@ -312,9 +312,9 @@ orderedmultidict==1.0.1 \ --hash=sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad \ --hash=sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3 # via furl -pyasn1==0.4.8 \ - --hash=sha256:39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d \ - --hash=sha256:aef77c9fb94a3ac588e87841208bdec464471d9871bd5050a287cc9a475cd0ba +pyasn1==0.5.0 \ + --hash=sha256:87a2121042a1ac9358cabcaf1d07680ff97ee6404333bacca15f76aa8ad01a57 \ + --hash=sha256:97b7290ca68e62a832558ec3976f15cbf911bf5d7c7039d8b861c2a0ece69fde # via rsa pycparser==2.21 \ --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ @@ -443,9 +443,9 @@ six==1.16.0 \ # furl # orderedmultidict # python-dateutil -sqlparse==0.4.3 \ - --hash=sha256:0323c0ec29cd52bceabc1b4d9d579e311f3e4961b98d174201d5622a23b85e34 \ - --hash=sha256:69ca804846bb114d2ec380e4360a8a340db83f0ccf3afceeb1404df028f57268 +sqlparse==0.4.4 \ + --hash=sha256:5430a4fe2ac7d0f93e66f1efc6e1338a41884b7ddf2a350cedd20ccc4d9d28f3 \ + --hash=sha256:d446183e84b8349fa3061f0fe7f06ca94ba65b426946ffebe6e3e8295332420c # via django typing-extensions==4.5.0 \ --hash=sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb \ From e6de2800325da9346a82663fdb156e1c49f56816 Mon Sep 17 00:00:00 2001 From: b32147 Date: Mon, 1 May 2023 12:07:08 +0000 Subject: [PATCH 07/25] fix(requirements): Updated Python requirements --- requirements.txt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/requirements.txt b/requirements.txt index c5edc0f6..14013a0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,21 +14,21 @@ asgiref==3.6.0 \ # via # django # django-countries -awscli==1.27.118 \ - --hash=sha256:590fcdd7c49194b09e575e2e1594c5838a2320d9c8216eda15fdc659c45866fd \ - --hash=sha256:a41d6d97398cc0ef7a54f788a1909fe98a69694acb1b3ae2a74d1a551abf02e0 +awscli==1.27.123 \ + --hash=sha256:0eb798ba1f733eb6f14cf7bfc18196cf083fc5b32f3bc434f436914100213855 \ + --hash=sha256:b5458e90d549454311046f0d6d814892872e5abdfe2a90309c7589588cf8c238 # via -r requirements.in blessed==1.20.0 \ --hash=sha256:0c542922586a265e699188e52d5f5ac5ec0dd517e5a1041d90d2bbf23f906058 \ --hash=sha256:2cdd67f8746e048f00df47a2880f4d6acbcdb399031b604e34ba8f71d5787680 # via django-q -boto3==1.26.118 \ - --hash=sha256:1ff703152553f4d5fc9774071d114dbf06ec661eb1b29b6051f6b1f9d0c24873 \ - --hash=sha256:d0ed43228952b55c9f44d1c733f74656418c39c55dbe36bc37feeef6aa583ded +boto3==1.26.123 \ + --hash=sha256:2ccb1827dc9d654970375b9bbe1bd77eb6a50ec6fa76cf227a34b4bff144814f \ + --hash=sha256:51722a3a791108236c8ce05e0ced030b13c08276001132d4153d29353038f8cc # via -r requirements.in -botocore==1.29.118 \ - --hash=sha256:44cb088a73b02dd716c5c5715143a64d5f10388957285246e11f3cc893eebf9d \ - --hash=sha256:b51fc5d50cbc43edaf58b3ec4fa933b82755801c453bf8908c8d3e70ae1142c1 +botocore==1.29.123 \ + --hash=sha256:88f743e1db5bb65e3770af674400642f1d09fca159a1508bdbb8f8cc691589ca \ + --hash=sha256:b69cdf1c6f451ab1314a363bf97753dfbfe1da93d33853ed241edbdcb806867d # via # awscli # boto3 @@ -418,9 +418,9 @@ redis==3.5.3 \ --hash=sha256:0e7e0cfca8660dea8b7d5cd8c4f6c5e29e11f31158c0b0ae91a397f00e5a05a2 \ --hash=sha256:432b788c4530cfe16d8d943a09d40ca6c16149727e4afe8c2c9d5580c59d9f24 # via django-q -requests==2.28.2 \ - --hash=sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa \ - --hash=sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf +requests==2.29.0 \ + --hash=sha256:e8f3c9be120d3333921d213eef078af392fba3933ab7ed2d1cba3b56f2568c3b \ + --hash=sha256:f2e34a75f4749019bb0e3effb66683630e4ffeaf75819fb51bebef1bf5aef059 # via # -r requirements.in # django-dbmi-client From 2f5a098415e617988628385f5e62a5290002b965 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Thu, 4 May 2023 17:06:05 -0600 Subject: [PATCH 08/25] DBMISVC-HOTFIX-050423 - Reworked migration due to Django issue --- ...articipant_created_participant_modified.py | 27 ++--------- ...articipant_created_participant_modified.py | 47 +++++++++++++++++++ 2 files changed, 51 insertions(+), 23 deletions(-) create mode 100644 app/projects/migrations/0100_participant_created_participant_modified.py diff --git a/app/projects/migrations/0097_participant_created_participant_modified.py b/app/projects/migrations/0097_participant_created_participant_modified.py index e09b65ee..69502161 100644 --- a/app/projects/migrations/0097_participant_created_participant_modified.py +++ b/app/projects/migrations/0097_participant_created_participant_modified.py @@ -11,29 +11,10 @@ def migrate_participants_model(apps, schema_editor): been created. This is calculated by fetching the date of the last signed SignedAgreementForm relevant to the DataProjects. """ - for participant in Participant.objects.all(): - - # Fetch signed agreement forms - signed_agreement_form = SignedAgreementForm.objects.filter(user=participant.user, project=participant.project).last() - if signed_agreement_form: - - # Set the dates - participant.created = signed_agreement_form.date_signed - participant.modified = signed_agreement_form.date_signed - - # Do the update - Participant.objects.filter(pk=participant.pk).update( - created=signed_agreement_form.date_signed, - modified=signed_agreement_form.date_signed - ) - - else: - - # Do the update - Participant.objects.filter(pk=participant.pk).update( - created="2018-02-20T00:00:00Z", - modified="2018-02-20T00:00:00Z", - ) + # Do nothing due to issue with Django accessing properties not added + # until later migration + # This migration was moved to 0100 + pass class Migration(migrations.Migration): diff --git a/app/projects/migrations/0100_participant_created_participant_modified.py b/app/projects/migrations/0100_participant_created_participant_modified.py new file mode 100644 index 00000000..0872fea7 --- /dev/null +++ b/app/projects/migrations/0100_participant_created_participant_modified.py @@ -0,0 +1,47 @@ +# Generated by Django 4.1.6 on 2023-02-14 17:07 + +from django.db import migrations, models + +from projects.models import Participant, SignedAgreementForm + + +def migrate_participants_model(apps, schema_editor): + """ + Attempts to set a roughly accurate date of when each object would have + been created. This is calculated by fetching the date of the last + signed SignedAgreementForm relevant to the DataProjects. + """ + for participant in Participant.objects.all(): + + # Fetch signed agreement forms + signed_agreement_form = SignedAgreementForm.objects.filter(user=participant.user, project=participant.project).last() + if signed_agreement_form: + + # Set the dates + participant.created = signed_agreement_form.date_signed + participant.modified = signed_agreement_form.date_signed + + # Do the update + Participant.objects.filter(pk=participant.pk).update( + created=signed_agreement_form.date_signed, + modified=signed_agreement_form.date_signed + ) + + else: + + # Do the update + Participant.objects.filter(pk=participant.pk).update( + created="2018-02-20T00:00:00Z", + modified="2018-02-20T00:00:00Z", + ) + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0099_bucket_dataproject_bucket_hostedfile_bucket_and_more'), + ] + + operations = [ + migrations.RunPython(migrate_participants_model), + ] From 05360b8f1d74ba9a7213984f4cd658b305fc2e40 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Fri, 5 May 2023 10:40:00 -0600 Subject: [PATCH 09/25] HYP-288 - Fixed bug when searching pending participant table --- app/hypatio/views.py | 2 +- app/manage/views.py | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/app/hypatio/views.py b/app/hypatio/views.py index ebbd2f30..1debe70e 100644 --- a/app/hypatio/views.py +++ b/app/hypatio/views.py @@ -30,7 +30,7 @@ def navigation_context(request): def group_context(): # Check for an active project and determine its group - groups = Group.objects.all() + groups = Group.objects.filter(dataproject__isnull=False).distinct() active_group = None project = DataProject.objects.filter(project_key=os.path.basename(os.path.normpath(request.path))).first() if project: diff --git a/app/manage/views.py b/app/manage/views.py index 38ebceaa..1848eef5 100644 --- a/app/manage/views.py +++ b/app/manage/views.py @@ -469,6 +469,11 @@ def get(self, request, project_key, *args, **kwargs): ) ) + # Add search if necessary + if search: + participants_waiting_access = participants_waiting_access.filter(user__email__icontains=search) + participants_awaiting_approval = participants_awaiting_approval.filter(user__email__icontains=search) + # We only want distinct Participants belonging to the users query # Django won't sort on a related field after this union so we annotate each queryset with the user's email to sort on query_set = participants_waiting_access.annotate(email=F("user__email")) \ @@ -477,8 +482,8 @@ def get(self, request, project_key, *args, **kwargs): # Setup paginator paginator = Paginator( - query_set.filter(user__email__icontains=search) if search else query_set, - length + query_set, + length, ) # Determine page index (1-index) from DT parameters From 7dd93db45940af4db0556440bf1bf83ed8ab4766 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Mon, 8 May 2023 17:31:41 -0600 Subject: [PATCH 10/25] HYP-HOTFIX-050823 - Fixed case-sensitivity bug for S3 keys --- app/projects/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/projects/models.py b/app/projects/models.py index 5e753fbb..da4825a1 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -206,9 +206,9 @@ def split_uri(cls, uri): try: # Separate URI pattern = r"(\w+):\/\/([^:\/\/]+?)\/(.+)" - provider, bucket, key = re.match(pattern, uri.lower()).groups() + provider, bucket, key = re.match(pattern, uri).groups() - return Bucket.Provider(provider), bucket, key + return Bucket.Provider(provider.lower()), bucket, key except ValueError: raise ValueError(f"Unsupported bucket provider: '{provider}'") From 6efe1c15fafc600a1210bcc07b6f3f67d55919a6 Mon Sep 17 00:00:00 2001 From: Bryan Larson Date: Fri, 19 May 2023 15:13:28 -0600 Subject: [PATCH 11/25] HYP-296 - Added HTML field for instructions on ChallengeTask --- ...01_challengetask_submission_instructions.py | 18 ++++++++++++++++++ app/projects/models.py | 3 +++ .../projects/participate/complete-tasks.html | 6 ++++++ 3 files changed, 27 insertions(+) create mode 100755 app/projects/migrations/0101_challengetask_submission_instructions.py diff --git a/app/projects/migrations/0101_challengetask_submission_instructions.py b/app/projects/migrations/0101_challengetask_submission_instructions.py new file mode 100755 index 00000000..5e649639 --- /dev/null +++ b/app/projects/migrations/0101_challengetask_submission_instructions.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2 on 2023-05-19 21:00 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0100_participant_created_participant_modified'), + ] + + operations = [ + migrations.AddField( + model_name='challengetask', + name='submission_instructions', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/app/projects/models.py b/app/projects/models.py index da4825a1..035ba540 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -600,6 +600,9 @@ class ChallengeTask(models.Model): # Optional path to an html file that contains a form that should be completed when uploading a task solution submission_form_file_path = models.CharField(max_length=300, blank=True, null=True) + # Optional HTML content to be displayed along with submission upload for instructions on upload preparation + submission_instructions = models.TextField(blank=True, null=True) + # If blank, allow infinite submissions max_submissions = models.IntegerField(default=1, blank=True, null=True, help_text="Leave blank if you want there to be no cap.") diff --git a/app/templates/projects/participate/complete-tasks.html b/app/templates/projects/participate/complete-tasks.html index c14c20c2..b3e480ab 100644 --- a/app/templates/projects/participate/complete-tasks.html +++ b/app/templates/projects/participate/complete-tasks.html @@ -40,6 +40,12 @@

      {% endif %} + {% if task_detail.task.submission_instructions %} + + {% endif %} +
      Date: Fri, 19 May 2023 15:34:33 -0600 Subject: [PATCH 12/25] HYP-296 - Fixed element ID bug --- app/templates/projects/participate/complete-tasks.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/templates/projects/participate/complete-tasks.html b/app/templates/projects/participate/complete-tasks.html index b3e480ab..2e77436e 100644 --- a/app/templates/projects/participate/complete-tasks.html +++ b/app/templates/projects/participate/complete-tasks.html @@ -41,7 +41,7 @@

      {% endif %} {% if task_detail.task.submission_instructions %} -