From 57cc8e770bdb913c113ea9e9b9ddbb36d23e3083 Mon Sep 17 00:00:00 2001 From: Chris Seal Date: Wed, 5 Jun 2024 17:21:08 +1200 Subject: [PATCH 1/3] Add programmatic creation of dataset directories if not handed off --- tardis/tardis_portal/api.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tardis/tardis_portal/api.py b/tardis/tardis_portal/api.py index 95b16b729..23d59a709 100644 --- a/tardis/tardis_portal/api.py +++ b/tardis/tardis_portal/api.py @@ -1050,9 +1050,9 @@ def dehydrate(self, bundle): if bundle.data["identifiers"] == []: bundle.data.pop("identifiers") if "tardis.apps.dataclassification" in settings.INSTALLED_APPS: - bundle.data[ - "classification" - ] = bundle.obj.data_classification.classification + bundle.data["classification"] = ( + bundle.obj.data_classification.classification + ) if settings.ONLY_EXPERIMENT_ACLS: dataset_count = exp.datasets.all().count() @@ -1361,9 +1361,9 @@ def dehydrate(self, bundle): if bundle.data["identifiers"] == []: bundle.data.pop("identifiers") if "tardis.apps.dataclassification" in settings.INSTALLED_APPS: - bundle.data[ - "classification" - ] = bundle.obj.data_classification.classification + bundle.data["classification"] = ( + bundle.obj.data_classification.classification + ) return bundle def prepend_urls(self): @@ -1651,6 +1651,10 @@ def obj_create(self, bundle, **kwargs): # pylint: disable=R1702 classification = bundle.data.pop("classification") bundle = super().obj_create(bundle, **kwargs) dataset = bundle.obj + if not dataset.directory: + # Create a dataset directory + dataset.directory = f"ds-{dataset.id}/data/" + dataset.save() # After the obj has been created if ( "tardis.apps.identifiers" in settings.INSTALLED_APPS From 1b8c4107066cdb40bf89f75a4b59abdaba51c4d0 Mon Sep 17 00:00:00 2001 From: Chris Seal Date: Thu, 6 Jun 2024 13:35:21 +1200 Subject: [PATCH 2/3] Add task to automatically update dataset directory field --- tardis/tardis_portal/tasks.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tardis/tardis_portal/tasks.py b/tardis/tardis_portal/tasks.py index 351dee88e..462ee85ad 100644 --- a/tardis/tardis_portal/tasks.py +++ b/tardis/tardis_portal/tasks.py @@ -315,4 +315,16 @@ def get_param_names(schema, metadata): dfp.save() else: dfp.string_value = metadata[pname.name] + dfp.save() + + +@tardis_app.task(name="tardis_portal.dataset.build_directory", ignore_result=True) +def dataset_build_directory(**kwargs): + from tardis.tardis_portal.models.dataset import Dataset + + datasets = Dataset.objects.filter(directory__isnull=True, directory__exact="") + for dataset in datasets: + primary_key = dataset.pk + dataset.directory = f"ds-{primary_key}/data/" + dataset.save() From 5cfb1421d1c9f3d6101766058b7ecc5640a64e47 Mon Sep 17 00:00:00 2001 From: Chris Seal Date: Thu, 6 Jun 2024 14:43:38 +1200 Subject: [PATCH 3/3] Add logging to dataset directory builder --- tardis/tardis_portal/tasks.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tardis/tardis_portal/tasks.py b/tardis/tardis_portal/tasks.py index 462ee85ad..af6fae546 100644 --- a/tardis/tardis_portal/tasks.py +++ b/tardis/tardis_portal/tasks.py @@ -320,11 +320,14 @@ def get_param_names(schema, metadata): @tardis_app.task(name="tardis_portal.dataset.build_directory", ignore_result=True) -def dataset_build_directory(**kwargs): +def dataset_build_directory(): from tardis.tardis_portal.models.dataset import Dataset - datasets = Dataset.objects.filter(directory__isnull=True, directory__exact="") + datasets = Dataset.objects.filter(directory__isnull=True) + logger.info("Processing datasets for directories") + logger.info(f"Found: {datasets}") for dataset in datasets: primary_key = dataset.pk + logger.info(f"Processing dataset: {primary_key}") dataset.directory = f"ds-{primary_key}/data/" dataset.save()