diff --git a/halodrops.cfg b/halodrops.cfg index a5358f7..fb84438 100644 --- a/halodrops.cfg +++ b/halodrops.cfg @@ -1,11 +1,6 @@ [MANDATORY] data_directory = ./example_data -[helper.paths.Flight.__init__] -path_structure = {platform}/Level_0/{flight} - -[helper.paths.Platform.__init__] -path_structure = {platform}/Level_0 - -[processor.Sonde.add_path_structure] -path_structure = {platform}/Level_0/{flight} +[OPTIONAL] +path_to_flight_ids = {platform}/Level_0 +path_to_l0_files = {platform}/Level_0/{flight_id} diff --git a/src/halodrops/helper/__init__.py b/src/halodrops/helper/__init__.py index a35de4b..8a538f3 100644 --- a/src/halodrops/helper/__init__.py +++ b/src/halodrops/helper/__init__.py @@ -93,6 +93,10 @@ _FillValue=np.finfo("float32").max, ) + +path_to_flight_ids = "{platform}/Level_0" +path_to_l0_files = "{platform}/Level_0/{flight_id}" + l2_flight_attributes_map = { "True Air Speed (m/s)": "true_air_speed_(ms-1)", "Ground Speed (m/s)": "ground_speed_(ms-1)", @@ -107,9 +111,7 @@ } -l2_filename_template = ( - "HALO-(AC)3_{platform}_{launch_time}_{flight_id}_{serial_id}_Level_2.nc" -) +l2_filename_template = "{platform}_{launch_time}_{flight_id}_{serial_id}_Level_2.nc" def get_bool(s): diff --git a/src/halodrops/helper/paths.py b/src/halodrops/helper/paths.py index c632141..cf0a7c4 100644 --- a/src/halodrops/helper/paths.py +++ b/src/halodrops/helper/paths.py @@ -6,6 +6,7 @@ from halodrops.helper import rawreader as rr from halodrops.processor import Sonde +from halodrops.helper import path_to_flight_ids, path_to_l0_files # create logger module_logger = logging.getLogger("halodrops.helper.paths") @@ -24,7 +25,7 @@ def __init__( data_directory, platform_id, platform_directory_name=None, - path_structure="{platform}/Level_0", + path_structure=path_to_flight_ids, ) -> None: self.platform_id = platform_id self.platform_directory_name = platform_directory_name @@ -43,7 +44,7 @@ def get_flight_ids(self): flight_ids = [] dir_with_flights = self.path_structure.format(platform=platform_dir) - print(dir_with_flights) + for flight_id in os.listdir(dir_with_flights): if os.path.isdir(os.path.join(dir_with_flights, flight_id)): flight_ids.append(flight_id) @@ -63,7 +64,7 @@ def __init__( data_directory, flight_id, platform_id, - path_structure="{platform}/Level_0/{flight}", + path_structure=path_to_l0_files, ): """Creates an instance of Paths object for a given flight @@ -100,7 +101,7 @@ def __init__( flight_dir = os.path.join( self.data_directory, self.path_structure.format( - platform=self.platform_id, flight=self.flight_id + platform=self.platform_id, flight_id=self.flight_id ), ) self.flight_idpath = flight_dir diff --git a/src/halodrops/pipeline.py b/src/halodrops/pipeline.py index dd3f431..049f5af 100644 --- a/src/halodrops/pipeline.py +++ b/src/halodrops/pipeline.py @@ -1,4 +1,5 @@ from .helper.paths import Platform, Flight +from .helper.__init__ import path_to_flight_ids, path_to_l0_files from .processor import Sonde import configparser import inspect @@ -160,14 +161,17 @@ def get_platforms(config): """ data_directory = config.get("MANDATORY", "data_directory") - if config.has_option("MANDATORY", "platforms"): - if not config.has_option("MANDATORY", "platform_directory_names"): + path_structure = config.get( + "OPTIONAL", "path_to_flight_ids", fallback=path_to_flight_ids + ) + if config.has_option("OPTIONAL", "platforms"): + if not config.has_option("OPTIONAL", "platform_directory_names"): raise ValueError( "platform_directory_names must be provided in the config file when platforms is specified" ) - platforms = config.get("MANDATORY", "platforms").split(",") + platforms = config.get("OPTIONAL", "platforms").split(",") platform_directory_names = config.get( - "MANDATORY", "platform_directory_names" + "OPTIONAL", "platform_directory_names" ).split(",") platforms = dict(zip(platforms, platform_directory_names)) for directory_name in platform_directory_names: @@ -181,6 +185,7 @@ def get_platforms(config): data_directory=data_directory, platform_id=platform, platform_directory_name=platform_directory_name, + path_structure=path_structure, ) else: platforms = [ @@ -191,7 +196,9 @@ def get_platforms(config): platform_objects = {} for platform in platforms: platform_objects[platform] = Platform( - data_directory=data_directory, platform_id=platform + data_directory=data_directory, + platform_id=platform, + path_structure=path_structure, ) return platform_objects @@ -213,7 +220,11 @@ def create_and_populate_flight_object( A Flight object. """ output = {} + platform_objects = get_platforms(config) + path_structure = config.get( + "OPTIONAL", "path_to_l0_files", fallback=path_to_l0_files + ) output["platforms"] = platform_objects output["sondes"] = {} for platform in platform_objects: @@ -222,6 +233,7 @@ def create_and_populate_flight_object( platform_objects[platform].data_directory, flight_id, platform, + path_structure=path_structure, ) output["sondes"].update(flight.populate_sonde_instances()) diff --git a/src/halodrops/processor.py b/src/halodrops/processor.py index f70fc60..ae1f578 100644 --- a/src/halodrops/processor.py +++ b/src/halodrops/processor.py @@ -1,6 +1,6 @@ import ast from dataclasses import dataclass, field, KW_ONLY -import datetime +from datetime import datetime from typing import Any, Optional, List import os import subprocess @@ -779,7 +779,6 @@ def get_other_global_attributes(self): # "doi": f"{halodrops.data_doi}", # "created with": f"pipeline.py doi:{halodrops.software_doi}", "Conventions": "CF-1.8", - "campaign_id": "HALO-(AC)3", "platform_id": self.platform_id, # "instrument_id": "Vaisala RD-41", "product_id": "Level-2", @@ -803,7 +802,7 @@ def get_other_global_attributes(self): "author_email": "g.george@tudelft.nl", "featureType": "trajectory", # "reference": halodrops.reference_study, - "creation_time": str(datetime.datetime.utcnow()) + " UTC", + "creation_time": str(datetime.utcnow()) + " UTC", } for attr in dir(self): @@ -884,7 +883,9 @@ def add_compression_and_encoding_properties( return self - def get_l2_filename(self, l2_filename: str = None): + def get_l2_filename( + self, l2_filename: str = None, l2_filename_template: str = None + ): """ Gets the L2 filename from the template provided. @@ -898,13 +899,26 @@ def get_l2_filename(self, l2_filename: str = None): self : object Returns the sonde object with the L2 filename added as an attribute. """ + if l2_filename is None: - l2_filename = hh.l2_filename_template.format( - platform=self.platform_id, - serial_id=self.serial_id, - flight_id=self.flight_id, - launch_time=self.launch_time, - ) + if l2_filename_template: + l2_filename = l2_filename_template.format( + platform=self.platform_id, + serial_id=self.serial_id, + flight_id=self.flight_id, + launch_time=self.launch_time.astype(datetime).strftime( + "%Y-%m-%d_%H-%M" + ), + ) + else: + l2_filename = hh.l2_filename_template.format( + platform=self.platform_id, + serial_id=self.serial_id, + flight_id=self.flight_id, + launch_time=self.launch_time.astype(datetime).strftime( + "%Y-%m-%d_%H-%M" + ), + ) object.__setattr__(self, "l2_filename", l2_filename) @@ -950,7 +964,7 @@ def add_l2_ds(self, l2_dir: str = None): Returns the sonde object with the L2 dataset added as an attribute. """ if l2_dir is None: - self.l2_dir + l2_dir = self.l2_dir object.__setattr__( self, "l2_ds", xr.open_dataset(os.path.join(l2_dir, self.l2_filename)) diff --git a/tests/test_paths.py b/tests/test_paths.py index 273fd9b..6a7bdeb 100644 --- a/tests/test_paths.py +++ b/tests/test_paths.py @@ -5,7 +5,7 @@ main_data_directory = "./example_data" platform_id = "HALO" flightdate = "20200119" -path_structure = "{platform}/Level_0/{flight}" +path_structure = "{platform}/Level_0/{flight_id}" platform_path_structure = "{platform}/Level_0" l1_path = os.path.join(main_data_directory, platform_id, "Level_1", flightdate)