Merge pull request #135 from hgloeckner/fix_data_structure

Fix data structure

halodrops.cfg

@@ -1,11 +1,6 @@
 [MANDATORY]
 data_directory = ./example_data
 
-[helper.paths.Flight.__init__]
-path_structure = {platform}/Level_0/{flight}
-
-[helper.paths.Platform.__init__]
-path_structure = {platform}/Level_0
-
-[processor.Sonde.add_path_structure]
-path_structure = {platform}/Level_0/{flight}
+[OPTIONAL]
+path_to_flight_ids = {platform}/Level_0
+path_to_l0_files = {platform}/Level_0/{flight_id}

src/halodrops/helper/__init__.py

@@ -93,6 +93,10 @@
     _FillValue=np.finfo("float32").max,
 )
 
+
+path_to_flight_ids = "{platform}/Level_0"
+path_to_l0_files = "{platform}/Level_0/{flight_id}"
+
 l2_flight_attributes_map = {
     "True Air Speed (m/s)": "true_air_speed_(ms-1)",
     "Ground Speed (m/s)": "ground_speed_(ms-1)",
@@ -107,9 +111,7 @@
 }
 
 
-l2_filename_template = (
-    "HALO-(AC)3_{platform}_{launch_time}_{flight_id}_{serial_id}_Level_2.nc"
-)
+l2_filename_template = "{platform}_{launch_time}_{flight_id}_{serial_id}_Level_2.nc"
 
 
 def get_bool(s):

src/halodrops/helper/paths.py

@@ -6,6 +6,7 @@
 
 from halodrops.helper import rawreader as rr
 from halodrops.processor import Sonde
+from halodrops.helper import path_to_flight_ids, path_to_l0_files
 
 # create logger
 module_logger = logging.getLogger("halodrops.helper.paths")
@@ -24,7 +25,7 @@ def __init__(
         data_directory,
         platform_id,
         platform_directory_name=None,
-        path_structure="{platform}/Level_0",
+        path_structure=path_to_flight_ids,
     ) -> None:
         self.platform_id = platform_id
         self.platform_directory_name = platform_directory_name
@@ -43,7 +44,7 @@ def get_flight_ids(self):
         flight_ids = []
 
         dir_with_flights = self.path_structure.format(platform=platform_dir)
-        print(dir_with_flights)
+
         for flight_id in os.listdir(dir_with_flights):
             if os.path.isdir(os.path.join(dir_with_flights, flight_id)):
                 flight_ids.append(flight_id)
@@ -63,7 +64,7 @@ def __init__(
         data_directory,
         flight_id,
         platform_id,
-        path_structure="{platform}/Level_0/{flight}",
+        path_structure=path_to_l0_files,
     ):
         """Creates an instance of Paths object for a given flight
 
@@ -100,7 +101,7 @@ def __init__(
         flight_dir = os.path.join(
             self.data_directory,
             self.path_structure.format(
-                platform=self.platform_id, flight=self.flight_id
+                platform=self.platform_id, flight_id=self.flight_id
             ),
         )
         self.flight_idpath = flight_dir

src/halodrops/pipeline.py

@@ -1,4 +1,5 @@
 from .helper.paths import Platform, Flight
+from .helper.__init__ import path_to_flight_ids, path_to_l0_files
 from .processor import Sonde
 import configparser
 import inspect
@@ -160,14 +161,17 @@ def get_platforms(config):
 
     """
     data_directory = config.get("MANDATORY", "data_directory")
-    if config.has_option("MANDATORY", "platforms"):
-        if not config.has_option("MANDATORY", "platform_directory_names"):
+    path_structure = config.get(
+        "OPTIONAL", "path_to_flight_ids", fallback=path_to_flight_ids
+    )
+    if config.has_option("OPTIONAL", "platforms"):
+        if not config.has_option("OPTIONAL", "platform_directory_names"):
             raise ValueError(
                 "platform_directory_names must be provided in the config file when platforms is specified"
             )
-        platforms = config.get("MANDATORY", "platforms").split(",")
+        platforms = config.get("OPTIONAL", "platforms").split(",")
         platform_directory_names = config.get(
-            "MANDATORY", "platform_directory_names"
+            "OPTIONAL", "platform_directory_names"
         ).split(",")
         platforms = dict(zip(platforms, platform_directory_names))
         for directory_name in platform_directory_names:
@@ -181,6 +185,7 @@ def get_platforms(config):
                 data_directory=data_directory,
                 platform_id=platform,
                 platform_directory_name=platform_directory_name,
+                path_structure=path_structure,
             )
     else:
         platforms = [
@@ -191,7 +196,9 @@ def get_platforms(config):
         platform_objects = {}
         for platform in platforms:
             platform_objects[platform] = Platform(
-                data_directory=data_directory, platform_id=platform
+                data_directory=data_directory,
+                platform_id=platform,
+                path_structure=path_structure,
             )
     return platform_objects
 
@@ -213,7 +220,11 @@ def create_and_populate_flight_object(
         A Flight object.
     """
     output = {}
+
     platform_objects = get_platforms(config)
+    path_structure = config.get(
+        "OPTIONAL", "path_to_l0_files", fallback=path_to_l0_files
+    )
     output["platforms"] = platform_objects
     output["sondes"] = {}
     for platform in platform_objects:
@@ -222,6 +233,7 @@ def create_and_populate_flight_object(
                 platform_objects[platform].data_directory,
                 flight_id,
                 platform,
+                path_structure=path_structure,
             )
 
             output["sondes"].update(flight.populate_sonde_instances())

src/halodrops/processor.py

@@ -1,6 +1,6 @@
 import ast
 from dataclasses import dataclass, field, KW_ONLY
-import datetime
+from datetime import datetime
 from typing import Any, Optional, List
 import os
 import subprocess
@@ -779,7 +779,6 @@ def get_other_global_attributes(self):
             # "doi": f"{halodrops.data_doi}",
             # "created with": f"pipeline.py doi:{halodrops.software_doi}",
             "Conventions": "CF-1.8",
-            "campaign_id": "HALO-(AC)3",
             "platform_id": self.platform_id,
             # "instrument_id": "Vaisala RD-41",
             "product_id": "Level-2",
@@ -803,7 +802,7 @@ def get_other_global_attributes(self):
             "author_email": "g.george@tudelft.nl",
             "featureType": "trajectory",
             # "reference": halodrops.reference_study,
-            "creation_time": str(datetime.datetime.utcnow()) + " UTC",
+            "creation_time": str(datetime.utcnow()) + " UTC",
         }
 
         for attr in dir(self):
@@ -884,7 +883,9 @@ def add_compression_and_encoding_properties(
 
         return self
 
-    def get_l2_filename(self, l2_filename: str = None):
+    def get_l2_filename(
+        self, l2_filename: str = None, l2_filename_template: str = None
+    ):
         """
         Gets the L2 filename from the template provided.
 
@@ -898,13 +899,26 @@ def get_l2_filename(self, l2_filename: str = None):
         self : object
             Returns the sonde object with the L2 filename added as an attribute.
         """
+
         if l2_filename is None:
-            l2_filename = hh.l2_filename_template.format(
-                platform=self.platform_id,
-                serial_id=self.serial_id,
-                flight_id=self.flight_id,
-                launch_time=self.launch_time,
-            )
+            if l2_filename_template:
+                l2_filename = l2_filename_template.format(
+                    platform=self.platform_id,
+                    serial_id=self.serial_id,
+                    flight_id=self.flight_id,
+                    launch_time=self.launch_time.astype(datetime).strftime(
+                        "%Y-%m-%d_%H-%M"
+                    ),
+                )
+            else:
+                l2_filename = hh.l2_filename_template.format(
+                    platform=self.platform_id,
+                    serial_id=self.serial_id,
+                    flight_id=self.flight_id,
+                    launch_time=self.launch_time.astype(datetime).strftime(
+                        "%Y-%m-%d_%H-%M"
+                    ),
+                )
 
         object.__setattr__(self, "l2_filename", l2_filename)
 
@@ -950,7 +964,7 @@ def add_l2_ds(self, l2_dir: str = None):
             Returns the sonde object with the L2 dataset added as an attribute.
         """
         if l2_dir is None:
-            self.l2_dir
+            l2_dir = self.l2_dir
 
         object.__setattr__(
             self, "l2_ds", xr.open_dataset(os.path.join(l2_dir, self.l2_filename))

tests/test_paths.py

@@ -5,7 +5,7 @@
 main_data_directory = "./example_data"
 platform_id = "HALO"
 flightdate = "20200119"
-path_structure = "{platform}/Level_0/{flight}"
+path_structure = "{platform}/Level_0/{flight_id}"
 platform_path_structure = "{platform}/Level_0"
 
 l1_path = os.path.join(main_data_directory, platform_id, "Level_1", flightdate)