Merge pull request #11 from emit-sds/develop

Merge develop into main for v1.3.0

CHANGELOG.md

@@ -4,11 +4,22 @@ All notable changes to this project will be documented in this file. Dates are d
 
 Generated by [`auto-changelog`](https://github.com/CookPete/auto-changelog).
 
+#### [v1.3.0](https://github.com/emit-sds/emit-utils/compare/v1.2.3...v1.3.0)
+
+> 1 September 2023
+
+- Reformat update [`#10`](https://github.com/emit-sds/emit-utils/pull/10)
+- add in l2b support [`eca74ad`](https://github.com/emit-sds/emit-utils/commit/eca74adc7e17a1d5ee0af072d5f02881146350ed)
+- Read in ffupdate paths from runconfig (not ENVI header) [`15ce3ee`](https://github.com/emit-sds/emit-utils/commit/15ce3ee7a01c3824def5914c72332a8d84024025)
+- Fix runconfig references [`3596959`](https://github.com/emit-sds/emit-utils/commit/3596959d5f1b103e82179361ffd4d5ed3b0a7ab2)
+
 #### [v1.2.3](https://github.com/emit-sds/emit-utils/compare/v1.2.2...v1.2.3)
 
 > 17 February 2023
 
+- Merge develop into main for v1.2.3 [`#9`](https://github.com/emit-sds/emit-utils/pull/9)
 - Add software delivery version to UMM-G and NetCDF metadata [`e881f7a`](https://github.com/emit-sds/emit-utils/commit/e881f7af6e77e97408426d211e6dd4d6c80e8b15)
+- Update change log [`b74962c`](https://github.com/emit-sds/emit-utils/commit/b74962c78a3f5af76e3706a5e2c27980fb2d91c9)
 - Update version to 1.2.3 [`dfd9a2f`](https://github.com/emit-sds/emit-utils/commit/dfd9a2f7ae978840f09e483dc7f75268858e189a)
 
 #### [v1.2.2](https://github.com/emit-sds/emit-utils/compare/v1.2.1...v1.2.2)

emit_utils/daac_converter.py

@@ -201,7 +201,7 @@ def makeGlobalAttrBase(nc_ds: netCDF4.Dataset):
     nc_ds.creator_name = "Jet Propulsion Laboratory/California Institute of Technology"
     nc_ds.creator_url = "https://earth.jpl.nasa.gov/emit/"
     nc_ds.project = "Earth Surface Mineral Dust Source Investigation"
-    nc_ds.project_url = "https://emit.jpl.nasa.gov/"
+    nc_ds.project_url = "https://earth.jpl.nasa.gov/emit/"
     nc_ds.publisher_name = "NASA LPDAAC"
     nc_ds.publisher_url = "https://lpdaac.usgs.gov"
     nc_ds.publisher_email = "lpdaac@usgs.gov"
@@ -215,7 +215,7 @@ def makeGlobalAttrBase(nc_ds: netCDF4.Dataset):
 
 
 def makeGlobalAttr(nc_ds: netCDF4.Dataset, primary_envi_file: str, software_delivery_version: str,
-                   glt_envi_file: str = None):
+                   glt_envi_file: str = None, rdn_runconfig_file: str = None):
     """
     Set up global attributes that are universal.  Required attributes that should be populated by individual PGEs
     are flagged with None values
@@ -237,6 +237,12 @@ def makeGlobalAttr(nc_ds: netCDF4.Dataset, primary_envi_file: str, software_deli
     nc_ds.software_build_version = primary_ds.metadata['emit software build version']
     nc_ds.software_delivery_version = software_delivery_version
     nc_ds.product_version = "V0" + primary_ds.metadata['emit data product version']
+    if rdn_runconfig_file is not None:
+        with open(rdn_runconfig_file, "r") as f:
+            runconfig = json.load(f)
+        ffupdate_files = [os.path.basename(p) for p in runconfig["flat_field_update_paths"]]
+        ffupdate_str = ",".join(ffupdate_files)
+        primary_ds.metadata['emit pge input files'].append(f"ffupdate_files=[{ffupdate_str}]")
     run_command = "PGE Run Command: {" + primary_ds.metadata['emit pge run command'] + "}"
     input_files = "PGE Input Files: {" + ", ".join(primary_ds.metadata['emit pge input files']) + "}"
     nc_ds.history = run_command + ", " + input_files
@@ -302,8 +308,8 @@ def initialize_ummg(granule_name: str, creation_time: datetime, collection_name:
     """
 
     ummg = get_required_ummg()
-    ummg['MetadataSpecification'] = {'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.3', 'Name': 'UMM-G',
-                                     'Version': '1.6.3'}
+    ummg['MetadataSpecification'] = {'URL': 'https://cdn.earthdata.nasa.gov/umm/granule/v1.6.5', 'Name': 'UMM-G',
+                                     'Version': '1.6.5'}
 
 
     ummg['Platforms'] = [{'ShortName': 'ISS', 'Instruments': [{'ShortName': 'EMIT Imaging Spectrometer'}]}]

emit_utils/reformat.py

@@ -71,10 +71,14 @@ def main(rawargs=None):
             if os.path.isfile(output_name) and args.overwrite is False:
                 err_str = f'File {output_name} already exists. Please use --overwrite to replace'
                 raise AttributeError(err_str)
+            nbands = 1
+            if len(nc_ds[ds].shape) > 2:
+                nbands = nc_ds[ds].shape[2]
+
             metadata = {
                 'lines': nc_ds[ds].shape[0],
                 'samples': nc_ds[ds].shape[1],
-                'bands': nc_ds[ds].shape[2],
+                'bands': nbands,
                 'interleave': args.interleave,
                 'header offset' : 0,
                 'file type' : 'ENVI Standard',
@@ -96,27 +100,32 @@ def main(rawargs=None):
 
                 metadata['coordinate system string'] = f'{{ {nc_ds.__dict__["spatial_ref"]} }}' 
 
-            band_parameters = nc_ds['sensor_band_parameters'].variables.keys() 
-            for bp in band_parameters:
-                if bp == 'wavelengths' or bp == 'radiance_wl':
-                    metadata['wavelength'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
-                elif bp == 'radiance_fwhm':
-                    metadata['fwhm'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
-                elif bp == 'observation_bands':
-                    metadata['band names'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
-                else:
-                    metadata[bp] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
+            if 'sensor_band_parameters' in nc_ds.__dict__.keys():
+                band_parameters = nc_ds['sensor_band_parameters'].variables.keys() 
+                for bp in band_parameters:
+                    if bp == 'wavelengths' or bp == 'radiance_wl':
+                        metadata['wavelength'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
+                    elif bp == 'radiance_fwhm':
+                        metadata['fwhm'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
+                    elif bp == 'observation_bands':
+                        metadata['band names'] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
+                    else:
+                        metadata[bp] = np.array(nc_ds['sensor_band_parameters'].variables[bp]).astype(str).tolist()
 
             if 'wavelength' in list(metadata.keys()) and 'band names' not in list(metadata.keys()):
                 metadata['band names'] = metadata['wavelength']
 
             envi_ds = envi.create_image(envi_header(output_name), metadata, ext='', force=args.overwrite) 
             mm = envi_ds.open_memmap(interleave='bip',writable=True)
 
+            dat = np.array(nc_ds[ds])
+            if len(dat.shape) == 2:
+                dat = dat.reshape((dat.shape[0],dat.shape[1],1))
+
             if args.orthorectify:
-                mm[...] = single_image_ortho(np.array(nc_ds[ds]), glt)
+                mm[...] = single_image_ortho(dat, glt)
             else:
-                mm[...] = np.array(nc_ds[ds])
+                mm[...] = np.array(dat)
             del mm, envi_ds
 
 

setup.py

@@ -9,7 +9,7 @@
 setup(name='emit_utils',
       packages=find_packages(),
       include_package_data=True,
-      version='1.2.3',
+      version='1.3.0',
       install_requires=[
           'gdal>=2.0',
           'spectral>=0.21',