Skip to content

Commit

Permalink
enhance code guard to only append relatedUrls when subTypeHashArray i…
Browse files Browse the repository at this point in the history 
…s not null or size zero. more unit test.
  • Loading branch information
Yen, David (398B-Affiliate) committed May 15, 2024
1 parent a20f213 commit bfa7cef
Show file tree
Hide file tree
Showing 6 changed files with 309 additions and 10 deletions.
8 changes: 4 additions & 4 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/MetadataAggregatorLambda.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import java.security.UnrecoverableKeyException;
import java.security.cert.CertificateException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.List;

Expand All @@ -22,7 +21,6 @@
import gov.nasa.cumulus.metadata.aggregator.processor.FootprintProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.ImageProcessor;
import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import gov.nasa.cumulus.metadata.state.WorkflowTypeEnum;
import gov.nasa.cumulus.metadata.util.S3Utils;
import org.apache.commons.io.FileUtils;
Expand Down Expand Up @@ -196,8 +194,10 @@ public String PerformFunction(String input, Context context) throws Exception {
String cmrFilePath = "/tmp/" + granuleId + ".cmr.json";
try {
JSONObject granuleJson = mtfe.createJson();
RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor();
granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files);
if(taskConfigBO.getSubTypeHashArray().size() > 0) {
RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor();
granuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson, taskConfigBO, files);
}
FileUtils.writeStringToFile(new File(cmrFilePath), granuleJson.toJSONString(), StandardCharsets.UTF_8);
} catch (IOException e) {
AdapterLogger.LogError(this.className + " mtfe.writeJson error:" + e.getMessage());
Expand Down
8 changes: 6 additions & 2 deletions src/main/java/gov/nasa/cumulus/metadata/aggregator/factory/TaskConfigFactory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,14 @@ public static TaskConfigBO createTaskConfigBO(JSONObject config) {
TaskConfigBO taskConfigBO = new TaskConfigBO();
//Construct isoXMLSpatialTypeHashset
JSONArray isoXMLSpatialTypeJsonArray = (JSONArray) config.get("isoXMLSpatialType");
taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray));
if(isoXMLSpatialTypeJsonArray !=null && isoXMLSpatialTypeJsonArray.size()>0) {
taskConfigBO.setIsoXMLSpatialTypeHashSet(createIsoXMLSpatialTypeSet(isoXMLSpatialTypeJsonArray));
}
//Construct subTypeHashArray
JSONArray subTypeTypeJsonArray = (JSONArray) config.get("relatedUrlSubTypeMap");
taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray));
if(subTypeTypeJsonArray!=null && subTypeTypeJsonArray.size()>0) {
taskConfigBO.setSubTypeHashArray(createSubTypeHashArray(subTypeTypeJsonArray));
}
return taskConfigBO;

}
Expand Down
4 changes: 3 additions & 1 deletion src/main/java/gov/nasa/cumulus/metadata/aggregator/processor/RelatedUrlsProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;

import java.util.ArrayList;
import java.util.HashMap;
Expand All @@ -21,6 +20,9 @@ public void RelatedUrlsProcessor() {

public JSONObject appendSubTypes(JSONObject granuleJson, TaskConfigBO taskConfigBO, JSONArray files) {
ArrayList<HashMap<String, String>> subTypeHashArray=taskConfigBO.getSubTypeHashArray();
if(subTypeHashArray==null || subTypeHashArray.size()==0) {
return granuleJson;
}
JSONArray relateUrlsArray = (JSONArray) granuleJson.get("RelatedUrls");
// 1: Loop through the existing RelatedUrls array to find any match for the subType regex.
// for the finding match, set Subtype to
Expand Down
122 changes: 122 additions & 0 deletions src/test/java/gov/nasa/cumulus/metadata/test/RelatedUrlsProcessorTest.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
package gov.nasa.cumulus.metadata.test;

import gov.nasa.cumulus.metadata.aggregator.IsoGranule;
import gov.nasa.cumulus.metadata.aggregator.MetadataAggregatorLambda;
import gov.nasa.cumulus.metadata.aggregator.MetadataFilesToEcho;
import gov.nasa.cumulus.metadata.aggregator.bo.TaskConfigBO;
import gov.nasa.cumulus.metadata.aggregator.factory.TaskConfigFactory;
import gov.nasa.cumulus.metadata.aggregator.processor.RelatedUrlsProcessor;
import gov.nasa.cumulus.metadata.state.MENDsIsoXMLSpatialTypeEnum;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
import org.xml.sax.SAXException;

import javax.xml.parsers.ParserConfigurationException;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathExpressionException;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;

import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

public class RelatedUrlsProcessorTest {
TaskConfigBO taskConfigBO = new TaskConfigBO();

@Before
public void setUp() throws Exception {
// Setup ArrayList<HashMap<String, String>> subTypeHashArray which is a data structure presenting relatedUrlSubTypeMap
// Read relatedUrlSubTypeMap from collection_config/OPERA_L3_DSWX-S1_PROVISIONAL_V0.json which is a collection configuration json
ClassLoader classLoader = getClass().getClassLoader();
File file = new File(classLoader.getResource("collection_config/OPERA_L3_DSWX-S1_PROVISIONAL_V0.json").getFile());
JSONParser parser = new JSONParser();
Object obj = parser.parse(new FileReader(file.getAbsolutePath()));
JSONObject collectionJSONObject = (JSONObject) obj;
// extract the relatedUrlSubTypeMap from collection configuration meta structure
JSONArray subTypeHasyArrayJson = (JSONArray)(((JSONObject)(collectionJSONObject.get("meta"))).get("relatedUrlSubTypeMap"));
JSONObject config = new JSONObject();
config.put("relatedUrlSubTypeMap", subTypeHasyArrayJson);
taskConfigBO = TaskConfigFactory.createTaskConfigBO(config);
}

@Test
public void testAppendSubTypes() throws IOException, ParseException, ParserConfigurationException,
SAXException, XPathExpressionException, URISyntaxException {
ClassLoader classLoader = getClass().getClassLoader();
File file = new File(classLoader.getResource("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.iso.xml").getFile());
File cfgFile = new File(classLoader.getResource("OPERA_L3_DSWX-HLS_PROVISIONAL_V0_test_1.cmr.cfg").getFile());
MetadataFilesToEcho mfte = new MetadataFilesToEcho(true);
mfte.getGranule().setName("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0");
Document doc = null;
XPath xpath = null;
mfte.readConfiguration(cfgFile.getAbsolutePath());
doc = mfte.makeDoc(file.getAbsolutePath());
xpath = mfte.makeXpath(doc);
IsoGranule isoGranule = mfte.readIsoMendsMetadataFile("s3://mybucket/mygranule.nc", doc, xpath);
// use a unrelated .mp file to patch required field so mfte.createJson() would work.
File file2 = new File(classLoader.getResource("JA1_GPN_2PeP374_172_20120303_112035_20120303_121638.nc.mp").getFile());
try {
mfte.readCommonMetadataFile(file2.getAbsolutePath(), "s3://a/path/to/s3");
} catch (Exception e) {
e.printStackTrace();
fail();
}
mfte.getGranule().setName("OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0");
JSONObject granuleJson = mfte.createJson();

org.json.simple.JSONArray files = new JSONArray();
JSONObject f1 = new JSONObject();
JSONObject f2 = new JSONObject();
JSONObject f3 = new JSONObject();
// construct the files:[] array under payload
f1.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif");
f1.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif");
f1.put("type", "data");
f1.put("checksumType", "md5");
f2.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif");
f2.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif");
f2.put("type", "data");
f2.put("checksumType", "md5");
f3.put("source", "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B03_CONF.tif");
f3.put("fileName", "OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B03_CONF.tif");
f3.put("type", "data");
f3.put("checksumType", "md5");
files.add(f1);files.add(f2);files.add(f3);
RelatedUrlsProcessor relatedUrlsProcessor = new RelatedUrlsProcessor();
// Use appendSubTypes to generate the new Json including certain file items with subType
JSONObject newGranuleJson = relatedUrlsProcessor.appendSubTypes(granuleJson,taskConfigBO, files);
JSONArray relatedUrls =(JSONArray) newGranuleJson.get("RelatedUrls");
ArrayList<HashMap<String, String>> subTypeHashArray = taskConfigBO.getSubTypeHashArray();
/**
* We loop through the RelatedUrls array. for each relatedUrl item. If the URL field match the regex of
* a subTypeHash (which for example contains "regex": "^.*_B01_WTR\\.tif$",
* "subType": "BROWSE IMAGE SOURCE"), the Subtype field should match the
* subTypeHash's subType value
*/
for(int i = 0; i < relatedUrls.size(); i++)
{
JSONObject relatedUrl = (JSONObject)relatedUrls.get(i);
String URLStr=relatedUrl.get("URL").toString();
for(HashMap<String, String> subTypeHash : subTypeHashArray) {
if(URLStr.matches(subTypeHash.get("regex"))) {
assertEquals(relatedUrl.get("Subtype"), subTypeHash.get("subType"));
}
}
}
// Compare the newGranuleJson with the pre-saved cmr.json file
assertTrue(UnitTestUtil.compareFileWithGranuleJson("ummgResults/setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json", newGranuleJson));
}
}
4 changes: 1 addition & 3 deletions src/test/java/gov/nasa/cumulus/metadata/test/UMMTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -852,8 +852,6 @@ public void testStringMatch() {
String regex = "^.*_B02_BWTR\\.tif$";
String s3File = "s3://src/test/resources/tester/asdaffaf_B02_BWTR.tif";
boolean matched = s3File.matches(regex);
System.out.println("matched? " + matched);
assertEquals(matched, true);
}


}
173 changes: 173 additions & 0 deletions ...setSubType/OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0.cmr.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,173 @@
{
"InputGranules": [
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B02.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B03.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B04.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B05.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B06.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.B07.tif",
"/home/conda/input_dir/HLS.L30.T14RNV.2021249T170251.v2.0.Fmask.tif",
"/home/conda/input_dir/dem.vrt",
"/home/conda/input_dir/dem_0.tif",
"/home/conda/input_dir/landcover.tif",
"/home/conda/input_dir/worldcover.vrt",
"/home/conda/input_dir/worldcover_0.tif"
],
"TemporalExtent": {
"RangeDateTime": {
"EndingDateTime": "2012-03-03T12:16:38.727Z",
"BeginningDateTime": "2012-03-03T11:20:35.236Z"
}
},
"MetadataSpecification": {
"Version": "1.6.6",
"URL": "https://cdn.earthdata.nasa.gov/umm/granule/v1.6.6",
"Name": "UMM-G"
},
"GranuleUR": "OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0",
"ProviderDates": [
{
"Type": "Insert",
"Date": "2024-05-15T03:19:06.144Z"
},
{
"Type": "Update",
"Date": "2024-05-15T03:19:06.154Z"
}
],
"SpatialExtent": {
"HorizontalSpatialDomain": {
"Geometry": {
"BoundingRectangles": [
{
"WestBoundingCoordinate": 0.238,
"SouthBoundingCoordinate": -66.147,
"EastBoundingCoordinate": 359.846,
"NorthBoundingCoordinate": 66.142
}
]
},
"Track": {
"Cycle": 374,
"Passes": [
{
"Pass": 172
}
]
}
}
},
"AdditionalAttributes": [
{
"Values": [
"HLS.L30.T14RNV.2021249T170251.v2.0"
],
"Name": "HlsDataset"
},
{
"Values": [
"LC08_L1TP_027038_20210906_20210915_02_T1; LC08_L1TP_027039_20210906_20210915_02_T1"
],
"Name": "SensorProductID"
},
{
"Values": [
"Lasrc; Lasrc"
],
"Name": "Accode"
},
{
"Values": [
"135.857670779298"
],
"Name": "MeanSunAzimuthAngle"
},
{
"Values": [
"32.3676212035439"
],
"Name": "MeanSunZenithAngle"
},
{
"Values": [
"30.8893421483768"
],
"Name": "NBAR_SolarZenith"
},
{
"Values": [
"106.186652634788"
],
"Name": "MeanViewAzimuthAngle"
},
{
"Values": [
"5.12004424472291"
],
"Name": "MeanViewZenithAngle"
},
{
"Values": [
"67"
],
"Name": "SpatialCoverage"
},
{
"Values": [
"76"
],
"Name": "PercentCloudCover"
},
{
"Values": [
"T14RNV"
],
"Name": "MGRS_TILE_ID"
}
],
"PGEVersionClass": {
"PGEName": "DSWX_HLS_PGE",
"PGEVersion": "1.0.0-rc.5"
},
"DataGranule": {
"DayNightFlag": "Unspecified",
"Identifiers": [
{
"Identifier": "OPERA_L3_DSWx_HLS_T14RNV_20210906T170251Z_20221026T184342Z_L8_30_v0.0",
"IdentifierType": "ProducerGranuleId"
},
{
"Identifier": "0.5",
"IdentifierType": "Other",
"IdentifierName": "SASVersionId"
},
{
"Identifier": "1.0.0-rc.5",
"IdentifierType": "Other",
"IdentifierName": "PGEVersionId"
}
],
"ProductionDateTime": "2016-03-16T03:02:03.000Z"
},
"CollectionReference": {
"Version": "0.0",
"ShortName": "OPERA_L3_DSWX-HLS_PROVISIONAL_V0"
},
"RelatedUrls": [
{
"Type": "GET DATA",
"Description": "The base directory location for the granule.",
"URL": "s3://a/path/to/s3"
},
{
"Type": "GET DATA",
"Subtype": "BROWSE IMAGE SOURCE",
"URL": "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B01_WTR.tif"
},
{
"Type": "GET DATA",
"Subtype": "IceBridge Portal",
"URL": "OPERA_L3_DSWX-S1_PROVISIONAL_V0/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4/OPERA_L3_DSWx-S1_T35TPJ_20240214T155240Z_20240502T001754Z_S1A_30_v0.4_B02_BWTR.tif"
}
]
}

0 comments on commit bfa7cef

Please sign in to comment.