Skip to content

Commit

Permalink
Merge pull request #75 from podaac/release/8.6.0
Browse files Browse the repository at this point in the history
Release/8.6.0
  • Loading branch information
skorper authored May 20, 2024
2 parents 4c8f1e0 + ed41164 commit b5fc731
Show file tree
Hide file tree
Showing 35 changed files with 133 additions and 142 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build-pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ jobs:
- uses: actions/setup-java@v2
with:
distribution: 'adopt'
java-version: '8.0.232'
java-version: '11.0.6'
- uses: gradle/gradle-build-action@v1
with:
gradle-version: 8.0.1
Expand Down
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,22 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [8.6.0]
### Added
- **PODAAC-5876**
- Update to use CMA 2.0.0, thus allowing 2.0.3 layer for lambda
- Update build to use java 11
- To generate java 11 compatible UMMG schema POJOs, jsonschema2pojo shall make use of command line parameter: --target-version 1.11
### Deprecated
### Removed
### Fixed
- **PODAAC-5857**
- Fixed the issue so footprint and bbx do not always appear together while isoXmlSpatial is configured to footprint only
- remove request.close() statement in CMRLambdaRestClient so the response could be pulled out and logged properly instead of always logging apache chunk read error due to the http channel was closed.
- adding more parameters to Jsonschema2pojo plugin to generate proper pojo from ummg schema
### Security
- Snyk: Security upgrade com.amazonaws:aws-java-sdk-s3 from 1.12.544 to 1.12.641

## [8.5.0]
### Added
- **PODAAC-5594**
Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
8.5.0
8.6.0
4 changes: 2 additions & 2 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
apply plugin: 'java'
sourceCompatibility = 1.8
targetCompatibility = 1.8
sourceCompatibility = 11
targetCompatibility = 11

dependencies {
implementation fileTree(dir: 'target/dependency/', include: '*.jar')
Expand Down
22 changes: 15 additions & 7 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@
<modelVersion>4.0.0</modelVersion>
<groupId>gov.nasa.podaac</groupId>
<artifactId>cumulus-metadata-aggregator</artifactId>
<version>8.5.0</version>
<version>8.6.0-rc.7</version>
<packaging>jar</packaging>

<name>Metadata-Aggregator</name>
<description>This module contains the main software for the Metadata-Aggregator lambda.</description>
<url>TODO</url>

<properties>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>11</maven.compiler.source>
<maven.compiler.target>11</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>
Expand All @@ -39,12 +39,12 @@
<dependency>
<groupId>gov.nasa.earthdata</groupId>
<artifactId>cumulus-message-adapter</artifactId>
<version>1.3.9</version>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.544</version>
<version>1.12.641</version>
</dependency>
<!-- For AWS Secret Manager -->
<dependency>
Expand Down Expand Up @@ -133,6 +133,12 @@
<version>4.5.1</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api -->
<dependency>
<groupId>javax.xml.bind</groupId>
<artifactId>jaxb-api</artifactId>
<version>2.3.1</version>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
Expand Down Expand Up @@ -172,14 +178,16 @@
<plugin>
<groupId>org.jsonschema2pojo</groupId>
<artifactId>jsonschema2pojo-maven-plugin</artifactId>
<version>1.0.2</version>
<version>1.2.1</version>
<configuration>
<sourceDirectory>${basedir}/src/main/resources/jsonschema</sourceDirectory>
<outputDirectory>src/main/java</outputDirectory>
<targetPackage>gov.nasa.cumulus.metadata.umm.generated</targetPackage>
<sourceType>jsonschema</sourceType>
<annotationStyle>GSON</annotationStyle>
<includeConstructors>true</includeConstructors>
<includeConstructors>false</includeConstructors>
<targetVersion>11</targetVersion>
<includeGeneratedAnnotation>false</includeGeneratedAnnotation>
<formatDateTimes>true</formatDateTimes>
<formatDates>true</formatDates>
<formatDateTimes>true</formatDateTimes>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,6 @@ public CMRLambdaRestClient(String cert, String pass, String tknHost,
this.echoHost = cmrHost;
URIBuilder uriBuilder = new URIBuilder(tknHost);
this.tokenHost = uriBuilder.setPath(uriBuilder.getPath() + "/gettoken").build().normalize().toString();
AdapterLogger.LogInfo(this.className + " final token url:" + this.tokenHost);
this.validHost = this.tokenHost.replaceAll("gettoken", "validate");
this.region = region;
this.tknBucket = tknBucket;
Expand Down Expand Up @@ -105,15 +104,13 @@ public String getToken()
JSONObject jsonTkn = readToken();
long fileTime = getTokenStartTime(jsonTkn);
if (jsonTkn == null || sessionExpired(fileTime, runTime)) {
AdapterLogger.LogDebug(this.className + " Generating new NAMS token...");
try {
this.token = buildToken(runTime);
} catch (Exception e) {
//ERROR getting new token from NAMS.
throw new IOException("Could not retrieve token..." + e);
}
} else {
AdapterLogger.LogDebug(this.className + " Session active, using saved token");
this.token = (String) jsonTkn.get("token");
}
return this.token;
Expand Down Expand Up @@ -261,7 +258,6 @@ private JSONObject readToken()
}
String localTokenFilePath = s3Utils.download(this.region, this.tknBucket, this.tknFilePath,
Paths.get(this.workingDir, "token.json").toString());
AdapterLogger.LogInfo(this.className + " downloaded token file to local:" + localTokenFilePath);
JSONObject json = null;
try (FileReader reader = new FileReader(String.valueOf(localTokenFilePath))) {
json = (JSONObject) parser.parse(reader);
Expand All @@ -278,7 +274,6 @@ private void writeToken(long timeStamp, String token)
JSONObject json = new JSONObject();
json.put("authTime", timeStamp);
json.put("token", token);
AdapterLogger.LogDebug(this.className + " Writing new token to file");
Path localTknFilePath = Files.write(Paths.get(this.workingDir, "token.json"),
json.toString().getBytes());
s3Utils.upload(region, this.tknBucket, this.tknFilePath, new File(localTknFilePath.toString()) );
Expand All @@ -302,7 +297,6 @@ private HttpResponse send(String url, HttpEntity entity)
request.setHeader("Content-Type", content_type);
// Send the request
HttpResponse response = httpClient.execute(request);
request.releaseConnection();
return response;
}

Expand All @@ -314,7 +308,6 @@ public HttpResponse validateUMMG(String provider, String granuleId, String strU
String validateUMMGUri = uriBuilder.setPath(uriBuilder.getPath() + "/ingest/providers/"
+ provider +"/validate/granule/" + granuleId)
.build().normalize().toString();
AdapterLogger.LogDebug("validateUri:" + validateUMMGUri);
HttpEntity httpEntity = new StringEntity(strUMMG, "utf-8");
HttpResponse httpResponse = send(validateUMMGUri, httpEntity);
return httpResponse;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,6 @@ private void send(String url, HttpEntity entity) throws IOException {
} else {
logHttpResponse(response, HttpOp.SEND);
}
// close the connection
request.releaseConnection();
}

public void closeScrollSession(String scrollJson) throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class IsoGranule extends UMMGranule {
private List<String> inputGranules;
private String PGEVersionClass;

private IsoType isoType;


public IsoGranule() {
this.identifiers = new HashMap<>();
Expand Down Expand Up @@ -130,12 +130,4 @@ public String getPGEVersionClass() {
public void setPGEVersionClass(String PGEVersionClass) {
this.PGEVersionClass = PGEVersionClass;
}

public IsoType getIsoType() {
return isoType;
}

public void setIsoType(IsoType isoType) {
this.isoType = isoType;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;

import javax.xml.bind.DatatypeConverter;
import javax.xml.parsers.DocumentBuilder;
Expand Down Expand Up @@ -324,12 +323,12 @@ public void readIsoMetadataFile(String file, String s3Location) throws ParserCon
// if we get here, we have the bare minimum fields already populated,
// so try and parse the rest of the granule metadata...
try {
((IsoGranule) this.granule).setIsoType(isoType);
if (isoType == IsoType.MENDS) {
AdapterLogger.LogInfo("Found MENDS file");
readIsoMendsMetadataFile(s3Location, doc, xpath);
} else if (isoType == IsoType.SMAP) {
AdapterLogger.LogInfo("Found SMAP file");
((IsoGranule) this.granule).setIsoType(isoType);
readIsoSmapMetadataFile(s3Location, doc, xpath);
} else {
AdapterLogger.LogWarning(isoType.name() + " didn't match any expected ISO type, skipping optional " +
Expand Down Expand Up @@ -486,7 +485,6 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat

((IsoGranule) granule).setOrbit(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.ORBIT, "IsoMendsXPath.ORBIT"));
((IsoGranule) granule).setSwotTrack(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.SWOT_TRACK, "IsoMendsXPath.SWOT_TRACK"));

Source source = new Source();
source.setSourceShortName(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.PLATFORM, "IsoMendsXPath.PLATFORM"));

Expand All @@ -511,7 +509,7 @@ public IsoGranule readIsoMendsMetadataFile(String s3Location, Document doc, XPat
String cyclePassTileSceneStr =StringUtils.trim(MENDsISOXmlUtiils.extractXPathValueSwallowException(doc, xpath, IsoMendsXPath.CYCLE_PASS_TILE_SCENE, "IsoMendsXPath.CYCLE_PASS_TILE_SCENE"));
if(!StringUtils.isBlank(cyclePassTileSceneStr)) {
try {
createIsoCyclePassTile(cyclePassTileSceneStr);
granule = createIsoCyclePassTile(cyclePassTileSceneStr);
} catch (Exception e) {
// Since TrackType which contains Cycle Pass Tile and Scenes is not a required field
// we catch exception with printStackTrace to know the exact line throwing error
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -420,7 +420,8 @@ private JSONObject exportTemporal() {
*/
private boolean shouldAddBBx(Granule granule) {
boolean shouldAddBBx = false;
if(granule !=null && granule instanceof gov.nasa.cumulus.metadata.aggregator.UMMGranule) {

if(granule !=null && granule.getIsoType() == null) {
shouldAddBBx = true;
}
// if the granule object is IsoGranule type and it is SMAP mission, then we check if polygon was added.
Expand All @@ -430,6 +431,7 @@ private boolean shouldAddBBx(Granule granule) {
&& StringUtils.isNotEmpty(((IsoGranule) granule).getPolygon())) {
shouldAddBBx = false;
}
AdapterLogger.LogInfo(this.className + " shouldAddBBx:" + shouldAddBBx);
return shouldAddBBx;
}

Expand All @@ -453,6 +455,7 @@ private JSONObject exportSpatial() throws ParseException{
AdapterLogger.LogInfo(this.className + " nc.iso.xml footprint processing ... ");
this.isLineFormattedPolygon = true;
geometry = line2Polygons(geometry,polygon);
horizontalSpatialDomain.put("Geometry", geometry);
}
if(this.isoXMLSpatialTypeEnumHashSet.contains(MENDsIsoXMLSpatialTypeEnum.ORBIT)) {
AdapterLogger.LogDebug(this.className + "UMMGranuleFile.exportSpatial ORBIT Processing");
Expand All @@ -478,13 +481,18 @@ private JSONObject exportSpatial() throws ParseException{
isoBBoxAlreadyProcessed = true;
horizontalSpatialDomain = this.appendBoundingRectangles(geometry, horizontalSpatialDomain);
}
// Export track
if (((IsoGranule) granule).getSwotTrack() != "") {
// Export track for isoXML SMAP
// look into : IsoSmapXPath.SWOT_TRACK and how we use it to grab and store coordinate string to granule's
// swotTrack string. We also store swotTrack to MENDs granule and IsoXPath.SWOT_TRACK and IsoMendsXPath.CYCLE_PASS_TILE_SCENE
// these 2 values are being used to "extrac" cycle pass string from eith MENDs or SMAP. However, smap cycle and pass
// are being processed below
if (((IsoGranule) granule).getSwotTrack() != "" && granule.getIsoType() == IsoType.SMAP) {
JSONObject track = new JSONObject();
horizontalSpatialDomain.put("Track", track);
Pattern trackPattern = Pattern.compile("Cycle:\\s(.*)\\sPass:\\s(.*)\\sTile:\\s(.*)");
Matcher trackMatcher = trackPattern.matcher(((IsoGranule) granule).getSwotTrack());
if (trackMatcher.find()) {
AdapterLogger.LogDebug("SWOT track found cycle");
track.put("Cycle", Integer.parseInt(trackMatcher.group(1)));
JSONArray passes = new JSONArray();
track.put("Passes", passes);
Expand All @@ -510,17 +518,15 @@ private JSONObject exportSpatial() throws ParseException{
}
} // end of processing IsoGranule

// following is a large block code to deal with None IsoXML (MENDs or SMAP) bounding box
// We can only include orbital or bounding-box data, not both
// if iso Bounding Box already processed in logic above, then don't enter this block
if (foundOrbitalData == false && !isoBBoxAlreadyProcessed) {

horizontalSpatialDomain.put("Geometry", geometry);

JSONArray boundingRectangles = new JSONArray();
geometry.put("BoundingRectangles", boundingRectangles);

double north = 0, south = 0, east = 0, west = 0;
if(granule !=null && granule instanceof gov.nasa.cumulus.metadata.aggregator.UMMGranule) {
if(granule !=null && granule.getIsoType() == null) {
east = ((UMMGranule) granule).getBbxEasternLongitude() != null ?
((UMMGranule) granule).getBbxEasternLongitude() : 0;
west = ((UMMGranule) granule).getBbxWesternLongitude() != null?
Expand Down Expand Up @@ -565,7 +571,6 @@ private JSONObject exportSpatial() throws ParseException{
// and make sure we turn off the rangeIs360 flag
this.rangeIs360 = false;
}

BigDecimal nrth = new BigDecimal(north);
BigDecimal sth = new BigDecimal(south);
nrth = nrth.setScale(3, RoundingMode.HALF_UP);
Expand Down Expand Up @@ -609,9 +614,17 @@ private JSONObject exportSpatial() throws ParseException{
}
}
}
}

// Export track if cycle and pass exists
if (boundingRectangles.size() > 0) {
geometry.put("BoundingRectangles", boundingRectangles);
}
}
// Export TrackType
// MENDS : ISO
// Or Non-iso track : such as MODIS_A and Sentinel-6
// for instance, iso MENDS and Sentinel-6 and MODIS_A are having TrackType created within MetadataFilesToEcho class
// if cycle and pass exists. MENDs is taking adavangate of UMMG generated pojo : TrackType
// which is stored within the UMMGranule
if (granule instanceof UMMGranule) {
/**
* Track include cycle and passes(array).
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -12,7 +11,6 @@
* A reference to an additional attribute in the parent collection. The attribute reference may contain a granule specific value that will override the value in the parent collection for this granule. An attribute with the same name must exist in the parent collection.
*
*/
@Generated("jsonschema2pojo")
public class AdditionalAttributeType {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -12,7 +11,6 @@
* A boundary is set of points connected by straight lines representing a polygon on the earth. It takes a minimum of three points to make a boundary. Points must be specified in counter-clockwise order and closed (the first and last vertices are the same).
*
*/
@Generated("jsonschema2pojo")
public class BoundaryType {

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@

package gov.nasa.cumulus.metadata.umm.generated;

import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -10,7 +9,6 @@
* This entity holds the horizontal spatial coverage of a bounding box.
*
*/
@Generated("jsonschema2pojo")
public class BoundingRectangleType {

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

import java.util.ArrayList;
import java.util.List;
import javax.annotation.Generated;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;

Expand All @@ -12,7 +11,6 @@
* Contains the excluded boundaries from the GPolygon.
*
*/
@Generated("jsonschema2pojo")
public class ExclusiveZoneType {

/**
Expand Down
Loading

0 comments on commit b5fc731

Please sign in to comment.