Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

minoc: infer FITS content type from filename if Artifact.contentType is null #574

Merged
merged 2 commits into from
Apr 10, 2024
Merged

minoc: infer FITS content type from filename if Artifact.contentType is null #574

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1f32ff4
minoc: infer FITS content type from filename if Artifact.contentType …
pdowler Apr 10, 2024
dbd6b75
Merge branch 'master' of https://github.com/opencadc/storage-inventory
pdowler Apr 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion minoc/VERSION
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@
# tags with and without build number so operators use the versioned
# tag but we always keep a timestamped tag in case a semantic tag gets
# replaced accidentally
VER=1.0.2
VER=1.0.3
TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
unset VER
32 changes: 31 additions & 1 deletion minoc/src/intTest/java/org/opencadc/minoc/FitsOperationsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,7 @@
import ca.nrc.cadc.auth.AuthMethod;
import ca.nrc.cadc.net.HttpDelete;
import ca.nrc.cadc.net.HttpGet;
import ca.nrc.cadc.net.HttpPost;
import ca.nrc.cadc.net.HttpTransfer;
import ca.nrc.cadc.net.HttpUpload;
import ca.nrc.cadc.net.NetUtil;
Expand Down Expand Up @@ -99,6 +100,9 @@
import java.nio.file.Path;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

/**
* Integration test to pull existing test FITS files from VOSpace (Vault) into a local directory, then PUT them into
Expand All @@ -117,6 +121,9 @@ public class FitsOperationsTest extends MinocTest {
+ "/.config/test-data").toPath();

protected URL filesVaultURL;

// normally true except for one test
private boolean setContentType = true;

static {
Log4jInit.setLevel("org.opencadc.minoc", Level.INFO);
Expand Down Expand Up @@ -190,6 +197,27 @@ public void testSimple() throws Exception {
};

uploadAndCompareCutout(artifactURI, SodaParamValidator.SUB, cutoutSpecs, testFilePrefix);

LOGGER.info("unset content-type and try again: rely on filename extension only...");
final URI noclArtifactURI = URI.create("cadc:TEST/" + testFilePrefix + "-nocl." + testFileExtension);
final URL noclArtifactURL = new URL(filesURL + "/" + noclArtifactURI.toString());
LOGGER.info("no content-length: " + noclArtifactURL);

try {
setContentType = false;
uploadAndCompareCutout(noclArtifactURI, SodaParamValidator.SUB, cutoutSpecs, testFilePrefix);
} finally {
setContentType = true;
}

Subject.doAs(userSubject, (PrivilegedExceptionAction<Object>) () -> {
HttpGet head = new HttpGet(noclArtifactURL, false);
head.setHeadOnly(true);
head.prepare();
Assert.assertNull("no content type", head.getResponseHeader("content-type"));
return null;
});

}

@Test
Expand Down Expand Up @@ -462,7 +490,9 @@ private void ensureFile(final URI artifactURI) throws Exception {
final HttpUpload upload = new HttpUpload(fileInputStream, artifactURL);
upload.setRequestProperty("X-Test-Method", fileName);
upload.setRequestProperty(HttpTransfer.CONTENT_LENGTH, Long.toString(localFile.length()));
upload.setRequestProperty(HttpTransfer.CONTENT_TYPE, "application/fits");
if (setContentType) {
upload.setRequestProperty(HttpTransfer.CONTENT_TYPE, "application/fits");
}
upload.run();
LOGGER.info("response code: " + upload.getResponseCode() + " " + upload.getThrowable());
Assert.assertNull("Upload contains error.", upload.getThrowable());
Expand Down
42 changes: 30 additions & 12 deletions minoc/src/main/java/org/opencadc/minoc/GetAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,18 @@ public class GetAction extends ArtifactAction {
private static final String CONTENT_DISPOSITION = "content-disposition";
private static final String CONTENT_RANGE = "content-range";
private static final String CONTENT_LENGTH = "content-length";

private static final String FITS_CONTENT_TYPE = "application/fits";
private static final String[] FITS_CONTENT_TYPES = new String[] {
"application/fits", "image/fits"
FITS_CONTENT_TYPE,
"image/fits" // alternative
};

private static final SodaParamValidator SODA_PARAM_VALIDATOR = new SodaParamValidator();
private static final String[] FITS_EXTENSIONS = new String[] {
".fits",
".fz"
};

private final SodaParamValidator sodaParamValidator = new SodaParamValidator();

// constructor for unit tests with no config/init
GetAction(boolean init) {
Expand Down Expand Up @@ -271,7 +278,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
log.debug("SUB supplied");
final Map<String, List<String>> parameterMap = new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.SUB, sodaCutout.requestedSubs);
final List<ExtensionSlice> slices = SODA_PARAM_VALIDATOR.validateSUB(parameterMap);
final List<ExtensionSlice> slices = sodaParamValidator.validateSUB(parameterMap);
final Cutout cutout = new Cutout();
cutout.pixelCutouts = slices;

Expand All @@ -296,7 +303,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.CIRCLE, sodaCutout.requestedCircles);
final List<Circle> validCircles = SODA_PARAM_VALIDATOR.validateCircle(parameterMap);
final List<Circle> validCircles = sodaParamValidator.validateCircle(parameterMap);

cutout.pos = assertSingleWCS(SodaParamValidator.CIRCLE, validCircles);
}
Expand All @@ -306,7 +313,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.POLYGON, sodaCutout.requestedPolygons);
final List<Polygon> validPolygons = SODA_PARAM_VALIDATOR.validatePolygon(parameterMap);
final List<Polygon> validPolygons = sodaParamValidator.validatePolygon(parameterMap);

cutout.pos = assertSingleWCS(SodaParamValidator.POLYGON, validPolygons);
}
Expand All @@ -316,7 +323,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.POS, sodaCutout.requestedPOSs);
final List<Shape> validShapes = SODA_PARAM_VALIDATOR.validatePOS(parameterMap);
final List<Shape> validShapes = sodaParamValidator.validatePOS(parameterMap);

cutout.pos = assertSingleWCS(SodaParamValidator.POS, validShapes);
}
Expand All @@ -326,7 +333,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.BAND, sodaCutout.requestedBands);
final List<Interval> validBandIntervals = SODA_PARAM_VALIDATOR.validateBAND(parameterMap);
final List<Interval> validBandIntervals = sodaParamValidator.validateBAND(parameterMap);

cutout.band = assertSingleWCS(SodaParamValidator.BAND, validBandIntervals);
}
Expand All @@ -336,7 +343,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.TIME, sodaCutout.requestedTimes);
final List<Interval> validTimeIntervals = SODA_PARAM_VALIDATOR.validateTIME(parameterMap);
final List<Interval> validTimeIntervals = sodaParamValidator.validateTIME(parameterMap);

cutout.time = assertSingleWCS(SodaParamValidator.TIME, validTimeIntervals);
}
Expand All @@ -346,7 +353,7 @@ private ByteCountOutputStream doOperation(FitsOperations fitsOperations, SodaCut
final Map<String, List<String>> parameterMap =
new TreeMap<>(new CaseInsensitiveStringComparator());
parameterMap.put(SodaParamValidator.POL, sodaCutout.requestedPOLs);
cutout.pol = SODA_PARAM_VALIDATOR.validatePOL(parameterMap);
cutout.pol = sodaParamValidator.validatePOL(parameterMap);

if (cutout.pol.size() != sodaCutout.requestedPOLs.size()) {
log.debug("Accepted " + cutout.pol + " valid POL states but " + sodaCutout.requestedPOLs
Expand Down Expand Up @@ -381,8 +388,19 @@ private <T> T assertSingleWCS(final String key, final List<T> wcsValues) {
}

private boolean isFITS(final Artifact artifact) {
return StringUtil.hasText(artifact.contentType)
&& Arrays.stream(FITS_CONTENT_TYPES).anyMatch(s -> s.equals(artifact.contentType));
final String contentType = (artifact.contentType != null
? artifact.contentType : getContentTypeFromFilename(artifact.getURI().getSchemeSpecificPart()));
return StringUtil.hasText(contentType)
&& Arrays.stream(FITS_CONTENT_TYPES).anyMatch(s -> s.equals(contentType));
}

private String getContentTypeFromFilename(String filename) {
for (String ext : FITS_EXTENSIONS) {
if (filename.endsWith(ext)) {
return FITS_CONTENT_TYPE;
}
}
return null;
}

/**
Expand Down
Loading