Skip to content

Commit

Permalink
Merge pull request #293 from pdowler/master
Browse files Browse the repository at this point in the history 
caom2-repo: fix charset encoding bug
  • Loading branch information
@pdowler
pdowler authored Nov 2, 2023
2 parents b632c86 + 590162e commit c3cb1b1
Show file tree
Hide file tree
Showing 5 changed files with 111 additions and 12 deletions.
2 changes: 1 addition & 1 deletion caom2-repo/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ sourceCompatibility = 1.8

group = 'org.opencadc'

version = '1.4.7'
version = '1.4.8'

description = 'OpenCADC CAOM repository client library'
def git_url = 'https://github.com/opencadc/caom2db'
Expand Down
13 changes: 6 additions & 7 deletions caom2-repo/src/main/java/ca/nrc/cadc/caom2/repo/client/Worker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
******************* CANADIAN ASTRONOMY DATA CENTRE *******************
************** CENTRE CANADIEN DE DONNÉES ASTRONOMIQUES **************
*
* (c) 2011. (c) 2011.
* (c) 2023. (c) 2023.
* Government of Canada Gouvernement du Canada
* National Research Council Conseil national de recherches
* Ottawa, Canada, K1A 0R6 Ottawa, Canada, K1A 0R6
Expand Down Expand Up @@ -73,17 +73,15 @@
import ca.nrc.cadc.caom2.ObservationResponse;
import ca.nrc.cadc.caom2.ObservationState;
import ca.nrc.cadc.caom2.xml.ObservationReader;
import ca.nrc.cadc.net.HttpDownload;

import ca.nrc.cadc.net.HttpGet;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.util.concurrent.Callable;

import javax.security.auth.Subject;

import org.apache.log4j.Logger;

/**
Expand Down Expand Up @@ -125,7 +123,7 @@ public ObservationResponse getObservation(URI uri) {
} catch (MalformedURLException e) {
throw new RuntimeException("Unable to create URL object for " + surl);
}
HttpDownload get = new HttpDownload(url, bos);
HttpGet get = new HttpGet(url, bos);

if (subject != null) {
Subject.doAs(subject, new RunnableAction(get));
Expand All @@ -143,7 +141,8 @@ public ObservationResponse getObservation(URI uri) {
} else {
try {
ObservationReader obsReader = new ObservationReader();
wr.observation = obsReader.read(bos.toString());
ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
wr.observation = obsReader.read(bis);
} catch (Exception e) {
wr.error = new IllegalArgumentException("failed to read observation document: " + e.getMessage(), e);
}
Expand Down
2 changes: 1 addition & 1 deletion icewind/VERSION
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
## deployable containers have a semantic and build tag
# semantic version tag: major.minor[.patch]
# build version tag: timestamp
VER=0.9.8
VER=0.9.9
TAGS="${VER} ${VER}-$(date --utc +"%Y%m%dT%H%M%S")"
unset VER
2 changes: 1 addition & 1 deletion icewind/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ dependencies {
implementation 'org.opencadc:cadc-util:[1.6,2.0)'
implementation 'org.opencadc:caom2:[2.4.4,2.5)'
implementation 'org.opencadc:caom2persistence:[2.4.14,2.5)'
implementation 'org.opencadc:caom2-repo:[1.4.7,1.5)'
implementation 'org.opencadc:caom2-repo:[1.4.8,1.5)'

// needed for validation
implementation 'org.opencadc:caom2-compute:[2.4.6,2.5)'
Expand Down
104 changes: 102 additions & 2 deletions icewind/src/main/java/org/opencadc/icewind/ObservationHarvester.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,12 @@
package org.opencadc.icewind;

import ca.nrc.cadc.caom2.Artifact;
import ca.nrc.cadc.caom2.Chunk;
import ca.nrc.cadc.caom2.Observation;
import ca.nrc.cadc.caom2.ObservationResponse;
import ca.nrc.cadc.caom2.ObservationState;
import ca.nrc.cadc.caom2.ObservationURI;
import ca.nrc.cadc.caom2.Part;
import ca.nrc.cadc.caom2.Plane;
import ca.nrc.cadc.caom2.compute.CaomWCSValidator;
import ca.nrc.cadc.caom2.harvester.state.HarvestSkipURI;
Expand All @@ -82,11 +84,13 @@
import ca.nrc.cadc.caom2.persistence.ObservationDAO;
import ca.nrc.cadc.caom2.repo.client.RepoClient;
import ca.nrc.cadc.caom2.util.CaomValidator;
import ca.nrc.cadc.caom2.xml.ObservationWriter;
import ca.nrc.cadc.db.ConnectionConfig;
import ca.nrc.cadc.db.DBUtil;
import ca.nrc.cadc.net.ResourceNotFoundException;
import ca.nrc.cadc.net.TransientException;
import java.net.URI;
import java.nio.charset.Charset;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.sql.SQLException;
Expand Down Expand Up @@ -339,7 +343,7 @@ private Progress doit() {
log.debug("skipped=" + skipped
+ " o=" + o
+ " ow.entity=" + ow.entity
+ " ow.entity.error=" + (ow.entity != null || ow.entity.error != null));
+ " ow.entity.error=" + (ow.entity == null ? null : ow.entity.error));
try {
// o could be null in skip mode cleanup
if (o != null) {
Expand Down Expand Up @@ -600,15 +604,111 @@ private void validateChecksum(Observation o) throws MismatchedChecksumException
try {
URI calculatedChecksum = o.computeAccMetaChecksum(MessageDigest.getInstance("MD5"));

log.debug("validateChecksum: " + o.getURI() + " -- " + o.getAccMetaChecksum() + " vs " + calculatedChecksum);
log.debug("validateChecksum: " + o.getURI() + " -- " + computeTreeSize(o) + " -- " + o.getAccMetaChecksum() + " vs " + calculatedChecksum);
if (!calculatedChecksum.equals(o.getAccMetaChecksum())) {
//detailedChecksumDiagnostics(o);
throw new MismatchedChecksumException("Observation.accMetaChecksum mismatch");
}
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException("MD5 digest algorithm not available");
}
}

private void detailedChecksumDiagnostics(Observation obs) {
// code yanked from caom2-validator
try {
int depth = 5;
boolean acc = true;
StringBuilder cs = new StringBuilder();
StringBuilder acs = new StringBuilder();
MessageDigest digest = MessageDigest.getInstance("MD5");
if (depth > 1) {
for (Plane pl : obs.getPlanes()) {
if (depth > 2) {
for (Artifact ar : pl.getArtifacts()) {
if (depth > 3) {
for (Part pa : ar.getParts()) {
if (depth > 4) {
for (Chunk ch : pa.getChunks()) {
URI chunkCS = ch.computeMetaChecksum(digest);
cs.append("\n chunk: ").append(ch.getID()).append(" ");
compare(cs, ch.getMetaChecksum(), chunkCS);
if (acc) {
URI chunkACS = ch.computeAccMetaChecksum(digest);
acs.append("\n chunk: ").append(ch.getID()).append(" ");
compare(acs, ch.getAccMetaChecksum(), chunkACS);
}
}
}
URI partCS = pa.computeMetaChecksum(digest);
cs.append("\n part: ").append(pa.getID()).append(" ");
compare(cs, pa.getMetaChecksum(), partCS);
if (acc) {
URI partACS = pa.computeAccMetaChecksum(digest);
acs.append("\n part: ").append(pa.getID()).append(" ");
compare(acs, pa.getAccMetaChecksum(), partACS);
}
}
}
URI artifactCS = ar.computeMetaChecksum(digest);
cs.append("\n artifact: ").append(ar.getID()).append(" ");
compare(cs, ar.getMetaChecksum(), artifactCS);
if (acc) {
URI artifactACS = ar.computeAccMetaChecksum(digest);
acs.append("\n artifact: ").append(ar.getID()).append(" ");
compare(acs, ar.getAccMetaChecksum(), artifactACS);
}
}
}
URI planeCS = pl.computeMetaChecksum(digest);
cs.append("\n plane: ").append(pl.getID()).append(" ");
compare(cs, pl.getMetaChecksum(), planeCS);
if (acc) {
URI planeACS = pl.computeAccMetaChecksum(digest);
acs.append("\n plane: ").append(pl.getID()).append(" ");
compare(acs, pl.getAccMetaChecksum(), planeACS);
}
}
}
URI observationCS = obs.computeMetaChecksum(digest);
cs.append("\nobservation: ").append(obs.getID()).append(" ");
compare(cs, obs.getMetaChecksum(), observationCS);

if (acc) {
URI observationACS = obs.computeAccMetaChecksum(digest);
acs.append("\nobservation: ").append(obs.getID()).append(" ");
compare(acs, obs.getAccMetaChecksum(), observationACS);
}

log.warn("** metaChecksum **");
log.warn(cs.toString());
if (acc) {
log.warn("** accMetaChecksum **");
log.warn(acs.toString());
}
log.warn("default charset: " + Charset.defaultCharset().displayName());
ObservationWriter ow = new ObservationWriter();
ow.write(obs, System.out);

} catch (Exception oops) {
log.error("failure during detailedChecksumDiagnostics", oops);
}
}

private void compare(StringBuilder sb, URI u1, URI u2) {
sb.append(u1);
boolean eq = u1.equals(u2);
if (eq) {
sb.append(" == ");
} else {
sb.append(" != ");
}
sb.append(u2);
if (!eq) {
sb.append(" [MISMATCH]");
}
}

private String computeTreeSize(Observation o) {
StringBuilder sb = new StringBuilder();
sb.append("[");
Expand Down

0 comments on commit c3cb1b1

Please sign in to comment.