Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Include Content-Length response header #151

Closed
wants to merge 1 commit into from
Closed

Include Content-Length response header #151

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,12 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest</artifactId>
<version>2.2</version>
<scope>test</scope>
</dependency>
</dependencies>

<build>
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/org/icatproject/ids/DataSelection.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;

import jakarta.json.Json;
Expand Down Expand Up @@ -53,6 +54,7 @@ public class DataSelection {
private Set<Long> emptyDatasets;
private boolean dsWanted;
private boolean dfWanted;
private long length;


public enum Returns {
Expand Down Expand Up @@ -135,6 +137,7 @@ private void resolveDatasetIds()
dsInfos.put(dsid, new DsInfoImpl(ds));
if (dfWanted) {
Datafile df = (Datafile) icat.get(userSessionId, "Datafile", dfid);
length += df.getFileSize();
String location = IdsBean.getLocation(dfid, df.getLocation());
dfInfos.add(
new DfInfoImpl(dfid, df.getName(), location, df.getCreateId(), df.getModId(), dsid));
Expand Down Expand Up @@ -315,4 +318,10 @@ public Set<Long> getEmptyDatasets() {
return emptyDatasets;
}

public OptionalLong getFileLength() {
if (!dfWanted || mustZip()) {
return OptionalLong.empty();
}
return OptionalLong.of(length);
}
}
12 changes: 9 additions & 3 deletions src/main/java/org/icatproject/ids/IdsBean.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.OptionalLong;
import java.util.Set;
import java.util.SortedMap;
import java.util.SortedSet;
Expand Down Expand Up @@ -50,6 +51,7 @@
import jakarta.json.JsonReader;
import jakarta.json.JsonValue;
import jakarta.json.stream.JsonGenerator;
import static jakarta.ws.rs.core.HttpHeaders.CONTENT_LENGTH;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;

Expand Down Expand Up @@ -859,6 +861,7 @@ public Response getData(String sessionId, String investigationIds, String datase
// Do it
Map<Long, DsInfo> dsInfos = dataSelection.getDsInfo();
Set<DfInfoImpl> dfInfos = dataSelection.getDfInfo();
var length = zip ? OptionalLong.empty() : dataSelection.getFileLength();

Lock lock = null;
try {
Expand Down Expand Up @@ -909,11 +912,14 @@ public Response getData(String sessionId, String investigationIds, String datase
}
}

return Response.status(offset == 0 ? HttpURLConnection.HTTP_OK : HttpURLConnection.HTTP_PARTIAL)
var response = Response.status(offset == 0 ? HttpURLConnection.HTTP_OK : HttpURLConnection.HTTP_PARTIAL)
.entity(new SO(dataSelection.getDsInfo(), dataSelection.getDfInfo(), offset, finalZip, compress, lock,
transferId, ip, start))
.header("Content-Disposition", "attachment; filename=\"" + name + "\"").header("Accept-Ranges", "bytes")
.build();
.header("Content-Disposition", "attachment; filename=\"" + name + "\"").header("Accept-Ranges", "bytes");
length.stream()
.map(l -> Math.max(0L, l - offset))
.forEach(l -> response.header(CONTENT_LENGTH, l));
return response.build();
} catch (AlreadyLockedException e) {
logger.debug("Could not acquire lock, getData failed");
throw new DataNotOnlineException("Data is busy");
Expand Down
12 changes: 12 additions & 0 deletions src/test/java/org/icatproject/ids/integration/BaseTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
Expand All @@ -20,6 +21,7 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.UUID;
import java.util.zip.CRC32;
Expand Down Expand Up @@ -369,6 +371,16 @@ protected void checkStream(InputStream stream, long id) throws IOException {
assertTrue(found);
}

protected long fileLength(long id) {
for (Entry<String, Long> e : ids.entrySet()) {
if (id == e.getValue()) {
var fileContents = contents.get(e.getKey());
return fileContents.getBytes(StandardCharsets.UTF_8).length;
}
}
throw new NoSuchElementException("No file with id " + id);
}

protected void writeToFile(Datafile df, String content, String key)
throws IOException, IcatException_Exception, NoSuchAlgorithmException {
Path path = setup.getStorageDir().resolve(df.getLocation());
Expand Down
55 changes: 46 additions & 9 deletions src/test/java/org/icatproject/ids/integration/one/GetDataExplicitTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import java.io.InputStream;
import java.util.Arrays;
import static org.hamcrest.Matchers.*;
import static org.hamcrest.MatcherAssert.assertThat;

import static org.junit.Assert.assertEquals;
import org.junit.BeforeClass;
Expand Down Expand Up @@ -99,51 +101,86 @@ public void forbiddenTest() throws Exception {
@Test
public void correctBehaviourTestNone() throws Exception {
try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafiles(datafileIds),
Flag.NONE, 0, 200)) {
Flag.NONE, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds, 57L, 0);
}

try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafile(datafileIds.get(0)),
Flag.NONE, 0, 200)) {
var datafileId = datafileIds.get(0);
var fileLength = fileLength(datafileId);
try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafile(datafileId),
Flag.NONE, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(not(nullValue())));
var contentLength = r.getFirstHeader("Content-Length").getValue();
assertThat(Long.valueOf(contentLength), is(equalTo(fileLength)));
assertThat(r.getFirstHeader("Transfer-Encoding"), is(nullValue()));
})) {
checkStream(stream, datafileIds.get(0));
}
}

@Test
public void correctBehaviourTestCompress() throws Exception {
try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafiles(datafileIds),
Flag.COMPRESS, 0, 200)) {
Flag.COMPRESS, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds, 36L, 0);
}

try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafile(datafileIds.get(0)),
Flag.COMPRESS, 0, 200)) {
Flag.COMPRESS, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkStream(stream, datafileIds.get(0));
}
}

@Test
public void correctBehaviourTestZip() throws Exception {
try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafiles(datafileIds),
Flag.ZIP, 0, 200)) {
Flag.ZIP, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds, 57L, 0);
}

try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafile(datafileIds.get(0)),
Flag.ZIP, 0, 200)) {
Flag.ZIP, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds.subList(0, 1), 57L, 0);
}
}

@Test
public void correctBehaviourTestZipAndCompress() throws Exception {
try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafiles(datafileIds),
Flag.ZIP_AND_COMPRESS, 0, 200)) {
Flag.ZIP_AND_COMPRESS, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds, 36L, 0);
}

try (InputStream stream = testingClient.getData(sessionId, new DataSelection().addDatafile(datafileIds.get(0)),
Flag.ZIP_AND_COMPRESS, 0, 200)) {
Flag.ZIP_AND_COMPRESS, 0, 200, r -> {
assertThat(r.getFirstHeader("Content-Length"), is(nullValue()));
assertThat(r.getFirstHeader("Transfer-Encoding").getValue(),
is(equalToIgnoringCase("chunked")));
})) {
checkZipStream(stream, datafileIds.subList(0, 1), 36L, 0);
}
}
Expand Down
14 changes: 12 additions & 2 deletions src/test/java/org/icatproject/ids/integration/util/client/TestingClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import jakarta.json.JsonObject;
import jakarta.json.JsonReader;
import jakarta.json.JsonValue;
import java.util.function.Consumer;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
Expand Down Expand Up @@ -266,6 +267,12 @@ public URL getIcatUrl(int sc) throws InternalException, ParseException, NotImple
public InputStream getData(String sessionId, DataSelection data, Flag flags, long offset, Integer sc)
throws NotImplementedException, BadRequestException, InsufficientPrivilegesException, NotFoundException,
InternalException, DataNotOnlineException {
return getData(sessionId, data, flags, offset, sc, null);
}

public InputStream getData(String sessionId, DataSelection data, Flag flags, long offset, Integer sc, Consumer<HttpResponse> responseAssertion)
throws NotImplementedException, BadRequestException, InsufficientPrivilegesException, NotFoundException,
InternalException, DataNotOnlineException {

URIBuilder uriBuilder = getUriBuilder("getData");
uriBuilder.setParameter("sessionId", sessionId);
Expand Down Expand Up @@ -293,6 +300,9 @@ public InputStream getData(String sessionId, DataSelection data, Flag flags, lon
response = httpclient.execute(httpGet);
checkStatus(response, sc);
checkResponseConformity(response);
if (responseAssertion != null) {
responseAssertion.accept(response);
}
closeNeeded = false;
return new HttpInputStream(httpclient, response);
} catch (IOException | InsufficientStorageException e) {
Expand Down Expand Up @@ -928,14 +938,14 @@ public void write(String sessionId, DataSelection data, Integer sc) throws NotIm
}

private void checkResponseConformity(HttpResponse response) throws InternalException {

StatusLine status = response.getStatusLine();
var statusCode = status.getStatusCode();
if(statusCode == 200 && response.getEntity() != null) {
//we have a status of 200 and a payload. Check for header conformity
Boolean chunked = response.getFirstHeader("Transfer-Encoding") != null ? response.getFirstHeader("Transfer-Encoding").getValue().contains("chunked")
: false;

if ( (response.getFirstHeader("Content-Length") == null && !chunked)
|| (response.getFirstHeader("Content-Length") != null && chunked ) ) {

Expand Down