Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extract PfbTestUtils #462

Merged
merged 1 commit into from
Jan 19, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import static org.assertj.core.api.Assertions.assertThat;
import static org.databiosphere.workspacedataservice.TestTags.SLOW;
import static org.databiosphere.workspacedataservice.dataimport.pfb.PfbTestUtils.buildPfbQuartzJob;
import static org.databiosphere.workspacedataservice.dataimport.pfb.PfbTestUtils.stubJobContext;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.any;
Expand All @@ -19,13 +18,9 @@
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
import org.databiosphere.workspacedataservice.activitylog.ActivityLogger;
import org.databiosphere.workspacedataservice.dao.JobDao;
import org.databiosphere.workspacedataservice.dao.SchedulerDao;
import org.databiosphere.workspacedataservice.generated.GenericJobServerModel;
import org.databiosphere.workspacedataservice.generated.ImportRequestServerModel;
import org.databiosphere.workspacedataservice.retry.RestClientRetry;
import org.databiosphere.workspacedataservice.service.BatchWriteService;
import org.databiosphere.workspacedataservice.service.ImportService;
import org.databiosphere.workspacedataservice.service.InstanceService;
import org.databiosphere.workspacedataservice.service.RecordOrchestratorService;
Expand Down Expand Up @@ -63,14 +58,10 @@
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class PfbQuartzJobE2ETest {

@Autowired JobDao jobDao;
@Autowired RestClientRetry restClientRetry;
@Autowired BatchWriteService batchWriteService;
@Autowired ActivityLogger activityLogger;
@Autowired RecordOrchestratorService recordOrchestratorService;
@Autowired ImportService importService;
@Autowired InstanceService instanceService;

@Autowired private PfbTestSupport testSupport;
@MockBean SchedulerDao schedulerDao;
@MockBean WorkspaceManagerDao wsmDao;

Expand Down Expand Up @@ -127,8 +118,7 @@ void importTestResource() throws IOException, JobExecutionException {
when(wsmDao.enumerateDataRepoSnapshotReferences(any(), anyInt(), anyInt()))
.thenReturn(new ResourceList());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

/* the testAvroResource should insert:
- 3202 record(s) of type activities
Expand Down Expand Up @@ -184,8 +174,7 @@ void importFourRowsResource() throws IOException, JobExecutionException {
when(wsmDao.enumerateDataRepoSnapshotReferences(any(), anyInt(), anyInt()))
.thenReturn(new ResourceList());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

/* the fourRowsAvroResource should insert:
- 3 record(s) of type data_release
Expand Down Expand Up @@ -231,8 +220,7 @@ void numberPrecision() throws IOException, JobExecutionException {
when(wsmDao.enumerateDataRepoSnapshotReferences(any(), anyInt(), anyInt()))
.thenReturn(new ResourceList());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

// this record, within the precision.avro file, is known to have numbers with high decimal
// precision. Retrieve some of them and check for the expected high-precision values.
Expand Down Expand Up @@ -271,8 +259,7 @@ void importWithForwardRelations() throws IOException, JobExecutionException {
when(wsmDao.enumerateDataRepoSnapshotReferences(any(), anyInt(), anyInt()))
.thenReturn(new ResourceList());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

/* the forwardRelationsAvroResource should insert:
- 1 record of type submitted_aligned_reads
Expand Down Expand Up @@ -321,8 +308,7 @@ void importCyclicalRelations() throws IOException, JobExecutionException {
when(wsmDao.enumerateDataRepoSnapshotReferences(any(), anyInt(), anyInt()))
.thenReturn(new ResourceList());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

RecordTypeSchema dataReleaseSchema =
recordOrchestratorService.describeRecordType(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.databiosphere.workspacedataservice.dataimport.pfb;

import static org.databiosphere.workspacedataservice.dataimport.pfb.PfbTestUtils.buildPfbQuartzJob;
import static org.databiosphere.workspacedataservice.dataimport.pfb.PfbTestUtils.stubJobContext;
import static org.databiosphere.workspacedataservice.recordstream.TwoPassStreamingWriteHandler.ImportMode.BASE_ATTRIBUTES;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand All @@ -26,7 +25,6 @@
import java.util.stream.IntStream;
import org.databiosphere.workspacedataservice.activitylog.ActivityLogger;
import org.databiosphere.workspacedataservice.dao.JobDao;
import org.databiosphere.workspacedataservice.retry.RestClientRetry;
import org.databiosphere.workspacedataservice.service.BatchWriteService;
import org.databiosphere.workspacedataservice.service.model.BatchWriteResult;
import org.databiosphere.workspacedataservice.workspacemanager.WorkspaceManagerDao;
Expand All @@ -50,7 +48,7 @@ class PfbQuartzJobTest {
@MockBean WorkspaceManagerDao wsmDao;
@MockBean BatchWriteService batchWriteService;
@MockBean ActivityLogger activityLogger;
@Autowired RestClientRetry restClientRetry;
@Autowired PfbTestSupport testSupport;

// test resources used below
@Value("classpath:avro/minimal_data.avro")
Expand All @@ -71,8 +69,7 @@ void linkAllNewSnapshots() {
.thenReturn(new ResourceList());

// call linkSnapshots
PfbQuartzJob pfbQuartzJob =
buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger);
PfbQuartzJob pfbQuartzJob = testSupport.buildPfbQuartzJob();
pfbQuartzJob.linkSnapshots(input);
// capture calls
ArgumentCaptor<SnapshotModel> argumentCaptor = ArgumentCaptor.forClass(SnapshotModel.class);
Expand Down Expand Up @@ -100,8 +97,7 @@ void linkNothingWhenAllExist() {
.thenReturn(resourceList);

// call linkSnapshots
PfbQuartzJob pfbQuartzJob =
buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger);
PfbQuartzJob pfbQuartzJob = testSupport.buildPfbQuartzJob();
pfbQuartzJob.linkSnapshots(input);
// should not call WSM's create-snapshot-reference at all
verify(wsmDao, times(0)).linkSnapshotForPolicy(any());
Expand Down Expand Up @@ -129,8 +125,7 @@ void linkSomeWhenSomeExist() {
.thenReturn(resourceList);

// call linkSnapshots
PfbQuartzJob pfbQuartzJob =
buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger);
PfbQuartzJob pfbQuartzJob = testSupport.buildPfbQuartzJob();
pfbQuartzJob.linkSnapshots(input);

// should call WSM's create-snapshot-reference only for the references that didn't already exist
Expand Down Expand Up @@ -158,8 +153,7 @@ void doNotFailOnMissingSnapshotId() throws JobExecutionException, IOException {
when(batchWriteService.batchWritePfbStream(any(), any(), any(), eq(BASE_ATTRIBUTES)))
.thenReturn(BatchWriteResult.empty());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

// Should not call wsm dao
verify(wsmDao, times(0)).linkSnapshotForPolicy(any());
Expand All @@ -179,8 +173,7 @@ void snapshotIdsAreParsed() throws JobExecutionException, IOException {
when(batchWriteService.batchWritePfbStream(any(), any(), any(), eq(BASE_ATTRIBUTES)))
.thenReturn(BatchWriteResult.empty());

buildPfbQuartzJob(jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger)
.execute(mockContext);
testSupport.buildPfbQuartzJob().execute(mockContext);

// The "790795c4..." UUID below is the snapshotId found in the "test.avro" resource used
// by this unit test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package org.databiosphere.workspacedataservice.dataimport.pfb;

import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.UUID;
import org.databiosphere.workspacedataservice.activitylog.ActivityLogger;
import org.databiosphere.workspacedataservice.dao.JobDao;
import org.databiosphere.workspacedataservice.retry.RestClientRetry;
import org.databiosphere.workspacedataservice.service.BatchWriteService;
import org.databiosphere.workspacedataservice.workspacemanager.WorkspaceManagerDao;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Component
class PfbTestSupport {
@Autowired private JobDao jobDao;
@Autowired private WorkspaceManagerDao wsmDao;
@Autowired private RestClientRetry restClientRetry;
@Autowired private BatchWriteService batchWriteService;
@Autowired private ActivityLogger activityLogger;
@Autowired private ObjectMapper objectMapper;

PfbQuartzJob buildPfbQuartzJob(UUID workspaceId) {
return new PfbQuartzJob(
jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger, workspaceId);
}

PfbQuartzJob buildPfbQuartzJob() {
return buildPfbQuartzJob(UUID.randomUUID());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,6 @@
import org.apache.avro.generic.GenericData;
import org.apache.avro.generic.GenericRecord;
import org.apache.avro.generic.GenericRecordBuilder;
import org.databiosphere.workspacedataservice.activitylog.ActivityLogger;
import org.databiosphere.workspacedataservice.dao.JobDao;
import org.databiosphere.workspacedataservice.retry.RestClientRetry;
import org.databiosphere.workspacedataservice.service.BatchWriteService;
import org.databiosphere.workspacedataservice.workspacemanager.WorkspaceManagerDao;
import org.mockito.Mockito;
import org.quartz.JobDataMap;
import org.quartz.JobExecutionContext;
Expand Down Expand Up @@ -173,14 +168,4 @@ public static JobExecutionContext stubJobContext(UUID jobId, Resource resource,

return mockContext;
}

public static PfbQuartzJob buildPfbQuartzJob(
JobDao jobDao,
WorkspaceManagerDao wsmDao,
RestClientRetry restClientRetry,
BatchWriteService batchWriteService,
ActivityLogger activityLogger) {
return new PfbQuartzJob(
jobDao, wsmDao, restClientRetry, batchWriteService, activityLogger, UUID.randomUUID());
}
}