From 7ea5676cb21f5d2c2273244c50ba60fbf22e5844 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 28 May 2017 23:33:54 +0200 Subject: [PATCH 01/42] Implement Couchbase repository --- pom.xml | 90 +++++++++++++++++++ .../exception/CouchMoveException.java | 10 +++ .../com/github/couchmove/pojo/ChangeLog.java | 30 +++++++ .../couchmove/pojo/CouchbaseEntity.java | 16 ++++ .../java/com/github/couchmove/pojo/Type.java | 10 +++ .../repository/CouchbaseRepository.java | 15 ++++ .../repository/CouchbaseRepositoryImpl.java | 71 +++++++++++++++ 7 files changed, 242 insertions(+) create mode 100644 pom.xml create mode 100644 src/main/java/com/github/couchmove/exception/CouchMoveException.java create mode 100644 src/main/java/com/github/couchmove/pojo/ChangeLog.java create mode 100644 src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java create mode 100644 src/main/java/com/github/couchmove/pojo/Type.java create mode 100644 src/main/java/com/github/couchmove/repository/CouchbaseRepository.java create mode 100644 src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..c39662f --- /dev/null +++ b/pom.xml @@ -0,0 +1,90 @@ + + + + 4.0.0 + + CouchMove + Couchbase data migration tool for Java + https://github.com/differentway/couchmove + + 1.8 + 2.4.5 + + + com.github + couchmove + develop-SNAPSHOT + + + + The Apache Software License, Version 2.0 + http://www.apache.org/licenses/LICENSE-2.0.txt + repo + + + + + scm:git:git@github.com:differentway/couchmove.git + scm:git:git@github.com:differentway/couchmove.git + git@github.com:differentway/couchmove.git + couchmove-tag + + + + + differentway + differentway + + + + + + org.projectlombok + lombok + 1.16.16 + provided + + + com.couchbase.client + java-client + ${couchbase.client.version} + + + org.slf4j + slf4j-api + 1.7.25 + + + com.github.adedayo.intellij.sdk + annotations-java8 + 142.1 + + + org.testcontainers + testcontainers + 1.2.1 + test + + + ch.qos.logback + logback-classic + 1.1.8 + test + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 2.4 + + ${java.version} + ${java.version} + + + + + \ No newline at end of file diff --git a/src/main/java/com/github/couchmove/exception/CouchMoveException.java b/src/main/java/com/github/couchmove/exception/CouchMoveException.java new file mode 100644 index 0000000..bd43a65 --- /dev/null +++ b/src/main/java/com/github/couchmove/exception/CouchMoveException.java @@ -0,0 +1,10 @@ +package com.github.couchmove.exception; + +/** + * Created by tayebchlyah on 28/05/2017. + */ +public class CouchMoveException extends RuntimeException { + public CouchMoveException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java new file mode 100644 index 0000000..52aa529 --- /dev/null +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -0,0 +1,30 @@ +package com.github.couchmove.pojo; + +import lombok.Builder; +import lombok.Data; + +import java.util.Date; + +/** + * Created by tayebchlyah on 27/05/2017. + */ +@Builder +@Data +public class ChangeLog extends CouchbaseEntity { + + private String version; + + private String description; + + private Type type; + + private int checksum; + + private String runner; + + private Date timestamp; + + private long duration; + + private boolean success; +} diff --git a/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java b/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java new file mode 100644 index 0000000..9c498c4 --- /dev/null +++ b/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java @@ -0,0 +1,16 @@ +package com.github.couchmove.pojo; + +import com.couchbase.client.deps.com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.jetbrains.annotations.Nullable; + +/** + * Created by tayebchlyah on 28/05/2017. + */ +@Data +public class CouchbaseEntity { + + @Nullable + @JsonIgnore + private Long cas; +} diff --git a/src/main/java/com/github/couchmove/pojo/Type.java b/src/main/java/com/github/couchmove/pojo/Type.java new file mode 100644 index 0000000..d21b306 --- /dev/null +++ b/src/main/java/com/github/couchmove/pojo/Type.java @@ -0,0 +1,10 @@ +package com.github.couchmove.pojo; + +/** + * Created by tayebchlyah on 27/05/2017. + */ +public enum Type { + JSON, + DESIGN_DOC, + N1QL +} diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java new file mode 100644 index 0000000..78356a7 --- /dev/null +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -0,0 +1,15 @@ +package com.github.couchmove.repository; + +import com.github.couchmove.pojo.CouchbaseEntity; + +/** + * Created by tayebchlyah on 27/05/2017. + */ +public interface CouchbaseRepository { + + E save(String id, E entity); + + void delete(String id); + + E findOne(String id); +} diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java new file mode 100644 index 0000000..bc3520a --- /dev/null +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -0,0 +1,71 @@ +package com.github.couchmove.repository; + +import com.couchbase.client.deps.com.fasterxml.jackson.core.JsonProcessingException; +import com.couchbase.client.deps.com.fasterxml.jackson.databind.ObjectMapper; +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.document.RawJsonDocument; +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.CouchbaseEntity; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +import java.io.IOException; + +/** + * Created by tayebchlyah on 27/05/2017. + */ +@RequiredArgsConstructor +public class CouchbaseRepositoryImpl implements CouchbaseRepository { + + @Getter(lazy = true) + private static final ObjectMapper jsonMapper = new ObjectMapper(); + + private final Bucket bucket; + + private final Class entityClass; + + /** + * If the {@link CouchbaseEntity#cas} of the entity is set, tries to replace the document with a Check And Set operation (Optimistic locking) + *

+ * otherwise if replaces the document + * + * @param id document id + * @param entity entity to save + * @return saved entity + * @throws com.couchbase.client.java.error.CASMismatchException if the cas of entity is different from existing one + */ + @Override + public E save(String id, E entity) { + try { + String content = getJsonMapper().writeValueAsString(entity); + if (entity.getCas() != null) { + bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); + } else { + bucket.upsert(RawJsonDocument.create(id, content)); + } + return entity; + } catch (JsonProcessingException e) { + throw new CouchMoveException("Unable to save document with id " + id, e); + } + } + + @Override + public void delete(String id) { + bucket.remove(id); + } + + @Override + public E findOne(String id) { + RawJsonDocument document = bucket.get(id, RawJsonDocument.class); + if (document == null) { + return null; + } + try { + E entity = getJsonMapper().readValue(document.content(), entityClass); + entity.setCas(document.cas()); + return entity; + } catch (IOException e) { + throw new CouchMoveException("Unable to read document with id " + id, e); + } + } +} From e12cdf06fdea50472def67575d33785407e976d7 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 28 May 2017 23:35:57 +0200 Subject: [PATCH 02/42] Implement tests with docker test containers --- .../container/AbstractCouchbaseTest.java | 55 ++++ .../container/CouchbaseContainer.java | 274 ++++++++++++++++++ .../container/CouchbaseWaitStrategy.java | 159 ++++++++++ .../repository/CouchbaseRepositoryTest.java | 102 +++++++ 4 files changed, 590 insertions(+) create mode 100644 src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java create mode 100644 src/test/java/com/github/couchmove/container/CouchbaseContainer.java create mode 100644 src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java create mode 100644 src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java new file mode 100644 index 0000000..9b40588 --- /dev/null +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -0,0 +1,55 @@ +package com.github.couchmove.container; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.bucket.BucketType; +import com.couchbase.client.java.cluster.DefaultBucketSettings; +import lombok.Getter; +import org.junit.ClassRule; +import org.slf4j.LoggerFactory; + +/** + * Created by tayebchlyah on 28/05/2017. + */ +public abstract class AbstractCouchbaseTest { + public static final String CLUSTER_USER = "Administrator"; + public static final String CLUSTER_PASSWORD = "password"; + public static final String DEFAULT_BUCKET = "default"; + + static { + ((Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME)).setLevel(Level.INFO); + } + + @ClassRule + public final static CouchbaseContainer couchbaseContainer = initCouchbaseContainer(); + + @Getter(lazy = true) + private final static Bucket bucket = openBucket(DEFAULT_BUCKET); + + private static CouchbaseContainer initCouchbaseContainer() { + return new CouchbaseContainer() + .withFTS(false) + .withIndex(false) + .withQuery(false) + .withClusterUsername(CLUSTER_USER) + .withClusterPassword(CLUSTER_PASSWORD) + .withNewBucket(DefaultBucketSettings.builder() + .enableFlush(true) + .name(DEFAULT_BUCKET) + .quota(100) + .replicas(0) + .type(BucketType.COUCHBASE) + .build()); + } + + private static Bucket openBucket(String bucketName) { + CouchbaseCluster cluster = couchbaseContainer.getCouchbaseCluster(); + return cluster.openBucket(bucketName); + } + + public static void flush() { + getBucket().bucketManager().flush(); + } +} diff --git a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java new file mode 100644 index 0000000..15db0b8 --- /dev/null +++ b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2016 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.couchmove.container; + +import com.couchbase.client.core.utils.Base64; +import com.couchbase.client.java.CouchbaseCluster; +import com.couchbase.client.java.cluster.BucketSettings; +import com.couchbase.client.java.env.CouchbaseEnvironment; +import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; +import com.couchbase.client.java.query.Index; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.HttpWaitStrategy; + +import java.io.DataOutputStream; +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.List; + +/** + * @author Laurent Doguin + */ +public class CouchbaseContainer> extends GenericContainer { + + private String memoryQuota = "400"; + + private String indexMemoryQuota = "400"; + + private String clusterUsername = "Administrator"; + + private String clusterPassword = "password"; + + private Boolean keyValue = true; + + private Boolean query = true; + + private Boolean index = true; + + private Boolean fts = true; + + private Boolean beerSample = false; + + private Boolean travelSample = false; + + private Boolean gamesIMSample = false; + + private CouchbaseEnvironment couchbaseEnvironment; + + private CouchbaseCluster couchbaseCluster; + + private List newBuckets = new ArrayList<>(); + + private String urlBase; + + public CouchbaseContainer() { + super("couchbase/server:latest"); + } + + public CouchbaseContainer(String containerName) { + super(containerName); + } + + @Override + protected Integer getLivenessCheckPort() { + return getMappedPort(8091); + } + + @Override + protected void configure() { + addFixedExposedPort(8091, 8091); + addFixedExposedPort(8092, 8092); + addFixedExposedPort(8093, 8093); + addFixedExposedPort(8094, 8094); + addFixedExposedPort(8095, 8095); + addFixedExposedPort(11207, 11207); + addFixedExposedPort(11210, 11210); + addFixedExposedPort(11211, 11211); + addFixedExposedPort(18091, 18091); + addFixedExposedPort(18092, 18092); + addFixedExposedPort(18093, 18093); + setWaitStrategy(new HttpWaitStrategy().forPath("/ui/index.html#/")); + } + + public CouchbaseEnvironment getCouchbaseEnvironnement() { + if (couchbaseEnvironment == null) { + initCluster(); + couchbaseEnvironment = DefaultCouchbaseEnvironment.builder() + .bootstrapCarrierDirectPort(getMappedPort(11210)) + .bootstrapCarrierSslPort(getMappedPort(11207)) + .bootstrapHttpDirectPort(getMappedPort(8091)) + .bootstrapHttpSslPort(getMappedPort(18091)) + .build(); + } + return couchbaseEnvironment; + } + + public CouchbaseCluster getCouchbaseCluster() { + if (couchbaseCluster == null) { + couchbaseCluster = CouchbaseCluster.create(getCouchbaseEnvironnement(), getContainerIpAddress()); + } + return couchbaseCluster; + } + + public SELF withClusterUsername(String username) { + this.clusterUsername = username; + return self(); + } + + public SELF withClusterPassword(String password) { + this.clusterPassword = password; + return self(); + } + + public SELF withMemoryQuota(String memoryQuota) { + this.memoryQuota = memoryQuota; + return self(); + } + + public SELF withIndexMemoryQuota(String indexMemoryQuota) { + this.indexMemoryQuota = indexMemoryQuota; + return self(); + } + + public SELF withKeyValue(Boolean withKV) { + this.keyValue = withKV; + return self(); + } + + public SELF withIndex(Boolean withIndex) { + this.index = withIndex; + return self(); + } + + public SELF withQuery(Boolean withQuery) { + this.query = withQuery; + return self(); + } + + public SELF withFTS(Boolean withFTS) { + this.fts = withFTS; + return self(); + } + + public SELF withTravelSample(Boolean withTravelSample) { + this.travelSample = withTravelSample; + return self(); + } + + public SELF withBeerSample(Boolean withBeerSample) { + this.beerSample = withBeerSample; + return self(); + } + + public SELF withGamesIMSample(Boolean withGamesIMSample) { + this.gamesIMSample = withGamesIMSample; + return self(); + } + + public SELF withNewBucket(BucketSettings bucketSettings) { + newBuckets.add(bucketSettings); + return self(); + } + + + public void initCluster() { + urlBase = String.format("http://%s:%s", getContainerIpAddress(), getMappedPort(8091)); + try { + String poolURL = "/pools/default"; + String poolPayload = "memoryQuota=" + URLEncoder.encode(memoryQuota, "UTF-8") + "&indexMemoryQuota=" + URLEncoder.encode(indexMemoryQuota, "UTF-8"); + + String setupServicesURL = "/node/controller/setupServices"; + StringBuilder servicePayloadBuilder = new StringBuilder(); + if (keyValue) { + servicePayloadBuilder.append("kv,"); + } + if (query) { + servicePayloadBuilder.append("n1ql,"); + } + if (index) { + servicePayloadBuilder.append("index,"); + } + if (fts) { + servicePayloadBuilder.append("fts,"); + } + String setupServiceContent = "services=" + URLEncoder.encode(servicePayloadBuilder.toString(), "UTF-8"); + + String webSettingsURL = "/settings/web"; + String webSettingsContent = "username=" + URLEncoder.encode(clusterUsername, "UTF-8") + "&password=" + URLEncoder.encode(clusterPassword, "UTF-8") + "&port=8091"; + + String bucketURL = "/sampleBuckets/install"; + + StringBuilder sampleBucketPayloadBuilder = new StringBuilder(); + sampleBucketPayloadBuilder.append('['); + if (travelSample) { + sampleBucketPayloadBuilder.append("\"travel-sample\","); + } + if (beerSample) { + sampleBucketPayloadBuilder.append("\"beer-sample\","); + } + if (gamesIMSample) { + sampleBucketPayloadBuilder.append("\"gamesim-sample\","); + } + sampleBucketPayloadBuilder.append(']'); + + callCouchbaseRestAPI(poolURL, poolPayload); + callCouchbaseRestAPI(setupServicesURL, setupServiceContent); + callCouchbaseRestAPI(webSettingsURL, webSettingsContent); + callCouchbaseRestAPI(bucketURL, sampleBucketPayloadBuilder.toString()); + + CouchbaseWaitStrategy s = new CouchbaseWaitStrategy(); + s.withBasicCredentials(clusterUsername, clusterPassword); + s.waitUntilReady(this); + callCouchbaseRestAPI("/settings/indexes", "indexerThreads=0&logLevel=info&maxRollbackPoints=5&storageMode=memory_optimized"); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public void createBucket(BucketSettings bucketSetting, Boolean createIndex) { + BucketSettings bucketSettings = getCouchbaseCluster().clusterManager(clusterUsername, clusterPassword).insertBucket(bucketSetting); + // allow some time for the query service to come up + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (createIndex) { + getCouchbaseCluster().openBucket().query(Index.createPrimaryIndex().on(bucketSetting.name())); + } + } + + public void callCouchbaseRestAPI(String url, String payload) throws IOException { + String fullUrl = urlBase + url; + HttpURLConnection httpConnection = (HttpURLConnection) ((new URL(fullUrl).openConnection())); + httpConnection.setDoOutput(true); + httpConnection.setRequestMethod("POST"); + httpConnection.setRequestProperty("Content-Type", + "application/x-www-form-urlencoded"); + String encoded = Base64.encode((clusterUsername + ":" + clusterPassword).getBytes("UTF-8")); + httpConnection.setRequestProperty("Authorization", "Basic " + encoded); + DataOutputStream out = new DataOutputStream(httpConnection.getOutputStream()); + out.writeBytes(payload); + out.flush(); + out.close(); + httpConnection.getResponseCode(); + httpConnection.disconnect(); + } + + @Override + public void start() { + super.start(); + if (!newBuckets.isEmpty()) { + for (BucketSettings bucketSetting : newBuckets) { + createBucket(bucketSetting, index); + } + } + } + +} diff --git a/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java b/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java new file mode 100644 index 0000000..024407d --- /dev/null +++ b/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2016 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.github.couchmove.container; + +import com.couchbase.client.deps.com.fasterxml.jackson.databind.JsonNode; +import com.couchbase.client.deps.com.fasterxml.jackson.databind.ObjectMapper; +import org.rnorth.ducttape.TimeoutException; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.shaded.com.google.common.base.Strings; +import org.testcontainers.shaded.com.google.common.io.BaseEncoding; + +import java.io.IOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URL; +import java.util.concurrent.TimeUnit; + +import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; + +/** + * Created by ldoguin on 18/07/16. + */ +public class CouchbaseWaitStrategy extends GenericContainer.AbstractWaitStrategy { + /** + * Authorization HTTP header. + */ + private static final String HEADER_AUTHORIZATION = "Authorization"; + + /** + * Basic Authorization scheme prefix. + */ + private static final String AUTH_BASIC = "Basic "; + + private String path = "/pools/default/"; + private int statusCode = HttpURLConnection.HTTP_OK; + private boolean tlsEnabled; + private String username; + private String password; + private ObjectMapper om = new ObjectMapper(); + + /** + * Indicates that the status check should use HTTPS. + * + * @return this + */ + public CouchbaseWaitStrategy usingTls() { + this.tlsEnabled = true; + return this; + } + + /** + * Authenticate with HTTP Basic Authorization credentials. + * + * @param username the username + * @param password the password + * @return this + */ + public CouchbaseWaitStrategy withBasicCredentials(String username, String password) { + this.username = username; + this.password = password; + return this; + } + + @Override + protected void waitUntilReady() { + final Integer livenessCheckPort = getLivenessCheckPort(); + if (null == livenessCheckPort) { + logger().warn("No exposed ports or mapped ports - cannot wait for status"); + return; + } + + final String uri = buildLivenessUri(livenessCheckPort).toString(); + logger().info("Waiting for {} seconds for URL: {}", startupTimeout.getSeconds(), uri); + + // try to connect to the URL + try { + retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { + getRateLimiter().doWhenReady(() -> { + try { + final HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); + + // authenticate + if (!Strings.isNullOrEmpty(username)) { + connection.setRequestProperty(HEADER_AUTHORIZATION, buildAuthString(username, password)); + connection.setUseCaches(false); + } + + connection.setRequestMethod("GET"); + connection.connect(); + + if (statusCode != connection.getResponseCode()) { + throw new RuntimeException(String.format("HTTP response code was: %s", + connection.getResponseCode())); + } + + // Specific Couchbase wait strategy to be sure the node is online and healthy + JsonNode node = om.readTree(connection.getInputStream()); + JsonNode statusNode = node.at("/nodes/0/status"); + String status = statusNode.asText(); + if (!"healthy".equals(status)){ + throw new RuntimeException(String.format("Couchbase Node status was: %s", status)); + } + + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + return true; + }); + + } catch (TimeoutException e) { + throw new ContainerLaunchException(String.format( + "Timed out waiting for URL to be accessible (%s should return HTTP %s)", uri, statusCode)); + } + } + + /** + * Build the URI on which to check if the container is ready. + * + * @param livenessCheckPort the liveness port + * @return the liveness URI + */ + private URI buildLivenessUri(int livenessCheckPort) { + final String scheme = (tlsEnabled ? "https" : "http") + "://"; + final String host = container.getContainerIpAddress(); + + final String portSuffix; + if ((tlsEnabled && 443 == livenessCheckPort) || (!tlsEnabled && 80 == livenessCheckPort)) { + portSuffix = ""; + } else { + portSuffix = ":" + String.valueOf(livenessCheckPort); + } + + return URI.create(scheme + host + portSuffix + path); + } + + /** + * @param username the username + * @param password the password + * @return a basic authentication string for the given credentials + */ + private String buildAuthString(String username, String password) { + return AUTH_BASIC + BaseEncoding.base64().encode((username + ":" + password).getBytes()); + } +} \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java new file mode 100644 index 0000000..0b7a65f --- /dev/null +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -0,0 +1,102 @@ +package com.github.couchmove.repository; + +import com.couchbase.client.java.error.CASMismatchException; +import com.github.couchmove.container.AbstractCouchbaseTest; +import com.github.couchmove.pojo.ChangeLog; +import org.jetbrains.annotations.NotNull; +import org.junit.After; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Random; +import java.util.UUID; + +/** + * Created by tayebchlyah on 28/05/2017. + */ +public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { + + private static CouchbaseRepository repository; + + private static CouchbaseRepository initializeRepository() { + return new CouchbaseRepositoryImpl<>(AbstractCouchbaseTest.getBucket(), ChangeLog.class); + } + + @BeforeClass + public static void setUp() { + repository = initializeRepository(); + } + + @After + public void clear() { + AbstractCouchbaseTest.flush(); + } + + @Test + public void should_save_and_get_entity() { + // Given a changeLog + ChangeLog changeLog = getRandomChangeLog(); + + // When we insert it with an id + String id = getRandomString(); + repository.save(id, changeLog); + + // Then we should get it by this id + ChangeLog result = repository.findOne(id); + + Assert.assertNotNull(result); + Assert.assertEquals(changeLog, result); + } + + @Test + public void should_delete_entity() { + // Given a changeLog saved on couchbase + ChangeLog changeLog = getRandomChangeLog(); + + String id = getRandomString(); + repository.save(id, changeLog); + Assert.assertNotNull(repository.findOne(id)); + + // When we delete it + repository.delete(id); + + // Then we no longer should get it + Assert.assertNull(repository.findOne(id)); + } + + @Test(expected = CASMismatchException.class) + public void should_not_insert_entity_with_different_cas() { + // Given a changeLog saved on couchbase + ChangeLog changeLog = getRandomChangeLog(); + String id = getRandomString(); + repository.save(id, changeLog); + ChangeLog savedChangeLog = repository.findOne(id); + + // Then it should have a cas + Assert.assertNotNull(savedChangeLog.getCas()); + + // When we change this cas + savedChangeLog.setCas(new Random().nextLong()); + + // Then we should have exception upon saving + repository.save(id, savedChangeLog); + } + + @NotNull + private String getRandomString() { + return UUID.randomUUID().toString(); + } + + @NotNull + private ChangeLog getRandomChangeLog() { + Random random = new Random(); + return ChangeLog.builder() + .description(getRandomString()) + .duration(random.nextInt()) + .checksum(random.nextInt()) + .success(true) + .build(); + } + +} \ No newline at end of file From 5f72272925e01e8c719d14685a09a396c1a1e325 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Mon, 29 May 2017 23:00:11 +0200 Subject: [PATCH 03/42] Implement optimistic locking with CAS operation (Check And Set) --- .../repository/CouchbaseRepository.java | 2 ++ .../repository/CouchbaseRepositoryImpl.java | 26 ++++++++++++++++--- .../repository/CouchbaseRepositoryTest.java | 25 +++++++++++++++--- 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java index 78356a7..5b3583b 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -9,6 +9,8 @@ public interface CouchbaseRepository { E save(String id, E entity); + E checkAndSave(String id, E entity); + void delete(String id); E findOne(String id); diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index bc3520a..1b7c1f8 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -24,24 +24,42 @@ public class CouchbaseRepositoryImpl implements Couch private final Class entityClass; + /** + * Save the entity on couchbase + * + * @param id document id + * @param entity entity to save + * @return saved entity + */ + @Override + public E save(String id, E entity) { + try { + bucket.upsert(RawJsonDocument.create(id, getJsonMapper().writeValueAsString(entity))); + return entity; + } catch (JsonProcessingException e) { + throw new CouchMoveException("Unable to save document with id " + id, e); + } + } + /** * If the {@link CouchbaseEntity#cas} of the entity is set, tries to replace the document with a Check And Set operation (Optimistic locking) *

- * otherwise if replaces the document + * otherwise it insert the document * * @param id document id * @param entity entity to save * @return saved entity - * @throws com.couchbase.client.java.error.CASMismatchException if the cas of entity is different from existing one + * @throws com.couchbase.client.java.error.CASMismatchException if the cas of entity is different from existing one + * @throws com.couchbase.client.java.error.DocumentAlreadyExistsException if the cas is not set and the document exists on couchbase */ @Override - public E save(String id, E entity) { + public E checkAndSave(String id, E entity) { try { String content = getJsonMapper().writeValueAsString(entity); if (entity.getCas() != null) { bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); } else { - bucket.upsert(RawJsonDocument.create(id, content)); + bucket.insert(RawJsonDocument.create(id, content)); } return entity; } catch (JsonProcessingException e) { diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 0b7a65f..bd6e1df 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -1,6 +1,7 @@ package com.github.couchmove.repository; import com.couchbase.client.java.error.CASMismatchException; +import com.couchbase.client.java.error.DocumentAlreadyExistsException; import com.github.couchmove.container.AbstractCouchbaseTest; import com.github.couchmove.pojo.ChangeLog; import org.jetbrains.annotations.NotNull; @@ -65,31 +66,46 @@ public void should_delete_entity() { Assert.assertNull(repository.findOne(id)); } + @Test(expected = DocumentAlreadyExistsException.class) + public void should_not_replace_entity_without_cas() { + // Given a changeLog saved on couchbase + ChangeLog changeLog = getRandomChangeLog(); + String id = getRandomString(); + repository.save(id, changeLog); + + // When we tries to insert it without cas + changeLog.setCas(null); + + // Then we should have exception upon saving with cas operation + repository.checkAndSave(id, changeLog); + } + @Test(expected = CASMismatchException.class) public void should_not_insert_entity_with_different_cas() { // Given a changeLog saved on couchbase ChangeLog changeLog = getRandomChangeLog(); String id = getRandomString(); repository.save(id, changeLog); - ChangeLog savedChangeLog = repository.findOne(id); // Then it should have a cas + ChangeLog savedChangeLog = repository.findOne(id); Assert.assertNotNull(savedChangeLog.getCas()); // When we change this cas savedChangeLog.setCas(new Random().nextLong()); // Then we should have exception upon saving - repository.save(id, savedChangeLog); + repository.checkAndSave(id, savedChangeLog); } + // @NotNull - private String getRandomString() { + private static String getRandomString() { return UUID.randomUUID().toString(); } @NotNull - private ChangeLog getRandomChangeLog() { + private static ChangeLog getRandomChangeLog() { Random random = new Random(); return ChangeLog.builder() .description(getRandomString()) @@ -98,5 +114,6 @@ private ChangeLog getRandomChangeLog() { .success(true) .build(); } + // } \ No newline at end of file From 67f5936cf008b93d58ed12fb6afd5e4f944e6c1f Mon Sep 17 00:00:00 2001 From: ctayeb Date: Tue, 30 May 2017 22:47:43 +0200 Subject: [PATCH 04/42] Implement lock service --- .../com/github/couchmove/pojo/ChangeLock.java | 20 ++++ .../couchmove/service/ChangeLockService.java | 110 ++++++++++++++++++ .../container/AbstractCouchbaseTest.java | 21 ++-- .../repository/CouchbaseRepositoryTest.java | 12 +- .../service/ChangeLockServiceTest.java | 46 ++++++++ 5 files changed, 189 insertions(+), 20 deletions(-) create mode 100644 src/main/java/com/github/couchmove/pojo/ChangeLock.java create mode 100644 src/main/java/com/github/couchmove/service/ChangeLockService.java create mode 100644 src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLock.java b/src/main/java/com/github/couchmove/pojo/ChangeLock.java new file mode 100644 index 0000000..6f42811 --- /dev/null +++ b/src/main/java/com/github/couchmove/pojo/ChangeLock.java @@ -0,0 +1,20 @@ +package com.github.couchmove.pojo; + +import lombok.Data; + +import java.util.Date; + +/** + * Created by tayebchlyah on 27/05/2017. + */ +@Data +public class ChangeLock extends CouchbaseEntity { + + private boolean locked; + + private String uuid; + + private String runner; + + private Date timestamp; +} diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java new file mode 100644 index 0000000..e8977ce --- /dev/null +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -0,0 +1,110 @@ +package com.github.couchmove.service; + +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.error.CASMismatchException; +import com.couchbase.client.java.error.DocumentAlreadyExistsException; +import com.github.couchmove.pojo.ChangeLock; +import com.github.couchmove.repository.CouchbaseRepository; +import com.github.couchmove.repository.CouchbaseRepositoryImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.Date; +import java.util.UUID; + +/** + * Created by tayebchlyah on 28/05/2017. + */ +public class ChangeLockService { + + private static Logger logger = LoggerFactory.getLogger(ChangeLockService.class); + + private static final String LOCK_ID = "DATABASE_CHANGELOG_LOCK"; + + private final CouchbaseRepository changeLockRepository; + + private String uuid; + + public ChangeLockService(Bucket bucket) { + this.changeLockRepository = new CouchbaseRepositoryImpl<>(bucket, ChangeLock.class); + } + + public boolean acquireLock() { + logger.info("Trying to acquire database lock..."); + // Verify if there is any lock on database + ChangeLock lock = changeLockRepository.findOne(LOCK_ID); + // If none, create one + if (lock == null) { + lock = new ChangeLock(); + } else if (lock.isLocked()) { + logger.warn("The database is already locked by '{}'", lock.getRunner()); + return false; + } + // Create Lock information + lock.setLocked(true); + lock.setTimestamp(new Date()); + lock.setRunner(getUsername()); + lock.setUuid(uuid = UUID.randomUUID().toString()); + // Tries to save it with Optimistic locking + try { + changeLockRepository.checkAndSave(LOCK_ID, lock); + } catch (CASMismatchException | DocumentAlreadyExistsException e) { + // In case of exception, this means an other process got the lock, logging its information + lock = changeLockRepository.findOne(LOCK_ID); + logger.warn("The database is already locked by '{}'", lock.getRunner()); + return false; + } + logger.info("Lock acquired"); + return true; + } + + public boolean isLockAcquired() { + ChangeLock lock = changeLockRepository.findOne(LOCK_ID); + if (lock == null) { + return false; + } + if (!lock.isLocked()) { + return false; + } + if (lock.getUuid() == null || !lock.getUuid().equals(uuid)) { + logger.warn("Lock is acquired by another process"); + return false; + } + return true; + } + + public void releaseLock() { + changeLockRepository.delete(LOCK_ID); + } + + // + private static String getUsername() { + String osName = System.getProperty("os.name").toLowerCase(); + String className = null; + String methodName = "getUsername"; + + if (osName.contains("windows")) { + className = "com.sun.security.auth.module.NTSystem"; + methodName = "getName"; + } else if (osName.contains("linux") || osName.contains("mac")) { + className = "com.sun.security.auth.module.UnixSystem"; + } else if (osName.contains("solaris") || osName.contains("sunos")) { + className = "com.sun.security.auth.module.SolarisSystem"; + } + + if (className != null) { + try { + Class c = Class.forName(className); + Method method = c.getDeclaredMethod(methodName); + Object o = c.newInstance(); + return method.invoke(o).toString(); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { + logger.error("Unable to get actual user name", e); + } + } + return "unknown"; + } + // +} diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index 9b40588..137e620 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -7,7 +7,7 @@ import com.couchbase.client.java.bucket.BucketType; import com.couchbase.client.java.cluster.DefaultBucketSettings; import lombok.Getter; -import org.junit.ClassRule; +import org.junit.After; import org.slf4j.LoggerFactory; /** @@ -22,14 +22,19 @@ public abstract class AbstractCouchbaseTest { ((Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME)).setLevel(Level.INFO); } - @ClassRule - public final static CouchbaseContainer couchbaseContainer = initCouchbaseContainer(); + @Getter(lazy = true) + private final static CouchbaseContainer couchbaseContainer = initCouchbaseContainer(); @Getter(lazy = true) private final static Bucket bucket = openBucket(DEFAULT_BUCKET); + @After + public void clear() { + getBucket().bucketManager().flush(); + } + private static CouchbaseContainer initCouchbaseContainer() { - return new CouchbaseContainer() + CouchbaseContainer couchbaseContainer = new CouchbaseContainer() .withFTS(false) .withIndex(false) .withQuery(false) @@ -42,14 +47,12 @@ private static CouchbaseContainer initCouchbaseContainer() { .replicas(0) .type(BucketType.COUCHBASE) .build()); + couchbaseContainer.start(); + return couchbaseContainer; } private static Bucket openBucket(String bucketName) { - CouchbaseCluster cluster = couchbaseContainer.getCouchbaseCluster(); + CouchbaseCluster cluster = getCouchbaseContainer().getCouchbaseCluster(); return cluster.openBucket(bucketName); } - - public static void flush() { - getBucket().bucketManager().flush(); - } } diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index bd6e1df..1dac7e4 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -5,7 +5,6 @@ import com.github.couchmove.container.AbstractCouchbaseTest; import com.github.couchmove.pojo.ChangeLog; import org.jetbrains.annotations.NotNull; -import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -20,18 +19,9 @@ public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { private static CouchbaseRepository repository; - private static CouchbaseRepository initializeRepository() { - return new CouchbaseRepositoryImpl<>(AbstractCouchbaseTest.getBucket(), ChangeLog.class); - } - @BeforeClass public static void setUp() { - repository = initializeRepository(); - } - - @After - public void clear() { - AbstractCouchbaseTest.flush(); + repository = new CouchbaseRepositoryImpl<>(AbstractCouchbaseTest.getBucket(), ChangeLog.class); } @Test diff --git a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java new file mode 100644 index 0000000..bca5439 --- /dev/null +++ b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java @@ -0,0 +1,46 @@ +package com.github.couchmove.service; + +import com.github.couchmove.container.AbstractCouchbaseTest; +import org.junit.Assert; +import org.junit.Test; + +/** + * Created by tayebchlyah on 29/05/2017. + */ +public class ChangeLockServiceTest extends AbstractCouchbaseTest { + + @Test + public void should_acquire_and_release_lock() { + // Given a changeLockService + ChangeLockService changeLockService = new ChangeLockService(getBucket()); + + // When we tries to acquire lock + Assert.assertTrue(changeLockService.acquireLock()); + + // Then we should get it + Assert.assertTrue(changeLockService.isLockAcquired()); + + // When we release the lock + changeLockService.releaseLock(); + + // The it should be released + Assert.assertFalse(changeLockService.isLockAcquired()); + } + + @Test + public void should_not_acquire_lock_when_already_acquired() { + // Given a first changeLockService that acquires lock + ChangeLockService changeLockService1 = new ChangeLockService(getBucket()); + changeLockService1.acquireLock(); + + // When an other changeLockService tries to get the lock + ChangeLockService changeLockService2 = new ChangeLockService(getBucket()); + + // Then it will fails + Assert.assertFalse(changeLockService2.acquireLock()); + Assert.assertFalse(changeLockService2.isLockAcquired()); + + // And the first service should keep the lock + Assert.assertTrue(changeLockService1.isLockAcquired()); + } +} \ No newline at end of file From 369f1aa4d67822a5d6a937bf9261a4a7d95b32ff Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 3 Jun 2017 10:16:21 +0200 Subject: [PATCH 05/42] Implement fetching ChangeLogs from files --- pom.xml | 22 +++++ .../exception/CouchMoveException.java | 4 + .../com/github/couchmove/pojo/ChangeLog.java | 14 ++- .../java/com/github/couchmove/pojo/Type.java | 2 +- .../service/ChangeLogFileService.java | 95 ++++++++++++++++++ .../com/github/couchmove/utils/FileUtils.java | 99 +++++++++++++++++++ .../repository/CouchbaseRepositoryTest.java | 13 +-- .../service/ChangeLogFileServiceTest.java | 82 +++++++++++++++ .../github/couchmove/utils/FileUtilsTest.java | 56 +++++++++++ .../com/github/couchmove/utils/TestUtils.java | 19 ++++ .../V1.1__insert_users/user::titi.json | 5 + .../V1.1__insert_users/user::toto.json | 5 + .../db/migration/V1__create_index.n1ql | 2 + src/test/resources/db/migration/V2__user.json | 7 ++ 14 files changed, 413 insertions(+), 12 deletions(-) create mode 100644 src/main/java/com/github/couchmove/service/ChangeLogFileService.java create mode 100644 src/main/java/com/github/couchmove/utils/FileUtils.java create mode 100644 src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java create mode 100644 src/test/java/com/github/couchmove/utils/FileUtilsTest.java create mode 100644 src/test/java/com/github/couchmove/utils/TestUtils.java create mode 100644 src/test/resources/db/migration/V1.1__insert_users/user::titi.json create mode 100644 src/test/resources/db/migration/V1.1__insert_users/user::toto.json create mode 100644 src/test/resources/db/migration/V1__create_index.n1ql create mode 100644 src/test/resources/db/migration/V2__user.json diff --git a/pom.xml b/pom.xml index c39662f..890438e 100644 --- a/pom.xml +++ b/pom.xml @@ -60,6 +60,28 @@ annotations-java8 142.1 + + commons-codec + commons-codec + 1.10 + + + commons-io + commons-io + 2.5 + + + junit + junit + 4.12 + test + + + com.tngtech.java + junit-dataprovider + 1.10.0 + test + org.testcontainers testcontainers diff --git a/src/main/java/com/github/couchmove/exception/CouchMoveException.java b/src/main/java/com/github/couchmove/exception/CouchMoveException.java index bd43a65..0336f25 100644 --- a/src/main/java/com/github/couchmove/exception/CouchMoveException.java +++ b/src/main/java/com/github/couchmove/exception/CouchMoveException.java @@ -7,4 +7,8 @@ public class CouchMoveException extends RuntimeException { public CouchMoveException(String message, Throwable cause) { super(message, cause); } + + public CouchMoveException(String message) { + super(message); + } } diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java index 52aa529..7e26500 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLog.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -2,6 +2,7 @@ import lombok.Builder; import lombok.Data; +import org.jetbrains.annotations.NotNull; import java.util.Date; @@ -10,15 +11,19 @@ */ @Builder @Data -public class ChangeLog extends CouchbaseEntity { +public class ChangeLog extends CouchbaseEntity implements Comparable { private String version; + private int order; + private String description; private Type type; - private int checksum; + private String script; + + private String checksum; private String runner; @@ -27,4 +32,9 @@ public class ChangeLog extends CouchbaseEntity { private long duration; private boolean success; + + @Override + public int compareTo(@NotNull ChangeLog o) { + return version == null ? 0 : version.compareTo(o.version); + } } diff --git a/src/main/java/com/github/couchmove/pojo/Type.java b/src/main/java/com/github/couchmove/pojo/Type.java index d21b306..b68fb8e 100644 --- a/src/main/java/com/github/couchmove/pojo/Type.java +++ b/src/main/java/com/github/couchmove/pojo/Type.java @@ -4,7 +4,7 @@ * Created by tayebchlyah on 27/05/2017. */ public enum Type { - JSON, + DOCUMENTS, DESIGN_DOC, N1QL } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java new file mode 100644 index 0000000..2bacceb --- /dev/null +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -0,0 +1,95 @@ +package com.github.couchmove.service; + +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Type; +import com.github.couchmove.utils.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.jetbrains.annotations.NotNull; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.file.Path; +import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Created by tayebchlyah on 30/05/2017. + */ +public class ChangeLogFileService { + + private static final Logger logger = LoggerFactory.getLogger(ChangeLogFileService.class); + public static final String JSON = "json"; + public static final String N1QL = "n1ql"; + + private static Pattern fileNamePattern = Pattern.compile("V([\\w.]+)__([\\w ]+)\\.?(\\w*)$"); + + private final File changeFolder; + + public ChangeLogFileService(String changePath) { + this.changeFolder = initializeFolder(changePath); + } + + public List fetch() { + logger.info("Fetching changeLogs from '{}'", changeFolder.getPath()); + SortedSet sortedChangeLogs = new TreeSet<>(); + //noinspection ConstantConditions + for (File file : changeFolder.listFiles()) { + String fileName = file.getName(); + Matcher matcher = fileNamePattern.matcher(fileName); + if (matcher.matches()) { + ChangeLog changeLog = ChangeLog.builder() + .version(matcher.group(1)) + .script(fileName) + .description(matcher.group(2).replace("_", " ")) + .type(getChangeLogType(file)) + .checksum(FileUtils.calculateChecksum(file, JSON, N1QL)) + .build(); + logger.debug("Fetched one : {}", changeLog); + sortedChangeLogs.add(changeLog); + } + } + logger.info("Fetched {} changeLogs", sortedChangeLogs.size()); + return Collections.unmodifiableList(new ArrayList<>(sortedChangeLogs)); + } + + // + static File initializeFolder(String changePath) { + Path path; + try { + path = FileUtils.getPathFromResource(changePath); + } catch (FileNotFoundException e) { + throw new CouchMoveException("The change path '" + changePath + "'doesn't exist"); + } catch (IOException e) { + throw new CouchMoveException("Unable to get change path '" + changePath + "'", e); + } + File file = path.toFile(); + if (!file.isDirectory()) { + throw new CouchMoveException("The change path '" + changePath + "' is not a directory"); + } + return file; + } + + @NotNull + static Type getChangeLogType(File file) { + if (file.isDirectory()) { + return Type.DOCUMENTS; + } else { + String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); + switch (extension) { + case N1QL: + return Type.N1QL; + case JSON: + return Type.DESIGN_DOC; + } + } + throw new CouchMoveException("Unknown ChangeLog type : " + file.getName()); + } + + // + +} diff --git a/src/main/java/com/github/couchmove/utils/FileUtils.java b/src/main/java/com/github/couchmove/utils/FileUtils.java new file mode 100644 index 0000000..9d61a74 --- /dev/null +++ b/src/main/java/com/github/couchmove/utils/FileUtils.java @@ -0,0 +1,99 @@ +package com.github.couchmove.utils; + +import com.github.couchmove.exception.CouchMoveException; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.io.FilenameUtils; +import org.jetbrains.annotations.NotNull; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; + +import static org.apache.commons.io.IOUtils.toByteArray; + +/** + * Created by tayebchlyah on 02/06/2017. + */ +public class FileUtils { + + /** + * Returns Path of a resource in classpath no matter whether in a jar or in a folder + * + * @param resource path + * @return Path of a resource + * @throws IOException + */ + public static Path getPathFromResource(String resource) throws IOException { + File file = new File(resource); + if (file.exists()) { + return file.toPath(); + } + URL resourceURL = Thread.currentThread().getContextClassLoader().getResource(resource); + if (resourceURL == null) { + resourceURL = FileUtils.class.getResource(resource); + } + if (resourceURL == null) { + throw new FileNotFoundException(resource); + } + URI uri; + try { + uri = resourceURL.toURI(); + } catch (URISyntaxException e) { + // Can not happen normally + throw new RuntimeException(e); + } + Path folder; + if (uri.getScheme().equals("jar")) { + FileSystem fileSystem; + try { + fileSystem = FileSystems.getFileSystem(uri); + } catch (FileSystemNotFoundException e) { + fileSystem = FileSystems.newFileSystem(uri, Collections.emptyMap()); + } + folder = fileSystem.getPath(resource); + } else { + folder = Paths.get(uri); + } + return folder; + } + + /** + * If the file is a Directory, calculate the checksum of all files in this directory (one level) + * Else, calculate the checksum of the file matching extensions + * + * @param file file or folder + * @param extensions of files to calculate checksum of + * @return checksum + */ + public static String calculateChecksum(@NotNull File file, String... extensions) { + if (file == null || !file.exists()) { + throw new CouchMoveException("File is null or doesn't exists"); + } + if (file.isDirectory()) { + //noinspection ConstantConditions + return Arrays.stream(file.listFiles()) + .filter(File::isFile) + .filter(f -> Arrays.stream(extensions) + .anyMatch(extension -> FilenameUtils + .getExtension(f.getName()).toLowerCase() + .equals(extension.toLowerCase()))) + .sorted(Comparator.comparing(File::getName)) + .map(FileUtils::calculateChecksum) + .reduce(String::concat) + .map(DigestUtils::sha256Hex) + .orElse(null); + } + try { + return DigestUtils.sha256Hex(toByteArray(file.toURI())); + } catch (IOException e) { + throw new CouchMoveException("Unable to calculate file checksum '" + file.getName() + "'"); + } + } +} diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 1dac7e4..5045ddf 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -10,7 +10,8 @@ import org.junit.Test; import java.util.Random; -import java.util.UUID; + +import static com.github.couchmove.utils.TestUtils.getRandomString; /** * Created by tayebchlyah on 28/05/2017. @@ -89,18 +90,12 @@ public void should_not_insert_entity_with_different_cas() { } // - @NotNull - private static String getRandomString() { - return UUID.randomUUID().toString(); - } - @NotNull private static ChangeLog getRandomChangeLog() { - Random random = new Random(); return ChangeLog.builder() .description(getRandomString()) - .duration(random.nextInt()) - .checksum(random.nextInt()) + .duration(new Random().nextInt()) + .checksum(getRandomString()) .success(true) .build(); } diff --git a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java new file mode 100644 index 0000000..df34195 --- /dev/null +++ b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java @@ -0,0 +1,82 @@ +package com.github.couchmove.service; + +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Type; +import org.apache.commons.io.FileUtils; +import org.junit.Assert; +import org.junit.Test; + +import java.io.File; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.github.couchmove.utils.TestUtils.getRandomString; + +/** + * Created by tayebchlyah on 01/06/2017. + */ +public class ChangeLogFileServiceTest { + + @Test(expected = CouchMoveException.class) + public void should_fail_if_path_does_not_exists() { + String folderPath; + //noinspection StatementWithEmptyBody + while (new File(folderPath = getRandomString()).exists()) ; + ChangeLogFileService.initializeFolder(folderPath); + } + + @Test(expected = CouchMoveException.class) + public void should_fail_if_path_is_not_directory() throws Exception { + File tempFile = File.createTempFile(getRandomString(), ""); + tempFile.deleteOnExit(); + ChangeLogFileService.initializeFolder(tempFile.getPath()); + } + + @Test + public void should_get_right_type_from_file() { + // For folder + Assert.assertEquals(Type.DOCUMENTS, ChangeLogFileService.getChangeLogType(FileUtils.getTempDirectory())); + // For JSON file + Assert.assertEquals(Type.DESIGN_DOC, ChangeLogFileService.getChangeLogType(new File("toto.json"))); + Assert.assertEquals(Type.DESIGN_DOC, ChangeLogFileService.getChangeLogType(new File("toto.JSON"))); + // For N1QL files + Assert.assertEquals(Type.N1QL, ChangeLogFileService.getChangeLogType(new File("toto.n1ql"))); + Assert.assertEquals(Type.N1QL, ChangeLogFileService.getChangeLogType(new File("toto.N1QL"))); + } + + @Test(expected = CouchMoveException.class) + public void should_throw_exception_when_unknown_file_type() { + ChangeLogFileService.getChangeLogType(new File("toto")); + } + + @Test + public void should_fetch_changeLogs() { + List changeLogs = Stream.of( + ChangeLog.builder() + .type(Type.N1QL) + .script("V1__create_index.n1ql") + .version("1") + .description("create index") + .checksum("bf0dae5d8fb638627eeabfb4b649d6100a8960da18859e12874e61063fbb16be") + .build(), + ChangeLog.builder() + .type(Type.DOCUMENTS) + .script("V1.1__insert_users") + .version("1.1") + .description("insert users") + .checksum("99a4aaf12e7505286afe2a5b074f7ebabd496f3ea8c4093116efd3d096c430a8") + .build(), + ChangeLog.builder() + .type(Type.DESIGN_DOC) + .script("V2__user.json") + .version("2") + .description("user") + .checksum("22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79") + .build()) + .collect(Collectors.toList()); + Assert.assertEquals(changeLogs, new ChangeLogFileService("db/migration").fetch()); + } + +} \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java new file mode 100644 index 0000000..2e726d5 --- /dev/null +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -0,0 +1,56 @@ +package com.github.couchmove.utils; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.nio.file.Path; + +import static com.github.couchmove.service.ChangeLogFileService.JSON; +import static com.github.couchmove.service.ChangeLogFileService.N1QL; + +/** + * Created by tayebchlyah on 02/06/2017. + */ +@RunWith(DataProviderRunner.class) +public class FileUtilsTest { + + public static final String DB_MIGRATION_PATH = "db/migration/"; + + @Test + public void should_get_file_path_from_resource() throws Exception { + Path path = FileUtils.getPathFromResource(DB_MIGRATION_PATH + "V2__user.json"); + Assert.assertNotNull(path); + File file = path.toFile(); + Assert.assertTrue(file.exists()); + Assert.assertTrue(file.isFile()); + } + + @Test + public void should_get_folder_path_from_resource() throws Exception { + Path path = FileUtils.getPathFromResource(DB_MIGRATION_PATH); + Assert.assertNotNull(path); + File file = path.toFile(); + Assert.assertTrue(file.exists()); + Assert.assertTrue(file.isDirectory()); + } + + @DataProvider + public static Object[][] fileProvider() { + return new Object[][]{ + {DB_MIGRATION_PATH + "V1.1__insert_users", "99a4aaf12e7505286afe2a5b074f7ebabd496f3ea8c4093116efd3d096c430a8"}, + {DB_MIGRATION_PATH + "V1__create_index.n1ql", "bf0dae5d8fb638627eeabfb4b649d6100a8960da18859e12874e61063fbb16be"}, + {DB_MIGRATION_PATH + "V2__user.json", "22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79"} + }; + } + + @Test + @UseDataProvider("fileProvider") + public void should_calculate_checksum_of_file_or_folder(String path, String expectedChecksum) throws Exception { + Assert.assertEquals(path, expectedChecksum, FileUtils.calculateChecksum(FileUtils.getPathFromResource(path).toFile(), JSON, N1QL)); + } +} \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java new file mode 100644 index 0000000..8abe570 --- /dev/null +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -0,0 +1,19 @@ +package com.github.couchmove.utils; + +import org.jetbrains.annotations.NotNull; + +import java.util.UUID; + +import static org.apache.commons.io.IOUtils.toByteArray; + +/** + * Created by tayebchlyah on 01/06/2017. + */ +public class TestUtils { + + @NotNull + public static String getRandomString() { + return UUID.randomUUID().toString(); + } + +} diff --git a/src/test/resources/db/migration/V1.1__insert_users/user::titi.json b/src/test/resources/db/migration/V1.1__insert_users/user::titi.json new file mode 100644 index 0000000..d97cf56 --- /dev/null +++ b/src/test/resources/db/migration/V1.1__insert_users/user::titi.json @@ -0,0 +1,5 @@ +{ + "type": "user", + "username": "titi", + "birthday": "01/09/1998" +} \ No newline at end of file diff --git a/src/test/resources/db/migration/V1.1__insert_users/user::toto.json b/src/test/resources/db/migration/V1.1__insert_users/user::toto.json new file mode 100644 index 0000000..38503c9 --- /dev/null +++ b/src/test/resources/db/migration/V1.1__insert_users/user::toto.json @@ -0,0 +1,5 @@ +{ + "type": "user", + "username": "toto", + "birthday": "10/01/1991" +} \ No newline at end of file diff --git a/src/test/resources/db/migration/V1__create_index.n1ql b/src/test/resources/db/migration/V1__create_index.n1ql new file mode 100644 index 0000000..a5e1ba6 --- /dev/null +++ b/src/test/resources/db/migration/V1__create_index.n1ql @@ -0,0 +1,2 @@ +CREATE INDEX 'user_index' ON default + WHERE type = 'user' \ No newline at end of file diff --git a/src/test/resources/db/migration/V2__user.json b/src/test/resources/db/migration/V2__user.json new file mode 100644 index 0000000..09959be --- /dev/null +++ b/src/test/resources/db/migration/V2__user.json @@ -0,0 +1,7 @@ +{ + "views": { + "findUser": { + "map": "function (doc, meta) {\n if (doc.type == \"user\") {\n emit(doc.username, null);\n } \n}" + } + } +} \ No newline at end of file From c42411e9dfd8aff50b81e61256fec84045c73279 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 3 Jun 2017 16:54:20 +0200 Subject: [PATCH 06/42] Add extension to ChangeLog Type --- .../java/com/github/couchmove/pojo/Type.java | 15 ++++++++++--- .../service/ChangeLogFileService.java | 17 +++++++------- .../repository/CouchbaseRepositoryTest.java | 22 +++++-------------- .../github/couchmove/utils/FileUtilsTest.java | 7 +++--- .../com/github/couchmove/utils/TestUtils.java | 22 +++++++++++++++++-- 5 files changed, 50 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/github/couchmove/pojo/Type.java b/src/main/java/com/github/couchmove/pojo/Type.java index b68fb8e..6b6e439 100644 --- a/src/main/java/com/github/couchmove/pojo/Type.java +++ b/src/main/java/com/github/couchmove/pojo/Type.java @@ -1,10 +1,19 @@ package com.github.couchmove.pojo; +import lombok.Getter; + /** * Created by tayebchlyah on 27/05/2017. */ public enum Type { - DOCUMENTS, - DESIGN_DOC, - N1QL + DOCUMENTS(""), + DESIGN_DOC("json"), + N1QL("n1ql"); + + @Getter + private final String extension; + + Type(String extension) { + this.extension = extension; + } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index 2bacceb..fe760f1 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -17,14 +17,15 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.github.couchmove.pojo.Type.DESIGN_DOC; +import static com.github.couchmove.pojo.Type.N1QL; + /** * Created by tayebchlyah on 30/05/2017. */ public class ChangeLogFileService { private static final Logger logger = LoggerFactory.getLogger(ChangeLogFileService.class); - public static final String JSON = "json"; - public static final String N1QL = "n1ql"; private static Pattern fileNamePattern = Pattern.compile("V([\\w.]+)__([\\w ]+)\\.?(\\w*)$"); @@ -47,7 +48,7 @@ public List fetch() { .script(fileName) .description(matcher.group(2).replace("_", " ")) .type(getChangeLogType(file)) - .checksum(FileUtils.calculateChecksum(file, JSON, N1QL)) + .checksum(FileUtils.calculateChecksum(file, DESIGN_DOC.getExtension(), N1QL.getExtension())) .build(); logger.debug("Fetched one : {}", changeLog); sortedChangeLogs.add(changeLog); @@ -80,11 +81,11 @@ static Type getChangeLogType(File file) { return Type.DOCUMENTS; } else { String extension = FilenameUtils.getExtension(file.getName()).toLowerCase(); - switch (extension) { - case N1QL: - return Type.N1QL; - case JSON: - return Type.DESIGN_DOC; + if (DESIGN_DOC.getExtension().equals(extension)) { + return DESIGN_DOC; + } + if (N1QL.getExtension().equals(extension)) { + return N1QL; } } throw new CouchMoveException("Unknown ChangeLog type : " + file.getName()); diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 5045ddf..3a6f3ea 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -4,7 +4,7 @@ import com.couchbase.client.java.error.DocumentAlreadyExistsException; import com.github.couchmove.container.AbstractCouchbaseTest; import com.github.couchmove.pojo.ChangeLog; -import org.jetbrains.annotations.NotNull; +import com.github.couchmove.utils.TestUtils; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -28,7 +28,7 @@ public static void setUp() { @Test public void should_save_and_get_entity() { // Given a changeLog - ChangeLog changeLog = getRandomChangeLog(); + ChangeLog changeLog = TestUtils.getRandomChangeLog(); // When we insert it with an id String id = getRandomString(); @@ -44,7 +44,7 @@ public void should_save_and_get_entity() { @Test public void should_delete_entity() { // Given a changeLog saved on couchbase - ChangeLog changeLog = getRandomChangeLog(); + ChangeLog changeLog = TestUtils.getRandomChangeLog(); String id = getRandomString(); repository.save(id, changeLog); @@ -60,7 +60,7 @@ public void should_delete_entity() { @Test(expected = DocumentAlreadyExistsException.class) public void should_not_replace_entity_without_cas() { // Given a changeLog saved on couchbase - ChangeLog changeLog = getRandomChangeLog(); + ChangeLog changeLog = TestUtils.getRandomChangeLog(); String id = getRandomString(); repository.save(id, changeLog); @@ -74,7 +74,7 @@ public void should_not_replace_entity_without_cas() { @Test(expected = CASMismatchException.class) public void should_not_insert_entity_with_different_cas() { // Given a changeLog saved on couchbase - ChangeLog changeLog = getRandomChangeLog(); + ChangeLog changeLog = TestUtils.getRandomChangeLog(); String id = getRandomString(); repository.save(id, changeLog); @@ -89,16 +89,4 @@ public void should_not_insert_entity_with_different_cas() { repository.checkAndSave(id, savedChangeLog); } - // - @NotNull - private static ChangeLog getRandomChangeLog() { - return ChangeLog.builder() - .description(getRandomString()) - .duration(new Random().nextInt()) - .checksum(getRandomString()) - .success(true) - .build(); - } - // - } \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java index 2e726d5..a89109b 100644 --- a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -1,5 +1,6 @@ package com.github.couchmove.utils; +import com.github.couchmove.pojo.Type; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; @@ -10,8 +11,8 @@ import java.io.File; import java.nio.file.Path; -import static com.github.couchmove.service.ChangeLogFileService.JSON; -import static com.github.couchmove.service.ChangeLogFileService.N1QL; +import static com.github.couchmove.pojo.Type.DESIGN_DOC; +import static com.github.couchmove.pojo.Type.N1QL; /** * Created by tayebchlyah on 02/06/2017. @@ -51,6 +52,6 @@ public static Object[][] fileProvider() { @Test @UseDataProvider("fileProvider") public void should_calculate_checksum_of_file_or_folder(String path, String expectedChecksum) throws Exception { - Assert.assertEquals(path, expectedChecksum, FileUtils.calculateChecksum(FileUtils.getPathFromResource(path).toFile(), JSON, N1QL)); + Assert.assertEquals(path, expectedChecksum, FileUtils.calculateChecksum(FileUtils.getPathFromResource(path).toFile(), DESIGN_DOC.getExtension(), N1QL.getExtension())); } } \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java index 8abe570..9974190 100644 --- a/src/test/java/com/github/couchmove/utils/TestUtils.java +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -1,19 +1,37 @@ package com.github.couchmove.utils; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Type; import org.jetbrains.annotations.NotNull; +import java.util.Random; import java.util.UUID; -import static org.apache.commons.io.IOUtils.toByteArray; - /** * Created by tayebchlyah on 01/06/2017. */ public class TestUtils { + private static final Random RANDOM = new Random(); + @NotNull public static String getRandomString() { return UUID.randomUUID().toString(); } + @NotNull + public static ChangeLog getRandomChangeLog() { + Type type = Type.values()[Math.abs(RANDOM.nextInt(Type.values().length))]; + String version = getRandomString(); + String description = getRandomString().replace("-", "_"); + return ChangeLog.builder() + .version(version) + .description(description) + .type(type) + .script("V" + version + "__" + description + (!type.getExtension().isEmpty() ? "." + type.getExtension() : "")) + .duration(RANDOM.nextInt()) + .checksum(getRandomString()) + .success(true) + .build(); + } } From d10daefaba7d58c41d2e6e1f766042046ffe35bc Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 3 Jun 2017 18:58:05 +0200 Subject: [PATCH 07/42] Implement fetching changeLogs from database --- pom.xml | 12 +++ .../couchmove/service/ChangeLogDBService.java | 54 +++++++++++ .../service/ChangeLogDBServiceTest.java | 93 +++++++++++++++++++ 3 files changed, 159 insertions(+) create mode 100644 src/main/java/com/github/couchmove/service/ChangeLogDBService.java create mode 100644 src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java diff --git a/pom.xml b/pom.xml index 890438e..becda4f 100644 --- a/pom.xml +++ b/pom.xml @@ -88,6 +88,18 @@ 1.2.1 test + + org.mockito + mockito-all + 1.10.19 + test + + + org.hamcrest + hamcrest-junit + 2.0.0.0 + test + ch.qos.logback logback-classic diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java new file mode 100644 index 0000000..54689e0 --- /dev/null +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -0,0 +1,54 @@ +package com.github.couchmove.service; + +import com.couchbase.client.java.Bucket; +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.repository.CouchbaseRepository; +import com.github.couchmove.repository.CouchbaseRepositoryImpl; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * Created by tayebchlyah on 03/06/2017. + */ +public class ChangeLogDBService { + + private static final Logger logger = LoggerFactory.getLogger(ChangeLogDBService.class); + + public static final String PREFIX_ID = "changelog::"; + + private CouchbaseRepository repository; + + public ChangeLogDBService(Bucket bucket) { + this.repository = new CouchbaseRepositoryImpl<>(bucket, ChangeLog.class); + } + + public List fetchAndCompare(List changeLogs) { + logger.info("Fetching changeLogs from database"); + List result = new ArrayList<>(changeLogs.size()); + for (ChangeLog changeLog : changeLogs) { + String version = changeLog.getVersion(); + ChangeLog dbChangeLog = repository.findOne(PREFIX_ID + version); + if (dbChangeLog == null) { + logger.debug("ChangeLog version '{}' not found", version); + result.add(changeLog); + } else if (dbChangeLog.getChecksum() != null && !dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { + logger.error("ChangeLog version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); + throw new CouchMoveException("ChangeLog checksum doesn't match"); + } else { + logger.warn("ChangeLog version '{}' description updated"); + logger.debug("{} was {}", dbChangeLog, changeLog); + dbChangeLog.setDescription(changeLog.getDescription()); + dbChangeLog.setScript(changeLog.getScript()); + result.add(dbChangeLog); + } + } + return Collections.unmodifiableList(result); + } +} diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java new file mode 100644 index 0000000..0c60677 --- /dev/null +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -0,0 +1,93 @@ +package com.github.couchmove.service; + +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.repository.CouchbaseRepository; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.runners.MockitoJUnitRunner; +import org.testcontainers.shaded.com.google.common.collect.Lists; + +import java.util.ArrayList; +import java.util.List; + +import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; +import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; +import static com.github.couchmove.utils.TestUtils.getRandomString; +import static org.mockito.Mockito.when; + +/** + * Created by tayebchlyah on 03/06/2017. + */ +@RunWith(MockitoJUnitRunner.class) +public class ChangeLogDBServiceTest { + + @InjectMocks + private ChangeLogDBService service = new ChangeLogDBService(null); + + @Mock + private CouchbaseRepository repository; + + @Test + public void should_fetch_return_same_changeLogs_when_absent() { + // Given changeLogs stored in DB + when(repository.findOne(Mockito.anyString())).thenReturn(null); + + // When we call service with the later + List changeLogs = Lists.newArrayList(getRandomChangeLog(), getRandomChangeLog()); + List result = service.fetchAndCompare(changeLogs); + + // Then we should return the same + Assert.assertEquals(changeLogs, result); + } + + @Test(expected = CouchMoveException.class) + public void should_fetch_fail_when_checksum_does_not_match() { + // Given a changeLog stored on DB + ChangeLog dbChangeLog = getRandomChangeLog(); + when(repository.findOne(PREFIX_ID + dbChangeLog.getVersion())).thenReturn(dbChangeLog); + + // And a changeLog with same version but different checksum + ChangeLog changeLog = getRandomChangeLog(); + changeLog.setVersion(dbChangeLog.getVersion()); + changeLog.setChecksum(getRandomString()); + + // When we call service with the later changeLog + service.fetchAndCompare(Lists.newArrayList(changeLog)); + + // Then an exception should rise + } + + @Test + public void should_return_updated_changeLog_if_info_changed() { + // Given a changeLog stored on DB + ChangeLog dbChangeLog = getRandomChangeLog(); + when(repository.findOne(PREFIX_ID + dbChangeLog.getVersion())).thenReturn(dbChangeLog); + + // And a changeLog with different description + ChangeLog changeLog = ChangeLog.builder() + .version(dbChangeLog.getVersion()) + .type(dbChangeLog.getType()) + .checksum(dbChangeLog.getChecksum()) + .runner(dbChangeLog.getRunner()) + .duration(dbChangeLog.getDuration()) + .order(dbChangeLog.getOrder()) + .status(dbChangeLog.getStatus()) + .description(getRandomString()) + .script(getRandomString()) + .build(); + changeLog.setVersion(dbChangeLog.getVersion()); + changeLog.setChecksum(dbChangeLog.getChecksum()); + + // When we call service with the later + ArrayList changeLogs = Lists.newArrayList(changeLog); + List result = service.fetchAndCompare(changeLogs); + + // Then it should be same + Assert.assertEquals(changeLogs, result); + } +} \ No newline at end of file From 287cdd2998bb44f5d87c0f4f0f5ed1bf99b09205 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 3 Jun 2017 17:05:22 +0200 Subject: [PATCH 08/42] Update logs --- .../repository/CouchbaseRepository.java | 3 +++ .../repository/CouchbaseRepositoryImpl.java | 5 +++++ .../couchmove/service/ChangeLockService.java | 19 ++++++++++--------- .../couchmove/service/ChangeLogDBService.java | 9 ++++++--- .../service/ChangeLogFileService.java | 4 ++-- .../service/ChangeLogDBServiceTest.java | 10 ++++++++-- 6 files changed, 34 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java index 5b3583b..be7f263 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -1,5 +1,6 @@ package com.github.couchmove.repository; +import com.couchbase.client.java.Bucket; import com.github.couchmove.pojo.CouchbaseEntity; /** @@ -14,4 +15,6 @@ public interface CouchbaseRepository { void delete(String id); E findOne(String id); + + String getBucketName(); } diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 1b7c1f8..251f512 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -86,4 +86,9 @@ public E findOne(String id) { throw new CouchMoveException("Unable to read document with id " + id, e); } } + + @Override + public String getBucketName() { + return bucket.name(); + } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index e8977ce..530d603 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -23,18 +23,18 @@ public class ChangeLockService { private static final String LOCK_ID = "DATABASE_CHANGELOG_LOCK"; - private final CouchbaseRepository changeLockRepository; + private final CouchbaseRepository repository; private String uuid; public ChangeLockService(Bucket bucket) { - this.changeLockRepository = new CouchbaseRepositoryImpl<>(bucket, ChangeLock.class); + this.repository = new CouchbaseRepositoryImpl<>(bucket, ChangeLock.class); } public boolean acquireLock() { - logger.info("Trying to acquire database lock..."); + logger.info("Trying to acquire bucket '{}' lock...", repository.getBucketName()); // Verify if there is any lock on database - ChangeLock lock = changeLockRepository.findOne(LOCK_ID); + ChangeLock lock = repository.findOne(LOCK_ID); // If none, create one if (lock == null) { lock = new ChangeLock(); @@ -49,11 +49,11 @@ public boolean acquireLock() { lock.setUuid(uuid = UUID.randomUUID().toString()); // Tries to save it with Optimistic locking try { - changeLockRepository.checkAndSave(LOCK_ID, lock); + repository.checkAndSave(LOCK_ID, lock); } catch (CASMismatchException | DocumentAlreadyExistsException e) { // In case of exception, this means an other process got the lock, logging its information - lock = changeLockRepository.findOne(LOCK_ID); - logger.warn("The database is already locked by '{}'", lock.getRunner()); + lock = repository.findOne(LOCK_ID); + logger.warn("The bucket '{}' is already locked by '{}'", repository.getBucketName(), lock.getRunner()); return false; } logger.info("Lock acquired"); @@ -61,7 +61,7 @@ public boolean acquireLock() { } public boolean isLockAcquired() { - ChangeLock lock = changeLockRepository.findOne(LOCK_ID); + ChangeLock lock = repository.findOne(LOCK_ID); if (lock == null) { return false; } @@ -76,7 +76,8 @@ public boolean isLockAcquired() { } public void releaseLock() { - changeLockRepository.delete(LOCK_ID); + logger.info("Release lock"); + repository.delete(LOCK_ID); } // diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 54689e0..35b7031 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -5,8 +5,6 @@ import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; -import lombok.AccessLevel; -import lombok.NoArgsConstructor; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,7 +28,7 @@ public ChangeLogDBService(Bucket bucket) { } public List fetchAndCompare(List changeLogs) { - logger.info("Fetching changeLogs from database"); + logger.info("Fetching changeLogs from bucket '{}'", repository.getBucketName()); List result = new ArrayList<>(changeLogs.size()); for (ChangeLog changeLog : changeLogs) { String version = changeLog.getVersion(); @@ -49,6 +47,11 @@ public List fetchAndCompare(List changeLogs) { result.add(dbChangeLog); } } + logger.info("Fetched {} changeLogs from bucket", result.size()); return Collections.unmodifiableList(result); } + + public String getBucketName() { + return repository.getBucketName(); + } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index fe760f1..dd5dd9f 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -36,7 +36,7 @@ public ChangeLogFileService(String changePath) { } public List fetch() { - logger.info("Fetching changeLogs from '{}'", changeFolder.getPath()); + logger.info("Fetching changeLogs from migration folder '{}'", changeFolder.getPath()); SortedSet sortedChangeLogs = new TreeSet<>(); //noinspection ConstantConditions for (File file : changeFolder.listFiles()) { @@ -54,7 +54,7 @@ public List fetch() { sortedChangeLogs.add(changeLog); } } - logger.info("Fetched {} changeLogs", sortedChangeLogs.size()); + logger.info("Fetched {} changeLogs from migration folder", sortedChangeLogs.size()); return Collections.unmodifiableList(new ArrayList<>(sortedChangeLogs)); } diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index 0c60677..abc5223 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -4,6 +4,7 @@ import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import org.junit.Assert; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.InjectMocks; @@ -32,6 +33,11 @@ public class ChangeLogDBServiceTest { @Mock private CouchbaseRepository repository; + @Before + public void init() { + when(repository.getBucketName()).thenReturn("default"); + } + @Test public void should_fetch_return_same_changeLogs_when_absent() { // Given changeLogs stored in DB @@ -41,8 +47,8 @@ public void should_fetch_return_same_changeLogs_when_absent() { List changeLogs = Lists.newArrayList(getRandomChangeLog(), getRandomChangeLog()); List result = service.fetchAndCompare(changeLogs); - // Then we should return the same - Assert.assertEquals(changeLogs, result); + // Then we should return the same + Assert.assertEquals(changeLogs, result); } @Test(expected = CouchMoveException.class) From 59ee12dcf6e1fcfa2bb2275f3916ab9f766d4f05 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 3 Jun 2017 18:59:58 +0200 Subject: [PATCH 09/42] Update ChangeLog : replace success by status --- src/main/java/com/github/couchmove/pojo/ChangeLog.java | 8 ++++---- src/main/java/com/github/couchmove/pojo/Status.java | 10 ++++++++++ .../java/com/github/couchmove/utils/TestUtils.java | 7 ++++--- 3 files changed, 18 insertions(+), 7 deletions(-) create mode 100644 src/main/java/com/github/couchmove/pojo/Status.java diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java index 7e26500..efcd700 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLog.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -9,13 +9,13 @@ /** * Created by tayebchlyah on 27/05/2017. */ -@Builder +@Builder(toBuilder = true) @Data public class ChangeLog extends CouchbaseEntity implements Comparable { private String version; - private int order; + private Integer order; private String description; @@ -29,9 +29,9 @@ public class ChangeLog extends CouchbaseEntity implements Comparable private Date timestamp; - private long duration; + private Long duration; - private boolean success; + private Status status; @Override public int compareTo(@NotNull ChangeLog o) { diff --git a/src/main/java/com/github/couchmove/pojo/Status.java b/src/main/java/com/github/couchmove/pojo/Status.java new file mode 100644 index 0000000..be82d2d --- /dev/null +++ b/src/main/java/com/github/couchmove/pojo/Status.java @@ -0,0 +1,10 @@ +package com.github.couchmove.pojo; + +/** + * Created by tayebchlyah on 03/06/2017. + */ +public enum Status { + EXECUTED, + FAILED, + SKIPPED +} diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java index 9974190..06f8fb8 100644 --- a/src/test/java/com/github/couchmove/utils/TestUtils.java +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -1,6 +1,7 @@ package com.github.couchmove.utils; import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Status; import com.github.couchmove.pojo.Type; import org.jetbrains.annotations.NotNull; @@ -12,7 +13,7 @@ */ public class TestUtils { - private static final Random RANDOM = new Random(); + public static final Random RANDOM = new Random(); @NotNull public static String getRandomString() { @@ -29,9 +30,9 @@ public static ChangeLog getRandomChangeLog() { .description(description) .type(type) .script("V" + version + "__" + description + (!type.getExtension().isEmpty() ? "." + type.getExtension() : "")) - .duration(RANDOM.nextInt()) + .duration(RANDOM.nextLong()) .checksum(getRandomString()) - .success(true) + .status(Status.values()[Math.abs(RANDOM.nextInt(Status.values().length))]) .build(); } } From af3571a2b1b4a73f2e02248da2897782753e8fa4 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 4 Jun 2017 13:39:11 +0200 Subject: [PATCH 10/42] Update fetching changeLogs from database algorithm --- .../couchmove/service/ChangeLogDBService.java | 29 ++++++++- .../service/ChangeLogDBServiceTest.java | 62 +++++++++++++++---- 2 files changed, 75 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 35b7031..dd52f3a 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -27,6 +27,21 @@ public ChangeLogDBService(Bucket bucket) { this.repository = new CouchbaseRepositoryImpl<>(bucket, ChangeLog.class); } + /** + * Get corresponding ChangeLogs from Couchbase bucket + *
    + *
  • if a {@link ChangeLog} doesn't exist => return it as it its + *
  • else : + *
      + *
    • if checksum ({@link ChangeLog#checksum}) is reset (set to null), or description ({@link ChangeLog#description}) updated => reset {@link ChangeLog#cas} + *
    • return database version + *
    + *
+ * + * @param changeLogs to load from database + * @return database version of changeLogs + * @throws CouchMoveException if checksum doesn't match + */ public List fetchAndCompare(List changeLogs) { logger.info("Fetching changeLogs from bucket '{}'", repository.getBucketName()); List result = new ArrayList<>(changeLogs.size()); @@ -36,16 +51,24 @@ public List fetchAndCompare(List changeLogs) { if (dbChangeLog == null) { logger.debug("ChangeLog version '{}' not found", version); result.add(changeLog); - } else if (dbChangeLog.getChecksum() != null && !dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { + continue; + } + if (dbChangeLog.getChecksum() == null) { + logger.warn("ChangeLog version '{}' checksum reset"); + dbChangeLog.setChecksum(changeLog.getChecksum()); + dbChangeLog.setCas(null); + } else if (!dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { logger.error("ChangeLog version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); throw new CouchMoveException("ChangeLog checksum doesn't match"); - } else { + } + if (!dbChangeLog.getDescription().equals(changeLog.getDescription())) { logger.warn("ChangeLog version '{}' description updated"); logger.debug("{} was {}", dbChangeLog, changeLog); dbChangeLog.setDescription(changeLog.getDescription()); dbChangeLog.setScript(changeLog.getScript()); - result.add(dbChangeLog); + dbChangeLog.setCas(null); } + result.add(dbChangeLog); } logger.info("Fetched {} changeLogs from bucket", result.size()); return Collections.unmodifiableList(result); diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index abc5223..f4e7918 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -17,8 +17,7 @@ import java.util.List; import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; -import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; -import static com.github.couchmove.utils.TestUtils.getRandomString; +import static com.github.couchmove.utils.TestUtils.*; import static org.mockito.Mockito.when; /** @@ -51,6 +50,26 @@ public void should_fetch_return_same_changeLogs_when_absent() { Assert.assertEquals(changeLogs, result); } + @Test + public void should_fetch_return_unchanged_changeLogs() { + // Given changeLogs stored in DB + ChangeLog changeLog1 = getRandomChangeLog(); + changeLog1.setCas(RANDOM.nextLong()); + when(repository.findOne(PREFIX_ID + changeLog1.getVersion())).thenReturn(changeLog1); + ChangeLog changeLog2 = getRandomChangeLog(); + changeLog2.setCas(RANDOM.nextLong()); + when(repository.findOne(PREFIX_ID + changeLog2.getVersion())).thenReturn(changeLog2); + + // When we call service with the later + List changeLogs = Lists.newArrayList(changeLog1, changeLog2); + List result = service.fetchAndCompare(changeLogs); + + // Then nothing should be returned + Assert.assertEquals(changeLogs, result); + Assert.assertNotNull(changeLogs.get(0).getCas()); + Assert.assertNotNull(changeLogs.get(1).getCas()); + } + @Test(expected = CouchMoveException.class) public void should_fetch_fail_when_checksum_does_not_match() { // Given a changeLog stored on DB @@ -69,25 +88,39 @@ public void should_fetch_fail_when_checksum_does_not_match() { } @Test - public void should_return_updated_changeLog_if_info_changed() { + public void should_return_updated_changeLog_checksum_with_cas_reset_if_checksum_reset() { + // Given a changeLog stored on DB + ChangeLog dbChangeLog = getRandomChangeLog(); + dbChangeLog.setChecksum(null); + dbChangeLog.setCas(RANDOM.nextLong()); + when(repository.findOne(PREFIX_ID + dbChangeLog.getVersion())).thenReturn(dbChangeLog); + + // And a changeLog with different description + ChangeLog changeLog = dbChangeLog.toBuilder() + .checksum(getRandomString()) + .build(); + + // When we call service with the later + ArrayList changeLogs = Lists.newArrayList(changeLog); + List result = service.fetchAndCompare(changeLogs); + + // Then it should be same + Assert.assertEquals(changeLogs, result); + Assert.assertNull(changeLog.getCas()); + } + + @Test + public void should_return_updated_changeLog_with_cas_reset_if_description_changed() { // Given a changeLog stored on DB ChangeLog dbChangeLog = getRandomChangeLog(); + dbChangeLog.setCas(RANDOM.nextLong()); when(repository.findOne(PREFIX_ID + dbChangeLog.getVersion())).thenReturn(dbChangeLog); // And a changeLog with different description - ChangeLog changeLog = ChangeLog.builder() - .version(dbChangeLog.getVersion()) - .type(dbChangeLog.getType()) - .checksum(dbChangeLog.getChecksum()) - .runner(dbChangeLog.getRunner()) - .duration(dbChangeLog.getDuration()) - .order(dbChangeLog.getOrder()) - .status(dbChangeLog.getStatus()) + ChangeLog changeLog = dbChangeLog.toBuilder() .description(getRandomString()) .script(getRandomString()) .build(); - changeLog.setVersion(dbChangeLog.getVersion()); - changeLog.setChecksum(dbChangeLog.getChecksum()); // When we call service with the later ArrayList changeLogs = Lists.newArrayList(changeLog); @@ -95,5 +128,8 @@ public void should_return_updated_changeLog_if_info_changed() { // Then it should be same Assert.assertEquals(changeLogs, result); + Assert.assertNull(changeLog.getCas()); } + + } \ No newline at end of file From ce83360a9ca8896a1176755734c7a30bfcc0ff43 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 4 Jun 2017 14:10:59 +0200 Subject: [PATCH 11/42] CouchbaseRepository : Add cas to inserted entities --- .../couchmove/repository/CouchbaseRepositoryImpl.java | 9 ++++++--- .../couchmove/repository/CouchbaseRepositoryTest.java | 11 +++++++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 251f512..41b4271 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -34,7 +34,8 @@ public class CouchbaseRepositoryImpl implements Couch @Override public E save(String id, E entity) { try { - bucket.upsert(RawJsonDocument.create(id, getJsonMapper().writeValueAsString(entity))); + RawJsonDocument insertedDocument = bucket.upsert(RawJsonDocument.create(id, getJsonMapper().writeValueAsString(entity))); + entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { throw new CouchMoveException("Unable to save document with id " + id, e); @@ -56,11 +57,13 @@ public E save(String id, E entity) { public E checkAndSave(String id, E entity) { try { String content = getJsonMapper().writeValueAsString(entity); + RawJsonDocument insertedDocument; if (entity.getCas() != null) { - bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); + insertedDocument = bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); } else { - bucket.insert(RawJsonDocument.create(id, content)); + insertedDocument = bucket.insert(RawJsonDocument.create(id, content)); } + entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { throw new CouchMoveException("Unable to save document with id " + id, e); diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 3a6f3ea..9de8335 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -32,13 +32,20 @@ public void should_save_and_get_entity() { // When we insert it with an id String id = getRandomString(); - repository.save(id, changeLog); + ChangeLog savedChangeLog = repository.save(id, changeLog); + + // Then inserted one should have a cas + Assert.assertNotNull(savedChangeLog.getCas()); - // Then we should get it by this id + // And we should get it by this id ChangeLog result = repository.findOne(id); Assert.assertNotNull(result); Assert.assertEquals(changeLog, result); + + // And it should have the same cas + Assert.assertNotNull(result.getCas()); + Assert.assertEquals(savedChangeLog.getCas(), result.getCas()); } @Test From cdb90a1e20a87159696d48b3c26879cf82e85aca Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 4 Jun 2017 18:31:15 +0200 Subject: [PATCH 12/42] Fix Lombok warnings --- src/main/java/com/github/couchmove/pojo/ChangeLock.java | 2 ++ src/main/java/com/github/couchmove/pojo/ChangeLog.java | 2 ++ 2 files changed, 4 insertions(+) diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLock.java b/src/main/java/com/github/couchmove/pojo/ChangeLock.java index 6f42811..a32e287 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLock.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLock.java @@ -1,12 +1,14 @@ package com.github.couchmove.pojo; import lombok.Data; +import lombok.EqualsAndHashCode; import java.util.Date; /** * Created by tayebchlyah on 27/05/2017. */ +@EqualsAndHashCode(callSuper = false) @Data public class ChangeLock extends CouchbaseEntity { diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java index efcd700..8870b43 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLog.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -2,6 +2,7 @@ import lombok.Builder; import lombok.Data; +import lombok.EqualsAndHashCode; import org.jetbrains.annotations.NotNull; import java.util.Date; @@ -9,6 +10,7 @@ /** * Created by tayebchlyah on 27/05/2017. */ +@EqualsAndHashCode(callSuper = false) @Builder(toBuilder = true) @Data public class ChangeLog extends CouchbaseEntity implements Comparable { From e81dc54c53ac80be5b5acd2c04f90763235a3303 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 4 Jun 2017 18:32:28 +0200 Subject: [PATCH 13/42] Move getUsername to Utils class --- .../couchmove/service/ChangeLockService.java | 32 +------------ .../com/github/couchmove/utils/Utils.java | 46 +++++++++++++++++++ 2 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 src/main/java/com/github/couchmove/utils/Utils.java diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index 530d603..bcf54d2 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -6,11 +6,10 @@ import com.github.couchmove.pojo.ChangeLock; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; +import com.github.couchmove.utils.Utils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.util.Date; import java.util.UUID; @@ -45,7 +44,7 @@ public boolean acquireLock() { // Create Lock information lock.setLocked(true); lock.setTimestamp(new Date()); - lock.setRunner(getUsername()); + lock.setRunner(Utils.getUsername()); lock.setUuid(uuid = UUID.randomUUID().toString()); // Tries to save it with Optimistic locking try { @@ -80,32 +79,5 @@ public void releaseLock() { repository.delete(LOCK_ID); } - // - private static String getUsername() { - String osName = System.getProperty("os.name").toLowerCase(); - String className = null; - String methodName = "getUsername"; - - if (osName.contains("windows")) { - className = "com.sun.security.auth.module.NTSystem"; - methodName = "getName"; - } else if (osName.contains("linux") || osName.contains("mac")) { - className = "com.sun.security.auth.module.UnixSystem"; - } else if (osName.contains("solaris") || osName.contains("sunos")) { - className = "com.sun.security.auth.module.SolarisSystem"; - } - - if (className != null) { - try { - Class c = Class.forName(className); - Method method = c.getDeclaredMethod(methodName); - Object o = c.newInstance(); - return method.invoke(o).toString(); - } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { - logger.error("Unable to get actual user name", e); - } - } - return "unknown"; - } // } diff --git a/src/main/java/com/github/couchmove/utils/Utils.java b/src/main/java/com/github/couchmove/utils/Utils.java new file mode 100644 index 0000000..0ccf806 --- /dev/null +++ b/src/main/java/com/github/couchmove/utils/Utils.java @@ -0,0 +1,46 @@ +package com.github.couchmove.utils; + +import lombok.Getter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; + +/** + * Created by tayebchlyah on 04/06/2017. + */ +public class Utils { + + private static final Logger logger = LoggerFactory.getLogger(Utils.class); + + @Getter(lazy = true) + private static final String username = initializeUserName(); + + private static String initializeUserName() { + String osName = System.getProperty("os.name").toLowerCase(); + String className = null; + String methodName = "getUsername"; + + if (osName.contains("windows")) { + className = "com.sun.security.auth.module.NTSystem"; + methodName = "getName"; + } else if (osName.contains("linux") || osName.contains("mac")) { + className = "com.sun.security.auth.module.UnixSystem"; + } else if (osName.contains("solaris") || osName.contains("sunos")) { + className = "com.sun.security.auth.module.SolarisSystem"; + } + + if (className != null) { + try { + Class c = Class.forName(className); + Method method = c.getDeclaredMethod(methodName); + Object o = c.newInstance(); + return method.invoke(o).toString(); + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | InstantiationException | InvocationTargetException e) { + logger.error("Unable to get actual user name", e); + } + } + return "unknown"; + } +} From 8cc39ddb20e6ba1a5631dcc38625fab3f9829639 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 4 Jun 2017 18:43:12 +0200 Subject: [PATCH 14/42] Implement migration algorithm --- pom.xml | 5 + .../java/com/github/couchmove/CouchMove.java | 168 +++++++++++++++++ .../couchmove/service/ChangeLogDBService.java | 16 +- .../com/github/couchmove/CouchMoveTest.java | 173 ++++++++++++++++++ 4 files changed, 360 insertions(+), 2 deletions(-) create mode 100644 src/main/java/com/github/couchmove/CouchMove.java create mode 100644 src/test/java/com/github/couchmove/CouchMoveTest.java diff --git a/pom.xml b/pom.xml index becda4f..b6825d4 100644 --- a/pom.xml +++ b/pom.xml @@ -70,6 +70,11 @@ commons-io 2.5 + + com.google.guava + guava + 22.0 + junit junit diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java new file mode 100644 index 0000000..391c56f --- /dev/null +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -0,0 +1,168 @@ +package com.github.couchmove; + +import com.couchbase.client.java.Bucket; +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Type; +import com.github.couchmove.service.ChangeLockService; +import com.github.couchmove.service.ChangeLogDBService; +import com.github.couchmove.service.ChangeLogFileService; +import com.google.common.base.Stopwatch; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Comparator; +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +import static com.github.couchmove.pojo.Status.*; +import static com.github.couchmove.utils.Utils.getUsername; + +/** + * Created by tayebchlyah on 03/06/2017. + */ +@NoArgsConstructor(access = AccessLevel.PACKAGE) +public class CouchMove { + + public static final String DEFAULT_MIGRATION_PATH = "db/migration"; + + private static final Logger logger = LoggerFactory.getLogger(CouchMove.class); + + private String bucketName; + + private ChangeLockService lockService; + + private ChangeLogDBService dbService; + + private ChangeLogFileService fileService; + + public CouchMove(Bucket bucket) { + this(bucket, DEFAULT_MIGRATION_PATH); + } + + public CouchMove(Bucket bucket, String changePath) { + logger.info("Connected to bucket '{}'", bucketName = bucket.name()); + lockService = new ChangeLockService(bucket); + dbService = new ChangeLogDBService(bucket); + fileService = new ChangeLogFileService(changePath); + } + + public void migrate() { + logger.info("Begin bucket '{}' migration", bucketName); + try { + // Acquire bucket lock + if (!lockService.acquireLock()) { + logger.error("CouchMove did not acquire bucket '{}' lock. Exiting", bucketName); + throw new CouchMoveException("Unable to acquire lock"); + } + + // Fetching ChangeLogs from migration directory + List changeLogs = fileService.fetch(); + if (changeLogs.isEmpty()) { + logger.info("CouchMove did not find any migration scripts"); + return; + } + + // Fetching corresponding ChangeLogs from bucket + changeLogs = dbService.fetchAndCompare(changeLogs); + + // Executing migration + executeMigration(changeLogs); + } finally { + // Release lock + lockService.releaseLock(); + } + logger.info("CouchMove has finished his job"); + } + + void executeMigration(List changeLogs) { + logger.info("Executing migration scripts..."); + int migrationCount = 0; + // Get version and order of last executed changeLog + String lastVersion = ""; + int lastOrder = 0; + Optional lastExecutedChangeLog = changeLogs.stream() + .filter(c -> c.getStatus() == EXECUTED) + .max(Comparator.naturalOrder()); + if (lastExecutedChangeLog.isPresent()) { + lastVersion = lastExecutedChangeLog.get().getVersion(); + lastOrder = lastExecutedChangeLog.get().getOrder(); + } + + for (ChangeLog changeLog : changeLogs) { + if (changeLog.getStatus() == EXECUTED) { + lastVersion = changeLog.getVersion(); + lastOrder = changeLog.getOrder(); + } + } + + for (ChangeLog changeLog : changeLogs) { + if (changeLog.getStatus() == EXECUTED) { + if (changeLog.getCas() == null) { + logger.info("Updating changeLog '{}'", changeLog.getVersion()); + dbService.save(changeLog); + } + continue; + } + + if (changeLog.getStatus() == SKIPPED) { + continue; + } + + if (lastVersion.compareTo(changeLog.getVersion()) >= 0) { + logger.warn("ChangeLog '{}' version is lower than last executed one '{}'. Skipping", changeLog.getVersion(), lastVersion); + changeLog.setStatus(SKIPPED); + dbService.save(changeLog); + continue; + } + + if (executeMigration(changeLog)) { + changeLog.setOrder(++lastOrder); + lastVersion = changeLog.getVersion(); + migrationCount++; + } else { + throw new CouchMoveException("Migration failed"); + } + } + if (migrationCount == 0) { + logger.info("No new migration scripts found"); + } else { + logger.info("Executed {} migration scripts", migrationCount); + } + } + + boolean executeMigration(ChangeLog changeLog) { + logger.info("Executing ChangeLog '{}'", changeLog.getVersion()); + Stopwatch sw = Stopwatch.createStarted(); + changeLog.setTimestamp(new Date()); + changeLog.setRunner(getUsername()); + if (executeMigration(changeLog.getType(), changeLog.getScript())) { + logger.info("ChangeLog '{}' successfully executed", changeLog.getVersion()); + changeLog.setStatus(EXECUTED); + } else { + logger.error("Unable to execute ChangeLog '{}'", changeLog.getVersion()); + changeLog.setStatus(FAILED); + } + changeLog.setDuration(sw.elapsed(TimeUnit.MILLISECONDS)); + dbService.save(changeLog); + return changeLog.getStatus() == EXECUTED; + } + + private boolean executeMigration(Type type, String script) { + switch (type) { + case DOCUMENTS: + return dbService.importDocuments(script); + case N1QL: + return dbService.executeN1ql(script); + case DESIGN_DOC: + return dbService.importDesignDoc(script); + default: + throw new IllegalArgumentException("Unknown ChangeLog Type '" + type + "'"); + } + } +} + diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index dd52f3a..d8f5df7 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -74,7 +74,19 @@ public List fetchAndCompare(List changeLogs) { return Collections.unmodifiableList(result); } - public String getBucketName() { - return repository.getBucketName(); + public ChangeLog save(ChangeLog changeLog) { + return repository.save(PREFIX_ID + changeLog.getVersion(), changeLog); + } + + public boolean importDesignDoc(String script) { + return true; + } + + public boolean executeN1ql(String script) { + return true; + } + + public boolean importDocuments(String script) { + return true; } } diff --git a/src/test/java/com/github/couchmove/CouchMoveTest.java b/src/test/java/com/github/couchmove/CouchMoveTest.java new file mode 100644 index 0000000..7f55ba0 --- /dev/null +++ b/src/test/java/com/github/couchmove/CouchMoveTest.java @@ -0,0 +1,173 @@ +package com.github.couchmove; + +import com.couchbase.client.java.Bucket; +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Status; +import com.github.couchmove.pojo.Type; +import com.github.couchmove.service.ChangeLockService; +import com.github.couchmove.service.ChangeLogDBService; +import com.github.couchmove.service.ChangeLogFileService; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.runners.MockitoJUnitRunner; + +import static com.github.couchmove.pojo.Status.EXECUTED; +import static com.github.couchmove.pojo.Status.SKIPPED; +import static com.github.couchmove.utils.TestUtils.RANDOM; +import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; +import static com.google.common.collect.Lists.newArrayList; +import static java.util.Collections.emptyList; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; + +/** + * Created by tayebchlyah on 04/06/2017. + */ +@RunWith(MockitoJUnitRunner.class) +public class CouchMoveTest { + + @InjectMocks + private CouchMove couchMove = new CouchMove(mockBucket()); + + @Mock + private ChangeLockService lockServiceMock; + + @Mock + private ChangeLogDBService dbServiceMock; + + @Mock + private ChangeLogFileService fileServiceMock; + + @Test(expected = CouchMoveException.class) + public void should_migration_fail_if_lock_not_acquired() { + when(lockServiceMock.acquireLock()).thenReturn(false); + couchMove.migrate(); + } + + @Test + public void should_release_lock_after_migration() { + when(lockServiceMock.acquireLock()).thenReturn(true); + when(fileServiceMock.fetch()).thenReturn(newArrayList(getRandomChangeLog())); + when(dbServiceMock.fetchAndCompare(any())).thenReturn(emptyList()); + couchMove.migrate(); + verify(lockServiceMock).releaseLock(); + } + + @Test + public void should_migration_save_updated_changeLog() { + ChangeLog changeLog = ChangeLog.builder() + .version("1") + .status(EXECUTED) + .order(1) + .build(); + couchMove.executeMigration(newArrayList(changeLog)); + verify(dbServiceMock).save(changeLog); + } + + @Test + public void should_migration_skip_unmodified_executed_changeLog() { + ChangeLog skippedChangeLog = ChangeLog.builder() + .version("1") + .status(EXECUTED) + .order(1) + .build(); + skippedChangeLog.setCas(RANDOM.nextLong()); + couchMove.executeMigration(newArrayList(skippedChangeLog)); + verify(dbServiceMock, never()).save(any()); + } + + @Test + public void should_migration_skip_skipped_changeLogs() { + ChangeLog skippedChangeLog = ChangeLog.builder() + .version("1") + .status(SKIPPED) + .build(); + couchMove.executeMigration(newArrayList(skippedChangeLog)); + verify(dbServiceMock, never()).save(any()); + } + + @Test + public void should_migration_skip_changeLog_with_old_version() { + ChangeLog changeLogToSkip = ChangeLog.builder() + .version("1") + .build(); + ChangeLog executedChangeLog = ChangeLog.builder() + .version("2") + .order(2) + .status(EXECUTED) + .build(); + couchMove.executeMigration(newArrayList(changeLogToSkip, executedChangeLog)); + verify(dbServiceMock).save(changeLogToSkip); + Assert.assertEquals(SKIPPED, changeLogToSkip.getStatus()); + } + + @Test + public void should_execute_migrations() { + CouchMove couchMove = spy(CouchMove.class); + ChangeLog executedChangeLog = ChangeLog.builder() + .version("1") + .order(1) + .status(EXECUTED) + .build(); + executedChangeLog.setCas(RANDOM.nextLong()); + ChangeLog changeLog = ChangeLog.builder() + .version("2") + .type(Type.DOCUMENTS) + .build(); + doReturn(true).when(couchMove).executeMigration(changeLog); + couchMove.executeMigration(newArrayList(newArrayList(executedChangeLog, changeLog))); + Assert.assertEquals((Integer) 2, changeLog.getOrder()); + } + + @Test(expected = CouchMoveException.class) + public void should_throw_exception_if_migration_failed() { + CouchMove couchMove = spy(CouchMove.class); + ChangeLog changeLog = ChangeLog.builder() + .version("1") + .type(Type.N1QL) + .build(); + doReturn(false).when(couchMove).executeMigration(changeLog); + couchMove.executeMigration(newArrayList(changeLog)); + } + + @Test + public void should_update_changeLog_on_migration_success() { + ChangeLog changeLog = ChangeLog.builder() + .version("1") + .type(Type.DESIGN_DOC) + .build(); + when(dbServiceMock.importDesignDoc(any())).thenReturn(true); + couchMove.executeMigration(changeLog); + verify(dbServiceMock).save(changeLog); + Assert.assertNotNull(changeLog.getTimestamp()); + Assert.assertNotNull(changeLog.getDuration()); + Assert.assertNotNull(changeLog.getRunner()); + Assert.assertEquals(changeLog.getStatus(), Status.EXECUTED); + } + + @Test + public void should_update_changeLog_on_migration_failure() { + ChangeLog changeLog = ChangeLog.builder() + .version("1") + .type(Type.DOCUMENTS) + .build(); + when(dbServiceMock.importDocuments(any())).thenReturn(false); + couchMove.executeMigration(changeLog); + verify(dbServiceMock).save(changeLog); + Assert.assertNotNull(changeLog.getTimestamp()); + Assert.assertNotNull(changeLog.getDuration()); + Assert.assertNotNull(changeLog.getRunner()); + Assert.assertEquals(changeLog.getStatus(), Status.FAILED); + } + + private static Bucket mockBucket() { + Bucket mockedBucket = mock(Bucket.class); + when(mockedBucket.name()).thenReturn("default"); + return mockedBucket; + } + +} \ No newline at end of file From 4fed0d65d4baf992829760851f9cfbd2e35cae61 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Mon, 5 Jun 2017 18:29:41 +0200 Subject: [PATCH 15/42] Implement migration execution : execute n1ql queries, import json and design documents --- .../java/com/github/couchmove/CouchMove.java | 32 +++++---- .../java/com/github/couchmove/pojo/Type.java | 9 ++- .../repository/CouchbaseRepository.java | 7 +- .../repository/CouchbaseRepositoryImpl.java | 41 ++++++++++++ .../couchmove/service/ChangeLogDBService.java | 31 +++++++-- .../service/ChangeLogFileService.java | 19 +++++- .../com/github/couchmove/utils/FileUtils.java | 32 +++++++++ .../github/couchmove/utils/FunctionUtils.java | 12 ++++ .../com/github/couchmove/CouchMoveTest.java | 22 +++---- .../container/AbstractCouchbaseTest.java | 8 ++- .../repository/CouchbaseRepositoryTest.java | 66 ++++++++++++++++++- .../service/ChangeLogDBServiceTest.java | 22 ++++++- .../github/couchmove/utils/FileUtilsTest.java | 42 +++++++++++- 13 files changed, 302 insertions(+), 41 deletions(-) create mode 100644 src/main/java/com/github/couchmove/utils/FunctionUtils.java diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java index 391c56f..1aa62c1 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -3,7 +3,6 @@ import com.couchbase.client.java.Bucket; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLog; -import com.github.couchmove.pojo.Type; import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; @@ -140,7 +139,7 @@ boolean executeMigration(ChangeLog changeLog) { Stopwatch sw = Stopwatch.createStarted(); changeLog.setTimestamp(new Date()); changeLog.setRunner(getUsername()); - if (executeMigration(changeLog.getType(), changeLog.getScript())) { + if (doExecute(changeLog)) { logger.info("ChangeLog '{}' successfully executed", changeLog.getVersion()); changeLog.setStatus(EXECUTED); } else { @@ -152,16 +151,25 @@ boolean executeMigration(ChangeLog changeLog) { return changeLog.getStatus() == EXECUTED; } - private boolean executeMigration(Type type, String script) { - switch (type) { - case DOCUMENTS: - return dbService.importDocuments(script); - case N1QL: - return dbService.executeN1ql(script); - case DESIGN_DOC: - return dbService.importDesignDoc(script); - default: - throw new IllegalArgumentException("Unknown ChangeLog Type '" + type + "'"); + private boolean doExecute(ChangeLog changeLog) { + try { + switch (changeLog.getType()) { + case DOCUMENTS: + dbService.importDocuments(fileService.readDocuments(changeLog.getScript())); + break; + case N1QL: + dbService.executeN1ql(fileService.readLines(changeLog.getScript())); + break; + case DESIGN_DOC: + dbService.importDesignDoc(changeLog.getDescription().replace(" ", "_"), fileService.readFile(changeLog.getScript())); + break; + default: + throw new IllegalArgumentException("Unknown ChangeLog Type '" + changeLog.getType() + "'"); + } + return true; + } catch (Exception e) { + logger.error("Unable to import " + changeLog.getType().name().toLowerCase().replace("_", " ") + " : '" + changeLog.getScript() + "'", e); + return false; } } } diff --git a/src/main/java/com/github/couchmove/pojo/Type.java b/src/main/java/com/github/couchmove/pojo/Type.java index 6b6e439..0b45d60 100644 --- a/src/main/java/com/github/couchmove/pojo/Type.java +++ b/src/main/java/com/github/couchmove/pojo/Type.java @@ -7,8 +7,8 @@ */ public enum Type { DOCUMENTS(""), - DESIGN_DOC("json"), - N1QL("n1ql"); + DESIGN_DOC(Constants.JSON), + N1QL(Constants.N1QL); @Getter private final String extension; @@ -16,4 +16,9 @@ public enum Type { Type(String extension) { this.extension = extension; } + + public static class Constants { + public static final String JSON = "json"; + public static final String N1QL = "n1ql"; + } } diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java index be7f263..f16189e 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -1,6 +1,5 @@ package com.github.couchmove.repository; -import com.couchbase.client.java.Bucket; import com.github.couchmove.pojo.CouchbaseEntity; /** @@ -16,5 +15,11 @@ public interface CouchbaseRepository { E findOne(String id); + void save(String id, String jsonContent); + + void importDesignDoc(String name, String jsonContent); + + void query(String request); + String getBucketName(); } diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 41b4271..3ac41d3 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -4,19 +4,32 @@ import com.couchbase.client.deps.com.fasterxml.jackson.databind.ObjectMapper; import com.couchbase.client.java.Bucket; import com.couchbase.client.java.document.RawJsonDocument; +import com.couchbase.client.java.document.json.JsonObject; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.query.N1qlQueryResult; +import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.CouchbaseEntity; +import lombok.AccessLevel; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; +import org.slf4j.Logger; import java.io.IOException; +import static org.slf4j.LoggerFactory.getLogger; + /** * Created by tayebchlyah on 27/05/2017. */ +// For tests +@NoArgsConstructor(access = AccessLevel.PACKAGE,force = true) @RequiredArgsConstructor public class CouchbaseRepositoryImpl implements CouchbaseRepository { + private static final Logger logger = getLogger(CouchbaseRepositoryImpl.class); + @Getter(lazy = true) private static final ObjectMapper jsonMapper = new ObjectMapper(); @@ -90,6 +103,34 @@ public E findOne(String id) { } } + @Override + public void save(String id, String jsonContent) { + bucket.upsert(RawJsonDocument.create(id, jsonContent)); + } + + @Override + public void importDesignDoc(String name, String jsonContent) { + bucket.bucketManager().upsertDesignDocument(DesignDocument.from(name, JsonObject.fromJson(jsonContent))); + } + + @Override + public void query(String n1qlStatement) { + logger.debug("Executing n1ql request : {}", n1qlStatement); + try { + N1qlQueryResult result = bucket.query(N1qlQuery.simple(n1qlStatement)); + if (!result.parseSuccess()) { + logger.info("Invalid N1QL request '{}'", n1qlStatement); + throw new CouchMoveException("Invalid n1ql request"); + } + if (!result.finalSuccess()) { + logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : ", n1qlStatement, result.status(), result.errors()); + throw new CouchMoveException("Unable to execute n1ql request"); + } + } catch (Exception e) { + throw new CouchMoveException("Unable to execute n1ql request", e); + } + } + @Override public String getBucketName() { return bucket.name(); diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index d8f5df7..ea1e148 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -5,12 +5,18 @@ import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; +import com.github.couchmove.utils.FileUtils; +import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import static com.github.couchmove.utils.FunctionUtils.not; /** * Created by tayebchlyah on 03/06/2017. @@ -78,15 +84,28 @@ public ChangeLog save(ChangeLog changeLog) { return repository.save(PREFIX_ID + changeLog.getVersion(), changeLog); } - public boolean importDesignDoc(String script) { - return true; + public void importDesignDoc(String name, String content) { + logger.info("Inserting Design Document '{}'...", name); + repository.importDesignDoc(name, content); + } + + public void executeN1ql(List lines) { + List requests = filterRequests(lines); + logger.info("Executing {} n1ql requests", requests.size()); + requests.forEach(repository::query); } - public boolean executeN1ql(String script) { - return true; + public void importDocuments(Map documents) { + logger.info("Importing {} documents", documents.size()); + documents.forEach((fileName, content) -> + repository.save(FilenameUtils.getBaseName(fileName), content)); } - public boolean importDocuments(String script) { - return true; + static List filterRequests(List lines) { + return lines.stream() + .map(String::trim) + .filter(not(String::isEmpty)) + .filter(s -> !s.startsWith("--")) + .collect(Collectors.toList()); } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index dd5dd9f..04f499b 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -12,11 +12,13 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; +import static com.github.couchmove.pojo.Type.Constants.JSON; import static com.github.couchmove.pojo.Type.DESIGN_DOC; import static com.github.couchmove.pojo.Type.N1QL; @@ -58,6 +60,18 @@ public List fetch() { return Collections.unmodifiableList(new ArrayList<>(sortedChangeLogs)); } + public String readFile(String path) throws IOException { + return new String(Files.readAllBytes(resolve(path))); + } + + public Map readDocuments(String path) throws IOException { + return FileUtils.readFilesInDirectory(resolve(path).toFile(), JSON); + } + + public List readLines(String path) throws IOException { + return Files.readAllLines(resolve(path)); + } + // static File initializeFolder(String changePath) { Path path; @@ -75,6 +89,10 @@ static File initializeFolder(String changePath) { return file; } + private Path resolve(String path) { + return changeFolder.toPath().resolve(path); + } + @NotNull static Type getChangeLogType(File file) { if (file.isDirectory()) { @@ -90,7 +108,6 @@ static Type getChangeLogType(File file) { } throw new CouchMoveException("Unknown ChangeLog type : " + file.getName()); } - // } diff --git a/src/main/java/com/github/couchmove/utils/FileUtils.java b/src/main/java/com/github/couchmove/utils/FileUtils.java index 9d61a74..b210b17 100644 --- a/src/main/java/com/github/couchmove/utils/FileUtils.java +++ b/src/main/java/com/github/couchmove/utils/FileUtils.java @@ -15,6 +15,8 @@ import java.util.Arrays; import java.util.Collections; import java.util.Comparator; +import java.util.Map; +import java.util.stream.Collectors; import static org.apache.commons.io.IOUtils.toByteArray; @@ -96,4 +98,34 @@ public static String calculateChecksum(@NotNull File file, String... extensions) throw new CouchMoveException("Unable to calculate file checksum '" + file.getName() + "'"); } } + + public static Map readFilesInDirectory(@NotNull File file, String... extensions) throws IOException { + if (file == null || !file.exists()) { + throw new IllegalArgumentException("File is null or doesn't exists"); + } + if (!file.isDirectory()) { + throw new IllegalArgumentException("'" + file.getPath() + "' is not a directory"); + } + try { + //noinspection ConstantConditions + return Arrays.stream(file.listFiles()) + .filter(File::isFile) + .filter(f -> Arrays.stream(extensions) + .anyMatch(extension -> FilenameUtils + .getExtension(f.getName()).toLowerCase() + .equals(extension.toLowerCase()))) + .collect(Collectors.toMap(File::getName, f -> { + try { + return new String(Files.readAllBytes(f.toPath())); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + } catch (RuntimeException e) { + if (e.getCause() instanceof IOException) { + throw (IOException) e.getCause(); + } + throw e; + } + } } diff --git a/src/main/java/com/github/couchmove/utils/FunctionUtils.java b/src/main/java/com/github/couchmove/utils/FunctionUtils.java new file mode 100644 index 0000000..e42b502 --- /dev/null +++ b/src/main/java/com/github/couchmove/utils/FunctionUtils.java @@ -0,0 +1,12 @@ +package com.github.couchmove.utils; + +import java.util.function.Predicate; + +/** + * Created by tayebchlyah on 05/06/2017. + */ +public class FunctionUtils { + public static Predicate not(Predicate predicate) { + return predicate.negate(); + } +} diff --git a/src/test/java/com/github/couchmove/CouchMoveTest.java b/src/test/java/com/github/couchmove/CouchMoveTest.java index 7f55ba0..464f6ed 100644 --- a/src/test/java/com/github/couchmove/CouchMoveTest.java +++ b/src/test/java/com/github/couchmove/CouchMoveTest.java @@ -3,8 +3,6 @@ import com.couchbase.client.java.Bucket; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLog; -import com.github.couchmove.pojo.Status; -import com.github.couchmove.pojo.Type; import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; @@ -15,8 +13,8 @@ import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; -import static com.github.couchmove.pojo.Status.EXECUTED; -import static com.github.couchmove.pojo.Status.SKIPPED; +import static com.github.couchmove.pojo.Status.*; +import static com.github.couchmove.pojo.Type.*; import static com.github.couchmove.utils.TestUtils.RANDOM; import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; import static com.google.common.collect.Lists.newArrayList; @@ -116,7 +114,7 @@ public void should_execute_migrations() { executedChangeLog.setCas(RANDOM.nextLong()); ChangeLog changeLog = ChangeLog.builder() .version("2") - .type(Type.DOCUMENTS) + .type(DOCUMENTS) .build(); doReturn(true).when(couchMove).executeMigration(changeLog); couchMove.executeMigration(newArrayList(newArrayList(executedChangeLog, changeLog))); @@ -128,7 +126,7 @@ public void should_throw_exception_if_migration_failed() { CouchMove couchMove = spy(CouchMove.class); ChangeLog changeLog = ChangeLog.builder() .version("1") - .type(Type.N1QL) + .type(N1QL) .build(); doReturn(false).when(couchMove).executeMigration(changeLog); couchMove.executeMigration(newArrayList(changeLog)); @@ -138,30 +136,30 @@ public void should_throw_exception_if_migration_failed() { public void should_update_changeLog_on_migration_success() { ChangeLog changeLog = ChangeLog.builder() .version("1") - .type(Type.DESIGN_DOC) + .description("description") + .type(DESIGN_DOC) .build(); - when(dbServiceMock.importDesignDoc(any())).thenReturn(true); couchMove.executeMigration(changeLog); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); Assert.assertNotNull(changeLog.getRunner()); - Assert.assertEquals(changeLog.getStatus(), Status.EXECUTED); + Assert.assertEquals(EXECUTED, changeLog.getStatus()); } @Test public void should_update_changeLog_on_migration_failure() { ChangeLog changeLog = ChangeLog.builder() .version("1") - .type(Type.DOCUMENTS) + .type(DOCUMENTS) .build(); - when(dbServiceMock.importDocuments(any())).thenReturn(false); + doThrow(CouchMoveException.class).when(dbServiceMock).importDocuments(any()); couchMove.executeMigration(changeLog); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); Assert.assertNotNull(changeLog.getRunner()); - Assert.assertEquals(changeLog.getStatus(), Status.FAILED); + Assert.assertEquals(FAILED, changeLog.getStatus()); } private static Bucket mockBucket() { diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index 137e620..56f191f 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -36,8 +36,8 @@ public void clear() { private static CouchbaseContainer initCouchbaseContainer() { CouchbaseContainer couchbaseContainer = new CouchbaseContainer() .withFTS(false) - .withIndex(false) - .withQuery(false) + .withIndex(true) + .withQuery(true) .withClusterUsername(CLUSTER_USER) .withClusterPassword(CLUSTER_PASSWORD) .withNewBucket(DefaultBucketSettings.builder() @@ -53,6 +53,8 @@ private static CouchbaseContainer initCouchbaseContainer() { private static Bucket openBucket(String bucketName) { CouchbaseCluster cluster = getCouchbaseContainer().getCouchbaseCluster(); - return cluster.openBucket(bucketName); + Bucket bucket = cluster.openBucket(bucketName); + Runtime.getRuntime().addShutdownHook(new Thread(bucket::close)); + return bucket; } } diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 9de8335..da2805d 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -2,15 +2,21 @@ import com.couchbase.client.java.error.CASMismatchException; import com.couchbase.client.java.error.DocumentAlreadyExistsException; +import com.couchbase.client.java.query.util.IndexInfo; +import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.container.AbstractCouchbaseTest; import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Type; import com.github.couchmove.utils.TestUtils; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import java.util.List; import java.util.Random; +import java.util.stream.Collectors; +import static com.github.couchmove.utils.FunctionUtils.not; import static com.github.couchmove.utils.TestUtils.getRandomString; /** @@ -22,7 +28,7 @@ public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { @BeforeClass public static void setUp() { - repository = new CouchbaseRepositoryImpl<>(AbstractCouchbaseTest.getBucket(), ChangeLog.class); + repository = new CouchbaseRepositoryImpl<>(getBucket(), ChangeLog.class); } @Test @@ -96,4 +102,62 @@ public void should_not_insert_entity_with_different_cas() { repository.checkAndSave(id, savedChangeLog); } + @Test + public void should_import_design_doc() { + // Given a Design Doc + String name = "user"; + String design_doc = "{\n" + + " \"views\": {\n" + + " \"findUser\": {\n" + + " \"map\": \"function (doc, meta) {\\n if (doc.type == \\\"user\\\") {\\n emit(doc.username, null);\\n } \\n}\"\n" + + " }\n" + + " }\n" + + "}"; + + // When we import it + repository.importDesignDoc(name, design_doc); + + // Then it should be saved + DesignDocument designDocument = AbstractCouchbaseTest.getBucket().bucketManager().getDesignDocument(name); + Assert.assertNotNull(designDocument); + } + + @Test + public void should_execute_n1ql() { + // Given a primary index request + String request = String.format("CREATE INDEX `%s` ON `%s`(`%s`)", "name", getBucket().name(), "name"); + + // When we execute the query + repository.query(request); + + // Then the index should be created + List indexInfos = getBucket().bucketManager().listN1qlIndexes().stream() + .filter(not(IndexInfo::isPrimary)) + .collect(Collectors.toList()); + Assert.assertEquals(1, indexInfos.size()); + IndexInfo indexInfo = indexInfos.get(0); + Assert.assertEquals("name", indexInfo.name()); + Assert.assertEquals("name", indexInfo.indexKey().get(0)); + } + + @Test + public void should_save_json_document() { + // Given a json document + String json = "{\n" + + " \"version\": \"1\",\n" + + " \"description\": \"insert users\",\n" + + " \"type\": \"N1QL\"\n" + + "}"; + + // When we save the document + repository.save("change::1", json); + + // Then we should be bale to get it + ChangeLog changeLog = repository.findOne("change::1"); + Assert.assertNotNull(changeLog); + Assert.assertEquals("1", changeLog.getVersion()); + Assert.assertEquals("insert users", changeLog.getDescription()); + Assert.assertEquals(Type.N1QL, changeLog.getType()); + } + } \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index f4e7918..7eb6369 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -15,10 +15,15 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; +import static com.github.couchmove.service.ChangeLogDBService.filterRequests; import static com.github.couchmove.utils.TestUtils.*; -import static org.mockito.Mockito.when; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.*; +import static org.mockito.Mockito.verify; /** * Created by tayebchlyah on 03/06/2017. @@ -131,5 +136,18 @@ public void should_return_updated_changeLog_with_cas_reset_if_description_change Assert.assertNull(changeLog.getCas()); } - + @Test + public void should_skip_n1ql_blank_and_comment_lines() { + String request = "CREATE INDEX PRIMARY INDEX 'primary' ON 'default'"; + List lines = Stream.of( + request, + " ", + " -- toto", + "-- hello" + ).collect(Collectors.toList()); + + List result = filterRequests(lines); + Assert.assertEquals(1, result.size()); + Assert.assertEquals(request, result.get(0)); + } } \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java index a89109b..f29a35f 100644 --- a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -1,6 +1,6 @@ package com.github.couchmove.utils; -import com.github.couchmove.pojo.Type; +import com.google.common.io.Files; import com.tngtech.java.junit.dataprovider.DataProvider; import com.tngtech.java.junit.dataprovider.DataProviderRunner; import com.tngtech.java.junit.dataprovider.UseDataProvider; @@ -9,10 +9,13 @@ import org.junit.runner.RunWith; import java.io.File; +import java.io.IOException; import java.nio.file.Path; +import java.util.Map; import static com.github.couchmove.pojo.Type.DESIGN_DOC; import static com.github.couchmove.pojo.Type.N1QL; +import static com.github.couchmove.utils.TestUtils.getRandomString; /** * Created by tayebchlyah on 02/06/2017. @@ -54,4 +57,41 @@ public static Object[][] fileProvider() { public void should_calculate_checksum_of_file_or_folder(String path, String expectedChecksum) throws Exception { Assert.assertEquals(path, expectedChecksum, FileUtils.calculateChecksum(FileUtils.getPathFromResource(path).toFile(), DESIGN_DOC.getExtension(), N1QL.getExtension())); } + + @Test(expected = IllegalArgumentException.class) + public void should_read_files_failed_if_not_exists() throws Exception { + FileUtils.readFilesInDirectory(new File("")); + } + + @Test(expected = IllegalArgumentException.class) + public void should_read_files_failed_if_not_directory() throws Exception { + File temp = File.createTempFile(getRandomString(), ""); + temp.deleteOnExit(); + FileUtils.readFilesInDirectory(temp); + } + + @Test + public void should_read_files_in_directory() throws IOException { + // Given a temp directory that contains + File tempDir = Files.createTempDir(); + tempDir.deleteOnExit(); + // json file + File file1 = File.createTempFile("file1", ".json", tempDir); + String content1 = "content1"; + Files.write(content1.getBytes(), file1); + // n1ql file + File file2 = File.createTempFile("file2", ".N1QL", tempDir); + String content2 = "content2"; + Files.write(content2.getBytes(), file2); + // txt file + Files.write(getRandomString().getBytes(), File.createTempFile(getRandomString(), ".txt", tempDir)); + + // When we read files in this directory with extension filter + Map result = FileUtils.readFilesInDirectory(tempDir, "json", "n1ql"); + + // Then we should have file content matching this extension + Assert.assertEquals(2, result.size()); + Assert.assertEquals(content1, result.get(file1.getName())); + Assert.assertEquals(content2, result.get(file2.getName())); + } } \ No newline at end of file From 00862eb58f540afe60dddf0b5fe60eab066ddf25 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Wed, 7 Jun 2017 00:08:33 +0200 Subject: [PATCH 16/42] [Test] Fix Couchbase Test Container query service not ready --- .../container/AbstractCouchbaseTest.java | 1 + .../container/CouchbaseContainer.java | 26 ++++++---- .../CouchbaseQueryServiceWaitStrategy.java | 52 +++++++++++++++++++ 3 files changed, 69 insertions(+), 10 deletions(-) create mode 100644 src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index 56f191f..c2f06b2 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -38,6 +38,7 @@ private static CouchbaseContainer initCouchbaseContainer() { .withFTS(false) .withIndex(true) .withQuery(true) + .withPrimaryIndex(false) .withClusterUsername(CLUSTER_USER) .withClusterPassword(CLUSTER_PASSWORD) .withNewBucket(DefaultBucketSettings.builder() diff --git a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java index 15db0b8..4ed9150 100644 --- a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java +++ b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java @@ -16,6 +16,7 @@ package com.github.couchmove.container; import com.couchbase.client.core.utils.Base64; +import com.couchbase.client.java.Bucket; import com.couchbase.client.java.CouchbaseCluster; import com.couchbase.client.java.cluster.BucketSettings; import com.couchbase.client.java.env.CouchbaseEnvironment; @@ -51,6 +52,8 @@ public class CouchbaseContainer> extends G private Boolean index = true; + private Boolean primaryIndex = true; + private Boolean fts = true; private Boolean beerSample = false; @@ -146,6 +149,11 @@ public SELF withIndex(Boolean withIndex) { return self(); } + public SELF withPrimaryIndex(Boolean primaryIndex) { + this.primaryIndex = primaryIndex; + return self(); + } + public SELF withQuery(Boolean withQuery) { this.query = withQuery; return self(); @@ -231,16 +239,14 @@ public void initCluster() { } } - public void createBucket(BucketSettings bucketSetting, Boolean createIndex) { + public void createBucket(BucketSettings bucketSetting, Boolean primaryIndex) { BucketSettings bucketSettings = getCouchbaseCluster().clusterManager(clusterUsername, clusterPassword).insertBucket(bucketSetting); - // allow some time for the query service to come up - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (createIndex) { - getCouchbaseCluster().openBucket().query(Index.createPrimaryIndex().on(bucketSetting.name())); + if (index) { + Bucket bucket = getCouchbaseCluster().openBucket(bucketSettings.name()); + new CouchbaseQueryServiceWaitStrategy(bucket).waitUntilReady(this); + if (primaryIndex) { + bucket.query(Index.createPrimaryIndex().on(bucketSetting.name())); + } } } @@ -266,7 +272,7 @@ public void start() { super.start(); if (!newBuckets.isEmpty()) { for (BucketSettings bucketSetting : newBuckets) { - createBucket(bucketSetting, index); + createBucket(bucketSetting, primaryIndex); } } } diff --git a/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java b/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java new file mode 100644 index 0000000..0d1d713 --- /dev/null +++ b/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java @@ -0,0 +1,52 @@ +package com.github.couchmove.container; + +import com.couchbase.client.core.message.cluster.GetClusterConfigRequest; +import com.couchbase.client.core.message.cluster.GetClusterConfigResponse; +import com.couchbase.client.core.service.ServiceType; +import com.couchbase.client.java.Bucket; +import org.rnorth.ducttape.TimeoutException; +import org.testcontainers.containers.ContainerLaunchException; +import org.testcontainers.containers.GenericContainer; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; + +/** + * Created by tayebchlyah on 06/06/2017. + */ +public class CouchbaseQueryServiceWaitStrategy extends GenericContainer.AbstractWaitStrategy { + + private final Bucket bucket; + + public CouchbaseQueryServiceWaitStrategy(Bucket bucket) { + this.bucket = bucket; + startupTimeout = Duration.ofSeconds(120); + } + + @Override + protected void waitUntilReady() { + logger().info("Waiting for {} seconds for QUERY service", startupTimeout.getSeconds()); + + // try to connect to the URL + try { + retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { + getRateLimiter().doWhenReady(() -> { + GetClusterConfigResponse clusterConfig = bucket.core() + .send(new GetClusterConfigRequest()) + .toBlocking().single(); + boolean queryServiceEnabled = clusterConfig.config() + .bucketConfig(bucket.name()) + .serviceEnabled(ServiceType.QUERY); + if (!queryServiceEnabled) { + throw new RuntimeException("Query service not ready yet"); + } + }); + return true; + }); + } catch (TimeoutException e) { + throw new ContainerLaunchException("Timed out waiting for QUERY service"); + } + } +} From f1acd69afcf2c1dc9956fd54c65e5a8978922ea3 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Wed, 7 Jun 2017 21:17:16 +0200 Subject: [PATCH 17/42] Extract requests from n1ql files ignoring simple and multiline comments --- pom.xml | 5 +++ .../java/com/github/couchmove/CouchMove.java | 2 +- .../couchmove/service/ChangeLogDBService.java | 31 +++++++++---------- .../service/ChangeLogFileService.java | 4 --- .../service/ChangeLogDBServiceTest.java | 30 ++++++++---------- 5 files changed, 33 insertions(+), 39 deletions(-) diff --git a/pom.xml b/pom.xml index b6825d4..34f54fa 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,11 @@ 1.10.19 test + + org.assertj + assertj-core + 3.1.0 + org.hamcrest hamcrest-junit diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java index 1aa62c1..8b95ec1 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -158,7 +158,7 @@ private boolean doExecute(ChangeLog changeLog) { dbService.importDocuments(fileService.readDocuments(changeLog.getScript())); break; case N1QL: - dbService.executeN1ql(fileService.readLines(changeLog.getScript())); + dbService.executeN1ql(fileService.readFile(changeLog.getScript())); break; case DESIGN_DOC: dbService.importDesignDoc(changeLog.getDescription().replace(" ", "_"), fileService.readFile(changeLog.getScript())); diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index ea1e148..084f9da 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -5,15 +5,11 @@ import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; -import com.github.couchmove.utils.FileUtils; import org.apache.commons.io.FilenameUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; import static com.github.couchmove.utils.FunctionUtils.not; @@ -36,12 +32,12 @@ public ChangeLogDBService(Bucket bucket) { /** * Get corresponding ChangeLogs from Couchbase bucket *
    - *
  • if a {@link ChangeLog} doesn't exist => return it as it its - *
  • else : - *
      - *
    • if checksum ({@link ChangeLog#checksum}) is reset (set to null), or description ({@link ChangeLog#description}) updated => reset {@link ChangeLog#cas} - *
    • return database version - *
    + *
  • if a {@link ChangeLog} doesn't exist => return it as it its + *
  • else : + *
      + *
    • if checksum ({@link ChangeLog#checksum}) is reset (set to null), or description ({@link ChangeLog#description}) updated => reset {@link ChangeLog#cas} + *
    • return database version + *
    *
* * @param changeLogs to load from database @@ -89,8 +85,8 @@ public void importDesignDoc(String name, String content) { repository.importDesignDoc(name, content); } - public void executeN1ql(List lines) { - List requests = filterRequests(lines); + public void executeN1ql(String content) { + List requests = extractRequests(content); logger.info("Executing {} n1ql requests", requests.size()); requests.forEach(repository::query); } @@ -101,11 +97,12 @@ public void importDocuments(Map documents) { repository.save(FilenameUtils.getBaseName(fileName), content)); } - static List filterRequests(List lines) { - return lines.stream() + static List extractRequests(String content) { + String commentsRemoved = content.replaceAll("((?:--[^\\n]*)|(?s)(?:\\/\\*.*?\\*\\/))", "") + .trim(); + + return Arrays.stream(commentsRemoved.split(";")) .map(String::trim) - .filter(not(String::isEmpty)) - .filter(s -> !s.startsWith("--")) .collect(Collectors.toList()); } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index 04f499b..7fc92f5 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -68,10 +68,6 @@ public Map readDocuments(String path) throws IOException { return FileUtils.readFilesInDirectory(resolve(path).toFile(), JSON); } - public List readLines(String path) throws IOException { - return Files.readAllLines(resolve(path)); - } - // static File initializeFolder(String changePath) { Path path; diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index 7eb6369..47f5079 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -3,6 +3,7 @@ import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; +import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -15,15 +16,11 @@ import java.util.ArrayList; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; -import static com.github.couchmove.service.ChangeLogDBService.filterRequests; +import static com.github.couchmove.service.ChangeLogDBService.extractRequests; import static com.github.couchmove.utils.TestUtils.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; /** * Created by tayebchlyah on 03/06/2017. @@ -138,16 +135,15 @@ public void should_return_updated_changeLog_with_cas_reset_if_description_change @Test public void should_skip_n1ql_blank_and_comment_lines() { - String request = "CREATE INDEX PRIMARY INDEX 'primary' ON 'default'"; - List lines = Stream.of( - request, - " ", - " -- toto", - "-- hello" - ).collect(Collectors.toList()); - - List result = filterRequests(lines); - Assert.assertEquals(1, result.size()); - Assert.assertEquals(request, result.get(0)); + String request1 = "CREATE INDEX 'user_index' ON default\n" + + " WHERE type = 'user'"; + String request2 = "INSERT { 'name': 'toto'} INTO default"; + String sql = "-- create Index\n" + + request1 + ";\n" + + "\n" + + "/*insert new users*/\n" + + request2 + "; "; + + Assertions.assertThat(extractRequests(sql)).containsExactly(request1, request2); } } \ No newline at end of file From 1aadae69011901438077e4fec451710738d850db Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 10 Jun 2017 01:36:09 +0200 Subject: [PATCH 18/42] implement integration tests + bug fixes --- .../java/com/github/couchmove/CouchMove.java | 13 +- .../repository/CouchbaseRepositoryImpl.java | 9 +- .../couchmove/service/ChangeLogDBService.java | 2 +- .../couchmove/CouchMoveIntegrationTest.java | 174 ++++++++++++++++++ .../com/github/couchmove/CouchMoveTest.java | 11 +- .../java/com/github/couchmove/pojo/User.java | 15 ++ .../service/ChangeLogFileServiceTest.java | 4 +- .../github/couchmove/utils/FileUtilsTest.java | 4 +- .../com/github/couchmove/utils/TestUtils.java | 16 ++ .../db/migration/V1__create_index.n1ql | 2 - .../db/migration/fail/V1__insert_users.n1ql | 8 + .../migration/fail/V2__invalid_request.n1ql | 2 + .../db/migration/skip/V1.2__type.json | 0 .../db/migration/skip/V2__create_index.n1ql | 2 + .../V1.1__insert_users/user::titi.json | 0 .../V1.1__insert_users/user::toto.json | 0 .../migration/success/V1__create_index.n1ql | 3 + .../db/migration/{ => success}/V2__user.json | 0 .../db/migration/update/V1__create_index.n1ql | 2 + 19 files changed, 249 insertions(+), 18 deletions(-) create mode 100644 src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java create mode 100644 src/test/java/com/github/couchmove/pojo/User.java delete mode 100644 src/test/resources/db/migration/V1__create_index.n1ql create mode 100644 src/test/resources/db/migration/fail/V1__insert_users.n1ql create mode 100644 src/test/resources/db/migration/fail/V2__invalid_request.n1ql create mode 100644 src/test/resources/db/migration/skip/V1.2__type.json create mode 100644 src/test/resources/db/migration/skip/V2__create_index.n1ql rename src/test/resources/db/migration/{ => success}/V1.1__insert_users/user::titi.json (100%) rename src/test/resources/db/migration/{ => success}/V1.1__insert_users/user::toto.json (100%) create mode 100644 src/test/resources/db/migration/success/V1__create_index.n1ql rename src/test/resources/db/migration/{ => success}/V2__user.json (100%) create mode 100644 src/test/resources/db/migration/update/V1__create_index.n1ql diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java index 8b95ec1..7be2530 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -9,6 +9,7 @@ import com.google.common.base.Stopwatch; import lombok.AccessLevel; import lombok.NoArgsConstructor; +import lombok.Setter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -35,6 +36,7 @@ public class CouchMove { private ChangeLockService lockService; + @Setter(AccessLevel.PACKAGE) private ChangeLogDBService dbService; private ChangeLogFileService fileService; @@ -71,6 +73,8 @@ public void migrate() { // Executing migration executeMigration(changeLogs); + } catch (Exception e) { + throw new CouchMoveException("Unable to migrate", e); } finally { // Release lock lockService.releaseLock(); @@ -119,8 +123,8 @@ void executeMigration(List changeLogs) { continue; } - if (executeMigration(changeLog)) { - changeLog.setOrder(++lastOrder); + if (executeMigration(changeLog, lastOrder + 1)) { + lastOrder++; lastVersion = changeLog.getVersion(); migrationCount++; } else { @@ -134,13 +138,14 @@ void executeMigration(List changeLogs) { } } - boolean executeMigration(ChangeLog changeLog) { + boolean executeMigration(ChangeLog changeLog, int order) { logger.info("Executing ChangeLog '{}'", changeLog.getVersion()); Stopwatch sw = Stopwatch.createStarted(); changeLog.setTimestamp(new Date()); changeLog.setRunner(getUsername()); if (doExecute(changeLog)) { logger.info("ChangeLog '{}' successfully executed", changeLog.getVersion()); + changeLog.setOrder(order); changeLog.setStatus(EXECUTED); } else { logger.error("Unable to execute ChangeLog '{}'", changeLog.getVersion()); @@ -151,7 +156,7 @@ boolean executeMigration(ChangeLog changeLog) { return changeLog.getStatus() == EXECUTED; } - private boolean doExecute(ChangeLog changeLog) { + boolean doExecute(ChangeLog changeLog) { try { switch (changeLog.getType()) { case DOCUMENTS: diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 3ac41d3..66d8969 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -5,6 +5,7 @@ import com.couchbase.client.java.Bucket; import com.couchbase.client.java.document.RawJsonDocument; import com.couchbase.client.java.document.json.JsonObject; +import com.couchbase.client.java.error.DocumentDoesNotExistException; import com.couchbase.client.java.query.N1qlQuery; import com.couchbase.client.java.query.N1qlQueryResult; import com.couchbase.client.java.view.DesignDocument; @@ -24,7 +25,7 @@ * Created by tayebchlyah on 27/05/2017. */ // For tests -@NoArgsConstructor(access = AccessLevel.PACKAGE,force = true) +@NoArgsConstructor(access = AccessLevel.PACKAGE, force = true) @RequiredArgsConstructor public class CouchbaseRepositoryImpl implements CouchbaseRepository { @@ -85,7 +86,11 @@ public E checkAndSave(String id, E entity) { @Override public void delete(String id) { - bucket.remove(id); + try { + bucket.remove(id); + } catch (DocumentDoesNotExistException e) { + logger.warn("Trying to delete document that does not exist : '{}'", id); + } } @Override diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 084f9da..08ccff7 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -64,7 +64,7 @@ public List fetchAndCompare(List changeLogs) { throw new CouchMoveException("ChangeLog checksum doesn't match"); } if (!dbChangeLog.getDescription().equals(changeLog.getDescription())) { - logger.warn("ChangeLog version '{}' description updated"); + logger.warn("ChangeLog version '{}' description updated", changeLog.getDescription()); logger.debug("{} was {}", dbChangeLog, changeLog); dbChangeLog.setDescription(changeLog.getDescription()); dbChangeLog.setScript(changeLog.getScript()); diff --git a/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java new file mode 100644 index 0000000..ae75caf --- /dev/null +++ b/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java @@ -0,0 +1,174 @@ +package com.github.couchmove; + +import com.couchbase.client.java.query.util.IndexInfo; +import com.couchbase.client.java.view.DesignDocument; +import com.github.couchmove.container.AbstractCouchbaseTest; +import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Status; +import com.github.couchmove.pojo.Type; +import com.github.couchmove.pojo.User; +import com.github.couchmove.repository.CouchbaseRepository; +import com.github.couchmove.repository.CouchbaseRepositoryImpl; +import com.github.couchmove.service.ChangeLogDBService; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static com.github.couchmove.pojo.Status.*; +import static com.github.couchmove.pojo.Type.*; +import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; +import static com.github.couchmove.utils.TestUtils.assertThrows; +import static org.junit.Assert.*; + +/** + * Created by tayebchlyah on 05/06/2017. + */ +public class CouchMoveIntegrationTest extends AbstractCouchbaseTest { + + public static final String DB_MIGRATION = "db/migration/"; + + private static CouchbaseRepository changeLogRepository; + + private static ChangeLogDBService changeLogDBService; + + private static CouchbaseRepositoryImpl userRepository; + + @BeforeClass + public static void init() { + changeLogRepository = new CouchbaseRepositoryImpl<>(getBucket(), ChangeLog.class); + changeLogDBService = new ChangeLogDBService(getBucket()); + userRepository = new CouchbaseRepositoryImpl<>(getBucket(), User.class); + } + + @Test + public void should_migrate_successfully() { + // Given a CouchMove instance configured for success migration folder + CouchMove couchMove = new CouchMove(getBucket(), DB_MIGRATION + "success"); + + // When we launch migration + couchMove.migrate(); + + // Then all changeLogs should be inserted in DB + List changeLogs = Stream.of("1", "1.1", "2") + .map(version -> PREFIX_ID + version) + .map(changeLogRepository::findOne) + .collect(Collectors.toList()); + + assertEquals(3, changeLogs.size()); + assertLike(changeLogs.get(0), + "1", 1, "create index", N1QL, "V1__create_index.n1ql", + "eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5", + EXECUTED); + assertLike(changeLogs.get(1), + "1.1", 2, "insert users", DOCUMENTS, "V1.1__insert_users", + "99a4aaf12e7505286afe2a5b074f7ebabd496f3ea8c4093116efd3d096c430a8", + EXECUTED); + assertLike(changeLogs.get(2), + "2", 3, "user", Type.DESIGN_DOC, "V2__user.json", + "22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79", + EXECUTED); + + // And successfully executed + + // Users inserted + assertEquals(new User("user", "titi", "01/09/1998"), userRepository.findOne("user::titi")); + assertEquals(new User("user", "toto", "10/01/1991"), userRepository.findOne("user::toto")); + + // Index inserted + Optional userIndexInfo = getBucket().bucketManager().listN1qlIndexes().stream() + .filter(i -> i.name().equals("user_index")) + .findFirst(); + assertTrue(userIndexInfo.isPresent()); + assertEquals("`username`", userIndexInfo.get().indexKey().get(0)); + + // Design Document inserted + DesignDocument designDocument = getBucket().bucketManager().getDesignDocument("user"); + assertNotNull(designDocument); + } + + @Test + public void should_skip_old_migration_version() { + // Given an executed changeLog + changeLogDBService.save(ChangeLog.builder() + .version("2") + .order(3) + .type(N1QL) + .description("create index") + .script("V2__create_index.n1ql") + .checksum("69eb9007c910c2b9cac46044a54de5e933b768ae874c6408356372576ab88dbd") + .runner("toto") + .timestamp(new Date()) + .duration(400L) + .status(EXECUTED) + .build()); + + // When we execute migration in skip migration folder + new CouchMove(getBucket(), DB_MIGRATION + "skip").migrate(); + + // Then the old ChangeLog is marked as skipped + assertLike(changeLogRepository.findOne(PREFIX_ID + "1.2"), "1.2", null, "type", DESIGN_DOC, "V1.2__type.json", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", SKIPPED); + } + + @Test + public void should_migration_fail_on_exception() { + // Given a CouchMove instance configured for fail migration folder + CouchMove couchMove = new CouchMove(getBucket(), DB_MIGRATION + "fail"); + + // When we launch migration, then an exception should be raised + assertThrows(couchMove::migrate, CouchMoveException.class); + + // Then new ChangeLog is executed + assertLike(changeLogRepository.findOne(PREFIX_ID + "1"), "1", 1, "insert users", N1QL, "V1__insert_users.n1ql", "efcc80f763e48e2a1d5b6689351ad1b4d678c70bebc0c0975a2d19f94e938f18", EXECUTED); + + assertEquals(new User("admin", "Administrator", "01/09/1998"), userRepository.findOne("user::Administrator")); + + // And the ChangeLog marked as failed + assertLike(changeLogRepository.findOne(PREFIX_ID + "2"), "2", null, "invalid request", N1QL, "V2__invalid_request.n1ql", "890c7bac55666a3073059c57f34e358f817e275eb68932e946ca35e9dcd428fe", FAILED); + } + + @Test + public void should_update_changeLog() { + // Given an executed changeLog + changeLogDBService.save(ChangeLog.builder() + .version("1") + .order(1) + .type(N1QL) + .description("insert users") + .script("V2__insert_users.n1ql") + .checksum("69eb9007c910c2b9cac46044a54de5e933b768ae874c6408356372576ab88dbd") + .runner("toto") + .timestamp(new Date()) + .duration(400L) + .status(EXECUTED) + .build()); + + // When we execute migration in update migration folder + new CouchMove(getBucket(), DB_MIGRATION + "update").migrate(); + + // Then executed changeLog description updated + assertLike(changeLogRepository.findOne(PREFIX_ID + "1"), "1", 1, "create index", N1QL, "V1__create_index.n1ql", "69eb9007c910c2b9cac46044a54de5e933b768ae874c6408356372576ab88dbd", EXECUTED); + } + + private static void assertLike(ChangeLog changeLog, String version, Integer order, String description, Type type, String script, String checksum, Status status) { + assertNotNull("ChangeLog", changeLog); + assertEquals("version", version, changeLog.getVersion()); + assertEquals("order", order, changeLog.getOrder()); + assertEquals("description", description, changeLog.getDescription()); + assertEquals("type", type, changeLog.getType()); + assertEquals("script", script, changeLog.getScript()); + assertEquals(checksum, changeLog.getChecksum()); + assertEquals(status, changeLog.getStatus()); + if (changeLog.getStatus() != SKIPPED) { + assertNotNull("runner", changeLog.getRunner()); + assertNotNull("timestamp", changeLog.getTimestamp()); + assertNotNull("duration", changeLog.getDuration()); + } + } + +} diff --git a/src/test/java/com/github/couchmove/CouchMoveTest.java b/src/test/java/com/github/couchmove/CouchMoveTest.java index 464f6ed..dc5553e 100644 --- a/src/test/java/com/github/couchmove/CouchMoveTest.java +++ b/src/test/java/com/github/couchmove/CouchMoveTest.java @@ -106,6 +106,7 @@ public void should_migration_skip_changeLog_with_old_version() { @Test public void should_execute_migrations() { CouchMove couchMove = spy(CouchMove.class); + couchMove.setDbService(dbServiceMock); ChangeLog executedChangeLog = ChangeLog.builder() .version("1") .order(1) @@ -116,8 +117,8 @@ public void should_execute_migrations() { .version("2") .type(DOCUMENTS) .build(); - doReturn(true).when(couchMove).executeMigration(changeLog); - couchMove.executeMigration(newArrayList(newArrayList(executedChangeLog, changeLog))); + doReturn(true).when(couchMove).doExecute(changeLog); + couchMove.executeMigration(newArrayList(executedChangeLog, changeLog)); Assert.assertEquals((Integer) 2, changeLog.getOrder()); } @@ -128,7 +129,7 @@ public void should_throw_exception_if_migration_failed() { .version("1") .type(N1QL) .build(); - doReturn(false).when(couchMove).executeMigration(changeLog); + doReturn(false).when(couchMove).executeMigration(changeLog, 1); couchMove.executeMigration(newArrayList(changeLog)); } @@ -139,7 +140,7 @@ public void should_update_changeLog_on_migration_success() { .description("description") .type(DESIGN_DOC) .build(); - couchMove.executeMigration(changeLog); + couchMove.executeMigration(changeLog, 1); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); @@ -154,7 +155,7 @@ public void should_update_changeLog_on_migration_failure() { .type(DOCUMENTS) .build(); doThrow(CouchMoveException.class).when(dbServiceMock).importDocuments(any()); - couchMove.executeMigration(changeLog); + couchMove.executeMigration(changeLog, 1); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); diff --git a/src/test/java/com/github/couchmove/pojo/User.java b/src/test/java/com/github/couchmove/pojo/User.java new file mode 100644 index 0000000..a4c740b --- /dev/null +++ b/src/test/java/com/github/couchmove/pojo/User.java @@ -0,0 +1,15 @@ +package com.github.couchmove.pojo; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * Created by tayebchlyah on 07/06/2017. + */ +@EqualsAndHashCode(callSuper = false) +@Data +public class User extends CouchbaseEntity { + private final String type; + private final String username; + private final String birthday; +} diff --git a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java index df34195..fa18a7e 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java @@ -59,7 +59,7 @@ public void should_fetch_changeLogs() { .script("V1__create_index.n1ql") .version("1") .description("create index") - .checksum("bf0dae5d8fb638627eeabfb4b649d6100a8960da18859e12874e61063fbb16be") + .checksum("eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5") .build(), ChangeLog.builder() .type(Type.DOCUMENTS) @@ -76,7 +76,7 @@ public void should_fetch_changeLogs() { .checksum("22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79") .build()) .collect(Collectors.toList()); - Assert.assertEquals(changeLogs, new ChangeLogFileService("db/migration").fetch()); + Assert.assertEquals(changeLogs, new ChangeLogFileService("db/migration/success").fetch()); } } \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java index f29a35f..cef12da 100644 --- a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -23,7 +23,7 @@ @RunWith(DataProviderRunner.class) public class FileUtilsTest { - public static final String DB_MIGRATION_PATH = "db/migration/"; + public static final String DB_MIGRATION_PATH = "db/migration/success/"; @Test public void should_get_file_path_from_resource() throws Exception { @@ -47,7 +47,7 @@ public void should_get_folder_path_from_resource() throws Exception { public static Object[][] fileProvider() { return new Object[][]{ {DB_MIGRATION_PATH + "V1.1__insert_users", "99a4aaf12e7505286afe2a5b074f7ebabd496f3ea8c4093116efd3d096c430a8"}, - {DB_MIGRATION_PATH + "V1__create_index.n1ql", "bf0dae5d8fb638627eeabfb4b649d6100a8960da18859e12874e61063fbb16be"}, + {DB_MIGRATION_PATH + "V1__create_index.n1ql", "eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5"}, {DB_MIGRATION_PATH + "V2__user.json", "22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79"} }; } diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java index 06f8fb8..4c2d513 100644 --- a/src/test/java/com/github/couchmove/utils/TestUtils.java +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -3,11 +3,14 @@ import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Status; import com.github.couchmove.pojo.Type; +import org.assertj.core.api.Assertions; import org.jetbrains.annotations.NotNull; import java.util.Random; import java.util.UUID; +import static org.junit.Assert.assertTrue; + /** * Created by tayebchlyah on 01/06/2017. */ @@ -35,4 +38,17 @@ public static ChangeLog getRandomChangeLog() { .status(Status.values()[Math.abs(RANDOM.nextInt(Status.values().length))]) .build(); } + + @SafeVarargs + public static void assertThrows(Runnable runnable, Class... throwables) { + boolean exceptionOccurred = false; + try { + runnable.run(); + // Then an exception should raise + } catch (Exception e) { + Assertions.assertThat(e).isOfAnyClassIn(throwables); + exceptionOccurred = true; + } + assertTrue("Expected exception occurred", exceptionOccurred); + } } diff --git a/src/test/resources/db/migration/V1__create_index.n1ql b/src/test/resources/db/migration/V1__create_index.n1ql deleted file mode 100644 index a5e1ba6..0000000 --- a/src/test/resources/db/migration/V1__create_index.n1ql +++ /dev/null @@ -1,2 +0,0 @@ -CREATE INDEX 'user_index' ON default - WHERE type = 'user' \ No newline at end of file diff --git a/src/test/resources/db/migration/fail/V1__insert_users.n1ql b/src/test/resources/db/migration/fail/V1__insert_users.n1ql new file mode 100644 index 0000000..9f0a2be --- /dev/null +++ b/src/test/resources/db/migration/fail/V1__insert_users.n1ql @@ -0,0 +1,8 @@ +INSERT INTO default (KEY, VALUE) + VALUES + ("user::Administrator", + { + "type": "admin", + "username": "Administrator", + "birthday": "01/09/1998" + }); \ No newline at end of file diff --git a/src/test/resources/db/migration/fail/V2__invalid_request.n1ql b/src/test/resources/db/migration/fail/V2__invalid_request.n1ql new file mode 100644 index 0000000..b6faf3d --- /dev/null +++ b/src/test/resources/db/migration/fail/V2__invalid_request.n1ql @@ -0,0 +1,2 @@ +-- should fail +INVALID REQUEST \ No newline at end of file diff --git a/src/test/resources/db/migration/skip/V1.2__type.json b/src/test/resources/db/migration/skip/V1.2__type.json new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/db/migration/skip/V2__create_index.n1ql b/src/test/resources/db/migration/skip/V2__create_index.n1ql new file mode 100644 index 0000000..fe07b04 --- /dev/null +++ b/src/test/resources/db/migration/skip/V2__create_index.n1ql @@ -0,0 +1,2 @@ +-- should not be executed +INVALID REQUEST \ No newline at end of file diff --git a/src/test/resources/db/migration/V1.1__insert_users/user::titi.json b/src/test/resources/db/migration/success/V1.1__insert_users/user::titi.json similarity index 100% rename from src/test/resources/db/migration/V1.1__insert_users/user::titi.json rename to src/test/resources/db/migration/success/V1.1__insert_users/user::titi.json diff --git a/src/test/resources/db/migration/V1.1__insert_users/user::toto.json b/src/test/resources/db/migration/success/V1.1__insert_users/user::toto.json similarity index 100% rename from src/test/resources/db/migration/V1.1__insert_users/user::toto.json rename to src/test/resources/db/migration/success/V1.1__insert_users/user::toto.json diff --git a/src/test/resources/db/migration/success/V1__create_index.n1ql b/src/test/resources/db/migration/success/V1__create_index.n1ql new file mode 100644 index 0000000..89065a6 --- /dev/null +++ b/src/test/resources/db/migration/success/V1__create_index.n1ql @@ -0,0 +1,3 @@ +-- create Index +CREATE INDEX user_index ON default(username) + WHERE type = 'user'; \ No newline at end of file diff --git a/src/test/resources/db/migration/V2__user.json b/src/test/resources/db/migration/success/V2__user.json similarity index 100% rename from src/test/resources/db/migration/V2__user.json rename to src/test/resources/db/migration/success/V2__user.json diff --git a/src/test/resources/db/migration/update/V1__create_index.n1ql b/src/test/resources/db/migration/update/V1__create_index.n1ql new file mode 100644 index 0000000..fe07b04 --- /dev/null +++ b/src/test/resources/db/migration/update/V1__create_index.n1ql @@ -0,0 +1,2 @@ +-- should not be executed +INVALID REQUEST \ No newline at end of file From 203d8c1610dd537f2c6f14dbc932313ff2659e84 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 10 Jun 2017 14:06:36 +0200 Subject: [PATCH 19/42] Fix maven compilation : lombok doesn't work with static import --- pom.xml | 8 +------- src/main/java/com/github/couchmove/CouchMove.java | 4 ++-- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/pom.xml b/pom.xml index 34f54fa..22e39a4 100644 --- a/pom.xml +++ b/pom.xml @@ -104,12 +104,6 @@ assertj-core 3.1.0 - - org.hamcrest - hamcrest-junit - 2.0.0.0 - test - ch.qos.logback logback-classic @@ -123,7 +117,7 @@ org.apache.maven.plugins maven-compiler-plugin - 2.4 + 3.6.1 ${java.version} ${java.version} diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java index 7be2530..216b81d 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -6,6 +6,7 @@ import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; +import com.github.couchmove.utils.Utils; import com.google.common.base.Stopwatch; import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -20,7 +21,6 @@ import java.util.concurrent.TimeUnit; import static com.github.couchmove.pojo.Status.*; -import static com.github.couchmove.utils.Utils.getUsername; /** * Created by tayebchlyah on 03/06/2017. @@ -142,7 +142,7 @@ boolean executeMigration(ChangeLog changeLog, int order) { logger.info("Executing ChangeLog '{}'", changeLog.getVersion()); Stopwatch sw = Stopwatch.createStarted(); changeLog.setTimestamp(new Date()); - changeLog.setRunner(getUsername()); + changeLog.setRunner(Utils.getUsername()); if (doExecute(changeLog)) { logger.info("ChangeLog '{}' successfully executed", changeLog.getVersion()); changeLog.setOrder(order); From 1f120b941f73e990b392c363d04482e49d00bd14 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 10 Jun 2017 18:24:28 +0200 Subject: [PATCH 20/42] Add javadoc --- .../java/com/github/couchmove/CouchMove.java | 62 +++++++++++++++++- .../exception/CouchMoveException.java | 3 +- .../com/github/couchmove/pojo/ChangeLock.java | 18 +++++- .../com/github/couchmove/pojo/ChangeLog.java | 36 ++++++++++- .../couchmove/pojo/CouchbaseEntity.java | 10 ++- .../com/github/couchmove/pojo/Status.java | 17 ++++- .../java/com/github/couchmove/pojo/Type.java | 19 +++++- .../repository/CouchbaseRepository.java | 63 ++++++++++++++++++- .../repository/CouchbaseRepositoryImpl.java | 22 +------ .../couchmove/service/ChangeLockService.java | 20 +++++- .../couchmove/service/ChangeLogDBService.java | 43 ++++++++++++- .../service/ChangeLogFileService.java | 33 +++++++++- .../com/github/couchmove/utils/FileUtils.java | 15 ++++- .../github/couchmove/utils/FunctionUtils.java | 12 +++- .../com/github/couchmove/utils/Utils.java | 6 +- .../couchmove/CouchMoveIntegrationTest.java | 3 +- .../com/github/couchmove/CouchMoveTest.java | 3 +- .../container/AbstractCouchbaseTest.java | 2 +- .../CouchbaseQueryServiceWaitStrategy.java | 3 +- .../java/com/github/couchmove/pojo/User.java | 3 +- .../repository/CouchbaseRepositoryTest.java | 5 +- .../service/ChangeLockServiceTest.java | 3 +- .../service/ChangeLogDBServiceTest.java | 3 +- .../service/ChangeLogFileServiceTest.java | 3 +- .../github/couchmove/utils/FileUtilsTest.java | 3 +- .../com/github/couchmove/utils/TestUtils.java | 3 +- 26 files changed, 362 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/CouchMove.java index 216b81d..12aed4f 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/CouchMove.java @@ -1,8 +1,13 @@ package com.github.couchmove; import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Status; +import com.github.couchmove.pojo.Type; +import com.github.couchmove.pojo.Type.Constants; import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; @@ -23,7 +28,10 @@ import static com.github.couchmove.pojo.Status.*; /** - * Created by tayebchlyah on 03/06/2017. + * Couchmove Runner + * + * @author ctayeb + * Created on 03/06/2017 */ @NoArgsConstructor(access = AccessLevel.PACKAGE) public class CouchMove { @@ -41,10 +49,21 @@ public class CouchMove { private ChangeLogFileService fileService; + /** + * Initialize a {@link CouchMove} instance with default migration path : {@value DEFAULT_MIGRATION_PATH} + * + * @param bucket Couchbase {@link Bucket} to execute the migrations on + */ public CouchMove(Bucket bucket) { this(bucket, DEFAULT_MIGRATION_PATH); } + /** + * Initialize a {@link CouchMove} instance + * + * @param bucket Couchbase {@link Bucket} to execute the migrations on + * @param changePath absolute or relative path of the migration folder containing {@link ChangeLog} + */ public CouchMove(Bucket bucket, String changePath) { logger.info("Connected to bucket '{}'", bucketName = bucket.name()); lockService = new ChangeLockService(bucket); @@ -52,6 +71,15 @@ public CouchMove(Bucket bucket, String changePath) { fileService = new ChangeLogFileService(changePath); } + /** + * Launch the migration process : + *
    + *
  1. Tries to acquire Couchbase {@link Bucket} lock + *
  2. Fetch all {@link ChangeLog}s from migration folder + *
  3. Fetch corresponding {@link ChangeLog}s from {@link Bucket} + *
  4. Execute found {@link ChangeLog}s : {@link CouchMove#executeMigration(List)} + *
+ */ public void migrate() { logger.info("Begin bucket '{}' migration", bucketName); try { @@ -82,6 +110,17 @@ public void migrate() { logger.info("CouchMove has finished his job"); } + /** + * Execute the {@link ChangeLog}s + *
    + *
  • If {@link ChangeLog#version} is lower than last executed one, ignore it and mark it as {@link Status#SKIPPED} + *
  • If an {@link Status#EXECUTED} ChangeLog was modified, fail + *
  • If an {@link Status#EXECUTED} ChangeLog description was modified, update it + *
  • Otherwise apply the ChangeLog : {@link CouchMove#executeMigration(ChangeLog, int)} + *
+ * + * @param changeLogs to execute + */ void executeMigration(List changeLogs) { logger.info("Executing migration scripts..."); int migrationCount = 0; @@ -138,6 +177,17 @@ void executeMigration(List changeLogs) { } } + /** + * Execute the migration {@link ChangeLog}, and save it to Couchbase {@link Bucket} + *
    + *
  • If the execution was successful, set the order and mark it as {@link Status#EXECUTED} + *
  • Otherwise, mark it as {@link Status#FAILED} + *
+ * + * @param changeLog {@link ChangeLog} to execute + * @param order the order to set if the execution was successful + * @return true if the execution was successful, false otherwise + */ boolean executeMigration(ChangeLog changeLog, int order) { logger.info("Executing ChangeLog '{}'", changeLog.getVersion()); Stopwatch sw = Stopwatch.createStarted(); @@ -156,6 +206,16 @@ boolean executeMigration(ChangeLog changeLog, int order) { return changeLog.getStatus() == EXECUTED; } + /** + * Applies the {@link ChangeLog} according to it's {@link ChangeLog#type} : + *
    + *
  • {@link Type#DOCUMENTS} : Imports all {@value Constants#JSON} documents contained in the folder + *
  • {@link Type#N1QL} : Execute all {@link N1qlQuery} contained in the {@value Constants#N1QL} file + *
  • {@link Type#DESIGN_DOC} : Imports {@link DesignDocument} contained in the {@value Constants#JSON} document + *
+ * @param changeLog {@link ChangeLog} to apply + * @return true if the execution was successful, false otherwise + */ boolean doExecute(ChangeLog changeLog) { try { switch (changeLog.getType()) { diff --git a/src/main/java/com/github/couchmove/exception/CouchMoveException.java b/src/main/java/com/github/couchmove/exception/CouchMoveException.java index 0336f25..3fd83f7 100644 --- a/src/main/java/com/github/couchmove/exception/CouchMoveException.java +++ b/src/main/java/com/github/couchmove/exception/CouchMoveException.java @@ -1,7 +1,8 @@ package com.github.couchmove.exception; /** - * Created by tayebchlyah on 28/05/2017. + * @author ctayeb + * Created on 28/05/2017 */ public class CouchMoveException extends RuntimeException { public CouchMoveException(String message, Throwable cause) { diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLock.java b/src/main/java/com/github/couchmove/pojo/ChangeLock.java index a32e287..aec0ab4 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLock.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLock.java @@ -1,22 +1,38 @@ package com.github.couchmove.pojo; +import com.couchbase.client.java.Bucket; import lombok.Data; import lombok.EqualsAndHashCode; import java.util.Date; /** - * Created by tayebchlyah on 27/05/2017. + * a {@link CouchbaseEntity} representing a pessimistic locking of a Couchbase {@link Bucket} + * + * @author ctayeb + * Created on 27/05/2017 */ @EqualsAndHashCode(callSuper = false) @Data public class ChangeLock extends CouchbaseEntity { + /** + * Determines if the {@link Bucket} is locked + */ private boolean locked; + /** + * Unique ID identifying instance that acquires the lock + */ private String uuid; + /** + * The OS username of the process holding the lock + */ private String runner; + /** + * The date when the {@link Bucket} was locked + */ private Date timestamp; } diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java index 8870b43..b55ec57 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLog.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -1,5 +1,6 @@ package com.github.couchmove.pojo; +import com.couchbase.client.java.Bucket; import lombok.Builder; import lombok.Data; import lombok.EqualsAndHashCode; @@ -8,31 +9,64 @@ import java.util.Date; /** - * Created by tayebchlyah on 27/05/2017. + * a {@link CouchbaseEntity} representing a change in Couchbase {@link Bucket} + * + * @author ctayeb + * Created on 27/05/2017 */ @EqualsAndHashCode(callSuper = false) @Builder(toBuilder = true) @Data public class ChangeLog extends CouchbaseEntity implements Comparable { + /** + * The version of the change + */ private String version; + /** + * The execution order of the change + */ private Integer order; + /** + * The description of the change + */ private String description; + /** + * The {@link Type} of the change + */ private Type type; + /** + * The script file or folder that was executed in the change + */ private String script; + /** + * A unique identifier of the file or folder of the change + */ private String checksum; + /** + * The OS username of the process that executed the change + */ private String runner; + /** + * Date of execution of the change + */ private Date timestamp; + /** + * The duration of the execution of the change in milliseconds + */ private Long duration; + /** + * The {@link Status} of the change + */ private Status status; @Override diff --git a/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java b/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java index 9c498c4..c54a724 100644 --- a/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java +++ b/src/main/java/com/github/couchmove/pojo/CouchbaseEntity.java @@ -5,11 +5,19 @@ import org.jetbrains.annotations.Nullable; /** - * Created by tayebchlyah on 28/05/2017. + * Class representing a json Document + * + * @author ctayeb + * Created on 28/05/2017 */ @Data public class CouchbaseEntity { + /** + * The last-known CAS value for the Document + *

+ * CAS is for Check And Swap, which is an identifier that permit optimistic concurrency + */ @Nullable @JsonIgnore private Long cas; diff --git a/src/main/java/com/github/couchmove/pojo/Status.java b/src/main/java/com/github/couchmove/pojo/Status.java index be82d2d..9c71f43 100644 --- a/src/main/java/com/github/couchmove/pojo/Status.java +++ b/src/main/java/com/github/couchmove/pojo/Status.java @@ -1,10 +1,25 @@ package com.github.couchmove.pojo; /** - * Created by tayebchlyah on 03/06/2017. + * Describes the current status of a {@link ChangeLog} execution + * + * @author ctayeb + * Created on 03/06/2017 */ public enum Status { + + /** + * The {@link ChangeLog} was successfully executed + */ EXECUTED, + + /** + * The {@link ChangeLog} execution has failed + */ FAILED, + + /** + * The {@link ChangeLog} execution was ignored + */ SKIPPED } diff --git a/src/main/java/com/github/couchmove/pojo/Type.java b/src/main/java/com/github/couchmove/pojo/Type.java index 0b45d60..0adeb6b 100644 --- a/src/main/java/com/github/couchmove/pojo/Type.java +++ b/src/main/java/com/github/couchmove/pojo/Type.java @@ -1,13 +1,30 @@ package com.github.couchmove.pojo; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.view.DesignDocument; import lombok.Getter; /** - * Created by tayebchlyah on 27/05/2017. + * Describes the type of the {@link ChangeLog} + * + * @author ctayeb + * Created on 27/05/2017 */ public enum Type { + + /** + * json documents + */ DOCUMENTS(""), + + /** + * json document representing a {@link DesignDocument} + */ DESIGN_DOC(Constants.JSON), + + /** + * n1ql file containing a list of {@link N1qlQuery} + */ N1QL(Constants.N1QL); @Getter diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java index f16189e..a8049d4 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -1,25 +1,86 @@ package com.github.couchmove.repository; +import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.pojo.CouchbaseEntity; /** - * Created by tayebchlyah on 27/05/2017. + * A repository for encapsulating storage, retrieval, and removal of json documents to Couchbase {@link Bucket} + * + * @param the domain type the repository manages + * @author ctayeb + * Created on 27/05/2017 */ public interface CouchbaseRepository { + /** + * Convert an {@link CouchbaseEntity} to json document, and save it to Couchbase {@link Bucket} + * + * @param id the per-bucket unique document id + * @param entity entity to convert and save + * @return saved entity with CAS (Compare and Swap) for optimistic concurrency + */ E save(String id, E entity); + /** + * If the {@link CouchbaseEntity#cas} of the entity is set, tries to replace the document with a Check And Swap operation (for optimistic concurrency) + *

+ * Otherwise it {@link CouchbaseRepository#save(String, CouchbaseEntity)} + * + * @param id the per-bucket unique document id + * @param entity entity to convert and save + * @return saved entity with CAS + * @throws com.couchbase.client.java.error.CASMismatchException if the cas of entity is different from existing one + * @throws com.couchbase.client.java.error.DocumentAlreadyExistsException if the cas is not set and the document exists on couchbase + */ E checkAndSave(String id, E entity); + /** + * Removes a {@link CouchbaseEntity} from Couchbase Bucket identified by its id + * + * @param id the id of the document to remove + */ void delete(String id); + /** + * Retrieves a document from Couchbase {@link Bucket} by its ID. + *

+ *

+ * - If the document exists, convert it to {@link CouchbaseEntity} with CAS set (Check And Swap for optimistic concurrency) + *
+ * - Otherwise it return null + * + * @param id the id of the document + * @return the found and converted {@link CouchbaseEntity} with CAS set, or null if absent + */ E findOne(String id); + /** + * Save a json document buy its ID + * + * @param id the per-bucket unique document id + * @param jsonContent content of the json document + */ void save(String id, String jsonContent); + /** + * Inserts a {@link DesignDocument} into production + * + * @param name name of the {@link DesignDocument} to insert + * @param jsonContent the content of the {@link DesignDocument} to insert + */ void importDesignDoc(String name, String jsonContent); + /** + * Queries Couchbase {@link Bucket} with a {@link N1qlQuery} + * + * @param request {@link N1qlQuery} in String format + */ void query(String request); + /** + * @return name of the repository Couchbase {@link Bucket} + */ String getBucketName(); } diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 66d8969..7d1ea63 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -22,7 +22,8 @@ import static org.slf4j.LoggerFactory.getLogger; /** - * Created by tayebchlyah on 27/05/2017. + * @author ctayeb + * Created on 27/05/2017 */ // For tests @NoArgsConstructor(access = AccessLevel.PACKAGE, force = true) @@ -38,13 +39,7 @@ public class CouchbaseRepositoryImpl implements Couch private final Class entityClass; - /** - * Save the entity on couchbase - * - * @param id document id - * @param entity entity to save - * @return saved entity - */ + @Override public E save(String id, E entity) { try { @@ -56,17 +51,6 @@ public E save(String id, E entity) { } } - /** - * If the {@link CouchbaseEntity#cas} of the entity is set, tries to replace the document with a Check And Set operation (Optimistic locking) - *

- * otherwise it insert the document - * - * @param id document id - * @param entity entity to save - * @return saved entity - * @throws com.couchbase.client.java.error.CASMismatchException if the cas of entity is different from existing one - * @throws com.couchbase.client.java.error.DocumentAlreadyExistsException if the cas is not set and the document exists on couchbase - */ @Override public E checkAndSave(String id, E entity) { try { diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index bcf54d2..41c2ee2 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -14,7 +14,10 @@ import java.util.UUID; /** - * Created by tayebchlyah on 28/05/2017. + * Service for acquiring a pessimistic lock of a Couchbase {@link Bucket} + * + * @author ctayeb + * Created on 27/05/2017 */ public class ChangeLockService { @@ -30,6 +33,11 @@ public ChangeLockService(Bucket bucket) { this.repository = new CouchbaseRepositoryImpl<>(bucket, ChangeLock.class); } + /** + * Tries to acquire a pessimistic lock of Couchbase {@link Bucket} + * + * @return true if lock successfully acquired, false otherwise + */ public boolean acquireLock() { logger.info("Trying to acquire bucket '{}' lock...", repository.getBucketName()); // Verify if there is any lock on database @@ -59,6 +67,11 @@ public boolean acquireLock() { return true; } + /** + * Check if the Couchbase {@link Bucket} is actually locked by this instance + * + * @return true if the current instance holds the lock, false otherwise. + */ public boolean isLockAcquired() { ChangeLock lock = repository.findOne(LOCK_ID); if (lock == null) { @@ -74,10 +87,11 @@ public boolean isLockAcquired() { return true; } + /** + * Releases the pessimistic lock of Couchbase {@link Bucket} + */ public void releaseLock() { logger.info("Release lock"); repository.delete(LOCK_ID); } - - // } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 08ccff7..28a4be6 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -1,6 +1,8 @@ package com.github.couchmove.service; import com.couchbase.client.java.Bucket; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; @@ -12,10 +14,11 @@ import java.util.*; import java.util.stream.Collectors; -import static com.github.couchmove.utils.FunctionUtils.not; - /** - * Created by tayebchlyah on 03/06/2017. + * Service for fetching and executing {@link ChangeLog}s + * + * @author ctayeb + * Created on 03/06/2017 */ public class ChangeLogDBService { @@ -76,27 +79,61 @@ public List fetchAndCompare(List changeLogs) { return Collections.unmodifiableList(result); } + /** + * Saves a {@link ChangeLog} in Couchbase {@link Bucket} using an ID composed by : + *

+ * {@value PREFIX_ID} + {@link ChangeLog#version} + * + * @param changeLog The ChangeLog to save + * @return {@link ChangeLog} entity with CAS (Check And Swap, for optimistic concurrency) set + */ public ChangeLog save(ChangeLog changeLog) { return repository.save(PREFIX_ID + changeLog.getVersion(), changeLog); } + /** + * Inserts a {@link DesignDocument} into production + * + * @param name name of the {@link DesignDocument} to insert + * @param content the content of the {@link DesignDocument} to insert + */ public void importDesignDoc(String name, String content) { logger.info("Inserting Design Document '{}'...", name); repository.importDesignDoc(name, content); } + /** + * Queries Couchbase {@link Bucket} with multiple {@link N1qlQuery} + * + * @param content containing multiple {@link N1qlQuery} + */ public void executeN1ql(String content) { List requests = extractRequests(content); logger.info("Executing {} n1ql requests", requests.size()); requests.forEach(repository::query); } + /** + * Save multiple json documents to Couchbase {@link Bucket} identified by the keys of the map + * + * @param documents a {@link Map} which keys represent a json document to be inserted, and the values the unique ID of the document + */ public void importDocuments(Map documents) { logger.info("Importing {} documents", documents.size()); documents.forEach((fileName, content) -> repository.save(FilenameUtils.getBaseName(fileName), content)); } + /** + * Extract multiple requests, separated by ';' ignoring : + *

    + *
  • multi-line (\/* ... *\/) comments + *
  • unique line (-- ...) comments + *
+ * + * @param content content from where the requests are extracted + * @return multiple requests + */ static List extractRequests(String content) { String commentsRemoved = content.replaceAll("((?:--[^\\n]*)|(?s)(?:\\/\\*.*?\\*\\/))", "") .trim(); diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index 7fc92f5..7b1fd5e 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -23,7 +23,10 @@ import static com.github.couchmove.pojo.Type.N1QL; /** - * Created by tayebchlyah on 30/05/2017. + * Service for fetching {@link ChangeLog}s from resource folder + * + * @author ctayeb + * Created on 30/05/2017 */ public class ChangeLogFileService { @@ -33,10 +36,18 @@ public class ChangeLogFileService { private final File changeFolder; + /** + * @param changePath The resource path of the folder containing {@link ChangeLog}s + */ public ChangeLogFileService(String changePath) { this.changeFolder = initializeFolder(changePath); } + /** + * Reads all the {@link ChangeLog}s contained in the Change Folder, ignoring unhandled files + * + * @return An ordered list of {@link ChangeLog}s by {@link ChangeLog#version} + */ public List fetch() { logger.info("Fetching changeLogs from migration folder '{}'", changeFolder.getPath()); SortedSet sortedChangeLogs = new TreeSet<>(); @@ -60,10 +71,24 @@ public List fetch() { return Collections.unmodifiableList(new ArrayList<>(sortedChangeLogs)); } + /** + * Read file content from a relative path from the Change Folder + * + * @param path relative path of the file to read + * @return content of the file + * @throws IOException if an I/O error occurs reading the file + */ public String readFile(String path) throws IOException { return new String(Files.readAllBytes(resolve(path))); } + /** + * Read json files content from a relative directory from the Change Folder + * + * @param path relative path of the directory containing json files to read + * @return {@link Map} which keys represents the name (with extension), and values the content of read files + * @throws IOException if an I/O error occurs reading the files + */ public Map readDocuments(String path) throws IOException { return FileUtils.readFilesInDirectory(resolve(path).toFile(), JSON); } @@ -89,6 +114,12 @@ private Path resolve(String path) { return changeFolder.toPath().resolve(path); } + /** + * Determines the {@link Type} of the file from its type and extension + * + * @param file file to analyse + * @return {@link Type} of the {@link ChangeLog} file + */ @NotNull static Type getChangeLogType(File file) { if (file.isDirectory()) { diff --git a/src/main/java/com/github/couchmove/utils/FileUtils.java b/src/main/java/com/github/couchmove/utils/FileUtils.java index b210b17..37db12e 100644 --- a/src/main/java/com/github/couchmove/utils/FileUtils.java +++ b/src/main/java/com/github/couchmove/utils/FileUtils.java @@ -21,16 +21,17 @@ import static org.apache.commons.io.IOUtils.toByteArray; /** - * Created by tayebchlyah on 02/06/2017. + * @author ctayeb + * Created on 02/06/2017 */ public class FileUtils { /** - * Returns Path of a resource in classpath no matter whether in a jar or in a folder + * Returns Path of a resource in classpath no matter whether it is in a jar or in absolute or relative folder * * @param resource path * @return Path of a resource - * @throws IOException + * @throws IOException if an I/O error occurs */ public static Path getPathFromResource(String resource) throws IOException { File file = new File(resource); @@ -99,6 +100,14 @@ public static String calculateChecksum(@NotNull File file, String... extensions) } } + /** + * Read files content from a (@link File} + * + * @param file The directory containing files to read + * @param extensions The extensions of the files to read + * @return {@link Map} which keys represents the name (with extension), and values the content of read files + * @throws IOException if an I/O error occurs reading the files + */ public static Map readFilesInDirectory(@NotNull File file, String... extensions) throws IOException { if (file == null || !file.exists()) { throw new IllegalArgumentException("File is null or doesn't exists"); diff --git a/src/main/java/com/github/couchmove/utils/FunctionUtils.java b/src/main/java/com/github/couchmove/utils/FunctionUtils.java index e42b502..5afa8e9 100644 --- a/src/main/java/com/github/couchmove/utils/FunctionUtils.java +++ b/src/main/java/com/github/couchmove/utils/FunctionUtils.java @@ -3,9 +3,19 @@ import java.util.function.Predicate; /** - * Created by tayebchlyah on 05/06/2017. + * @author ctayeb + * Created on 05/06/2017 */ public class FunctionUtils { + /** + * Returns a {@link Predicate} that represents the logical negation of this + * predicate. + * + * @param predicate {@link Predicate} to negate + * @param the type of the input to the predicate + * @return a predicate that represents the logical negation of this + * predicate + */ public static Predicate not(Predicate predicate) { return predicate.negate(); } diff --git a/src/main/java/com/github/couchmove/utils/Utils.java b/src/main/java/com/github/couchmove/utils/Utils.java index 0ccf806..318367c 100644 --- a/src/main/java/com/github/couchmove/utils/Utils.java +++ b/src/main/java/com/github/couchmove/utils/Utils.java @@ -8,12 +8,16 @@ import java.lang.reflect.Method; /** - * Created by tayebchlyah on 04/06/2017. + * @author ctayeb + * Created on 04/06/2017 */ public class Utils { private static final Logger logger = LoggerFactory.getLogger(Utils.class); + /** + * Get the username for the current OS user + */ @Getter(lazy = true) private static final String username = initializeUserName(); diff --git a/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java index ae75caf..d44d427 100644 --- a/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java +++ b/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java @@ -27,7 +27,8 @@ import static org.junit.Assert.*; /** - * Created by tayebchlyah on 05/06/2017. + * @author ctayeb + * Created on 05/06/2017 */ public class CouchMoveIntegrationTest extends AbstractCouchbaseTest { diff --git a/src/test/java/com/github/couchmove/CouchMoveTest.java b/src/test/java/com/github/couchmove/CouchMoveTest.java index dc5553e..49b9b25 100644 --- a/src/test/java/com/github/couchmove/CouchMoveTest.java +++ b/src/test/java/com/github/couchmove/CouchMoveTest.java @@ -23,7 +23,8 @@ import static org.mockito.Mockito.*; /** - * Created by tayebchlyah on 04/06/2017. + * @author ctayeb + * Created on 04/06/2017 */ @RunWith(MockitoJUnitRunner.class) public class CouchMoveTest { diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index c2f06b2..2407e19 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -11,7 +11,7 @@ import org.slf4j.LoggerFactory; /** - * Created by tayebchlyah on 28/05/2017. + * @author ctayeb */ public abstract class AbstractCouchbaseTest { public static final String CLUSTER_USER = "Administrator"; diff --git a/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java b/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java index 0d1d713..fe950d5 100644 --- a/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java +++ b/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java @@ -14,7 +14,8 @@ import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; /** - * Created by tayebchlyah on 06/06/2017. + * @author ctayeb + * Created on 06/06/2017 */ public class CouchbaseQueryServiceWaitStrategy extends GenericContainer.AbstractWaitStrategy { diff --git a/src/test/java/com/github/couchmove/pojo/User.java b/src/test/java/com/github/couchmove/pojo/User.java index a4c740b..bce8cec 100644 --- a/src/test/java/com/github/couchmove/pojo/User.java +++ b/src/test/java/com/github/couchmove/pojo/User.java @@ -4,7 +4,8 @@ import lombok.EqualsAndHashCode; /** - * Created by tayebchlyah on 07/06/2017. + * @author ctayeb + * Created on 07/06/2017 */ @EqualsAndHashCode(callSuper = false) @Data diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index da2805d..8dfa786 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -20,7 +20,8 @@ import static com.github.couchmove.utils.TestUtils.getRandomString; /** - * Created by tayebchlyah on 28/05/2017. + * @author ctayeb + * Created on 28/05/2017 */ public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { @@ -130,7 +131,7 @@ public void should_execute_n1ql() { // When we execute the query repository.query(request); - // Then the index should be created + // Then the index should be Created List indexInfos = getBucket().bucketManager().listN1qlIndexes().stream() .filter(not(IndexInfo::isPrimary)) .collect(Collectors.toList()); diff --git a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java index bca5439..3a71551 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java @@ -5,7 +5,8 @@ import org.junit.Test; /** - * Created by tayebchlyah on 29/05/2017. + * @author ctayeb + * Created on 29/05/2017 */ public class ChangeLockServiceTest extends AbstractCouchbaseTest { diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index 47f5079..c127e2f 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -23,7 +23,8 @@ import static org.mockito.Mockito.when; /** - * Created by tayebchlyah on 03/06/2017. + * @author ctayeb + * Created on 03/06/2017 */ @RunWith(MockitoJUnitRunner.class) public class ChangeLogDBServiceTest { diff --git a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java index fa18a7e..9a86289 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java @@ -15,7 +15,8 @@ import static com.github.couchmove.utils.TestUtils.getRandomString; /** - * Created by tayebchlyah on 01/06/2017. + * @author ctayeb + * Created on 01/06/2017 */ public class ChangeLogFileServiceTest { diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java index cef12da..88c46a4 100644 --- a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -18,7 +18,8 @@ import static com.github.couchmove.utils.TestUtils.getRandomString; /** - * Created by tayebchlyah on 02/06/2017. + * @author ctayeb + * Created on 02/06/2017 */ @RunWith(DataProviderRunner.class) public class FileUtilsTest { diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java index 4c2d513..6a5ed3a 100644 --- a/src/test/java/com/github/couchmove/utils/TestUtils.java +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -12,7 +12,8 @@ import static org.junit.Assert.assertTrue; /** - * Created by tayebchlyah on 01/06/2017. + * @author ctayeb + * Created on 01/06/2017 */ public class TestUtils { From 63894ff6086f60d9283c31f24067c12daab7a554 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 10 Jun 2017 18:55:24 +0200 Subject: [PATCH 21/42] Travis integration --- .travis.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..33aee02 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,6 @@ +sudo: required + +language: java + +services: + - docker \ No newline at end of file From c6c4f89329552461424528208dc0307e031c7ac2 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 16:33:41 +0200 Subject: [PATCH 22/42] Fix n1ql request test --- .../couchmove/repository/CouchbaseRepositoryTest.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 8dfa786..aa831f4 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -18,6 +18,7 @@ import static com.github.couchmove.utils.FunctionUtils.not; import static com.github.couchmove.utils.TestUtils.getRandomString; +import static java.lang.String.format; /** * @author ctayeb @@ -25,6 +26,8 @@ */ public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { + public static final String INDEX_NAME = "name"; + private static CouchbaseRepository repository; @BeforeClass @@ -126,7 +129,7 @@ public void should_import_design_doc() { @Test public void should_execute_n1ql() { // Given a primary index request - String request = String.format("CREATE INDEX `%s` ON `%s`(`%s`)", "name", getBucket().name(), "name"); + String request = format("CREATE INDEX `%s` ON `%s`(`%s`)", INDEX_NAME, getBucket().name(), INDEX_NAME); // When we execute the query repository.query(request); @@ -137,8 +140,8 @@ public void should_execute_n1ql() { .collect(Collectors.toList()); Assert.assertEquals(1, indexInfos.size()); IndexInfo indexInfo = indexInfos.get(0); - Assert.assertEquals("name", indexInfo.name()); - Assert.assertEquals("name", indexInfo.indexKey().get(0)); + Assert.assertEquals(INDEX_NAME, indexInfo.name()); + Assert.assertEquals(format("`%s`", INDEX_NAME), indexInfo.indexKey().get(0)); } @Test From f53d73105c66851dfc8651955c94cf0f5156dc29 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 16:34:54 +0200 Subject: [PATCH 23/42] Customize all configurable ports to fix travis issue : Port already in use --- .../container/AbstractCouchbaseTest.java | 17 +++++++++-------- .../couchmove/container/CouchbaseContainer.java | 10 +++++----- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index 2407e19..eb39f66 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -40,15 +40,16 @@ private static CouchbaseContainer initCouchbaseContainer() { .withQuery(true) .withPrimaryIndex(false) .withClusterUsername(CLUSTER_USER) - .withClusterPassword(CLUSTER_PASSWORD) - .withNewBucket(DefaultBucketSettings.builder() - .enableFlush(true) - .name(DEFAULT_BUCKET) - .quota(100) - .replicas(0) - .type(BucketType.COUCHBASE) - .build()); + .withClusterPassword(CLUSTER_PASSWORD); couchbaseContainer.start(); + couchbaseContainer.createBucket(DefaultBucketSettings.builder() + .enableFlush(true) + .name(DEFAULT_BUCKET) + .quota(100) + .replicas(0) + .port(couchbaseContainer.getMappedPort(CouchbaseContainer.BUCKET_PORT)) + .type(BucketType.COUCHBASE) + .build(), false); return couchbaseContainer; } diff --git a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java index 4ed9150..5055f15 100644 --- a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java +++ b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java @@ -38,6 +38,7 @@ */ public class CouchbaseContainer> extends GenericContainer { + public static final int BUCKET_PORT = 11211; private String memoryQuota = "400"; private String indexMemoryQuota = "400"; @@ -85,15 +86,14 @@ protected Integer getLivenessCheckPort() { @Override protected void configure() { - addFixedExposedPort(8091, 8091); + // Configurable ports + addExposedPorts(11210, 11207, 8091, 18091, BUCKET_PORT); + + // Non configurable ports addFixedExposedPort(8092, 8092); addFixedExposedPort(8093, 8093); addFixedExposedPort(8094, 8094); addFixedExposedPort(8095, 8095); - addFixedExposedPort(11207, 11207); - addFixedExposedPort(11210, 11210); - addFixedExposedPort(11211, 11211); - addFixedExposedPort(18091, 18091); addFixedExposedPort(18092, 18092); addFixedExposedPort(18093, 18093); setWaitStrategy(new HttpWaitStrategy().forPath("/ui/index.html#/")); From 2887331018d2962c6b94bd1afe7f30994a29e190 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 18:32:51 +0200 Subject: [PATCH 24/42] Run all Couchbase queries with retrying strategy --- .../repository/CouchbaseRepositoryImpl.java | 66 +++++++++++++++---- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 7d1ea63..d8e7bc2 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -1,13 +1,21 @@ package com.github.couchmove.repository; +import com.couchbase.client.core.BackpressureException; +import com.couchbase.client.core.RequestCancelledException; +import com.couchbase.client.core.time.Delay; import com.couchbase.client.deps.com.fasterxml.jackson.core.JsonProcessingException; import com.couchbase.client.deps.com.fasterxml.jackson.databind.ObjectMapper; +import com.couchbase.client.java.AsyncBucket; import com.couchbase.client.java.Bucket; import com.couchbase.client.java.document.RawJsonDocument; import com.couchbase.client.java.document.json.JsonObject; import com.couchbase.client.java.error.DocumentDoesNotExistException; +import com.couchbase.client.java.error.TemporaryFailureException; +import com.couchbase.client.java.query.AsyncN1qlQueryResult; +import com.couchbase.client.java.query.N1qlParams; import com.couchbase.client.java.query.N1qlQuery; -import com.couchbase.client.java.query.N1qlQueryResult; +import com.couchbase.client.java.util.retry.RetryBuilder; +import com.couchbase.client.java.util.retry.RetryWhenFunction; import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.CouchbaseEntity; @@ -16,14 +24,19 @@ import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import org.slf4j.Logger; +import rx.Observable; +import javax.annotation.Nullable; import java.io.IOException; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import static com.couchbase.client.java.query.consistency.ScanConsistency.STATEMENT_PLUS; import static org.slf4j.LoggerFactory.getLogger; /** * @author ctayeb - * Created on 27/05/2017 + * Created on 27/05/2017 */ // For tests @NoArgsConstructor(access = AccessLevel.PACKAGE, force = true) @@ -39,11 +52,14 @@ public class CouchbaseRepositoryImpl implements Couch private final Class entityClass; + @Getter(lazy = true, value = AccessLevel.PRIVATE) + private static final RetryWhenFunction retryStrategy = retryStrategy(); @Override public E save(String id, E entity) { try { - RawJsonDocument insertedDocument = bucket.upsert(RawJsonDocument.create(id, getJsonMapper().writeValueAsString(entity))); + String json = getJsonMapper().writeValueAsString(entity); + RawJsonDocument insertedDocument = runAsync(bucket -> bucket.upsert(RawJsonDocument.create(id, json))); entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { @@ -56,11 +72,12 @@ public E checkAndSave(String id, E entity) { try { String content = getJsonMapper().writeValueAsString(entity); RawJsonDocument insertedDocument; - if (entity.getCas() != null) { - insertedDocument = bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); - } else { - insertedDocument = bucket.insert(RawJsonDocument.create(id, content)); - } + insertedDocument = runAsync(bucket -> { + if (entity.getCas() != null) { + return bucket.replace(RawJsonDocument.create(id, content, entity.getCas())); + } + return bucket.insert(RawJsonDocument.create(id, content)); + }); entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { @@ -71,7 +88,7 @@ public E checkAndSave(String id, E entity) { @Override public void delete(String id) { try { - bucket.remove(id); + runAsync(bucket -> bucket.remove(id)); } catch (DocumentDoesNotExistException e) { logger.warn("Trying to delete document that does not exist : '{}'", id); } @@ -79,7 +96,7 @@ public void delete(String id) { @Override public E findOne(String id) { - RawJsonDocument document = bucket.get(id, RawJsonDocument.class); + RawJsonDocument document = runAsync(bucket -> bucket.get(id, RawJsonDocument.class)); if (document == null) { return null; } @@ -94,7 +111,7 @@ public E findOne(String id) { @Override public void save(String id, String jsonContent) { - bucket.upsert(RawJsonDocument.create(id, jsonContent)); + runAsync(bucket -> bucket.upsert(RawJsonDocument.create(id, jsonContent))); } @Override @@ -106,13 +123,15 @@ public void importDesignDoc(String name, String jsonContent) { public void query(String n1qlStatement) { logger.debug("Executing n1ql request : {}", n1qlStatement); try { - N1qlQueryResult result = bucket.query(N1qlQuery.simple(n1qlStatement)); + AsyncN1qlQueryResult result = runAsync(bucket -> bucket + .query(N1qlQuery.simple(n1qlStatement, + N1qlParams.build().consistency(STATEMENT_PLUS)))); if (!result.parseSuccess()) { logger.info("Invalid N1QL request '{}'", n1qlStatement); throw new CouchMoveException("Invalid n1ql request"); } - if (!result.finalSuccess()) { - logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : ", n1qlStatement, result.status(), result.errors()); + if (!single(result.finalSuccess())) { + logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : ", n1qlStatement, single(result.status()), single(result.errors())); throw new CouchMoveException("Unable to execute n1ql request"); } } catch (Exception e) { @@ -124,4 +143,23 @@ public void query(String n1qlStatement) { public String getBucketName() { return bucket.name(); } + + @Nullable + private T single(Observable observable) { + return observable.toBlocking().singleOrDefault(null); + } + + private R runAsync(Function> function) { + return single(function.apply(bucket.async()) + .retryWhen(getRetryStrategy())); + } + + @SuppressWarnings("unchecked") + private static RetryWhenFunction retryStrategy() { + return RetryBuilder + .anyOf(TemporaryFailureException.class, RequestCancelledException.class, BackpressureException.class) + .delay(Delay.exponential(TimeUnit.MILLISECONDS, 100)) + .max(3) + .build(); + } } From 60e3012a399539a1ec4209cb1f4930129bc3e125 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 18:33:19 +0200 Subject: [PATCH 25/42] Fix n1ql request test --- .../github/couchmove/repository/CouchbaseRepositoryTest.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index aa831f4..582155b 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -16,13 +16,12 @@ import java.util.Random; import java.util.stream.Collectors; -import static com.github.couchmove.utils.FunctionUtils.not; import static com.github.couchmove.utils.TestUtils.getRandomString; import static java.lang.String.format; /** * @author ctayeb - * Created on 28/05/2017 + * Created on 28/05/2017 */ public class CouchbaseRepositoryTest extends AbstractCouchbaseTest { @@ -136,7 +135,7 @@ public void should_execute_n1ql() { // Then the index should be Created List indexInfos = getBucket().bucketManager().listN1qlIndexes().stream() - .filter(not(IndexInfo::isPrimary)) + .filter(indexInfo -> indexInfo.name().equals(INDEX_NAME)) .collect(Collectors.toList()); Assert.assertEquals(1, indexInfos.size()); IndexInfo indexInfo = indexInfos.get(0); From dfa9e8427e25c1bcfdc9ba2cc099d9b46dc7cc7a Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 18:35:47 +0200 Subject: [PATCH 26/42] Accelerate integration tests : replacing bucket flush with N1ql delete statement --- .../container/AbstractCouchbaseTest.java | 15 +++++++++++--- .../container/CouchbaseContainer.java | 20 +++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java index eb39f66..ece88b8 100644 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java @@ -6,6 +6,9 @@ import com.couchbase.client.java.CouchbaseCluster; import com.couchbase.client.java.bucket.BucketType; import com.couchbase.client.java.cluster.DefaultBucketSettings; +import com.couchbase.client.java.query.N1qlParams; +import com.couchbase.client.java.query.N1qlQuery; +import com.couchbase.client.java.query.consistency.ScanConsistency; import lombok.Getter; import org.junit.After; import org.slf4j.LoggerFactory; @@ -30,7 +33,13 @@ public abstract class AbstractCouchbaseTest { @After public void clear() { - getBucket().bucketManager().flush(); + if (getCouchbaseContainer().isIndex() && getCouchbaseContainer().isQuery() && getCouchbaseContainer().isPrimaryIndex()) { + getBucket().query( + N1qlQuery.simple(String.format("DELETE FROM `%s`", getBucket().name()), + N1qlParams.build().consistency(ScanConsistency.STATEMENT_PLUS))); + } else { + getBucket().bucketManager().flush(); + } } private static CouchbaseContainer initCouchbaseContainer() { @@ -38,7 +47,7 @@ private static CouchbaseContainer initCouchbaseContainer() { .withFTS(false) .withIndex(true) .withQuery(true) - .withPrimaryIndex(false) + .withPrimaryIndex(true) .withClusterUsername(CLUSTER_USER) .withClusterPassword(CLUSTER_PASSWORD); couchbaseContainer.start(); @@ -49,7 +58,7 @@ private static CouchbaseContainer initCouchbaseContainer() { .replicas(0) .port(couchbaseContainer.getMappedPort(CouchbaseContainer.BUCKET_PORT)) .type(BucketType.COUCHBASE) - .build(), false); + .build(), couchbaseContainer.isPrimaryIndex()); return couchbaseContainer; } diff --git a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java index 5055f15..567bd9d 100644 --- a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java +++ b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java @@ -22,6 +22,7 @@ import com.couchbase.client.java.env.CouchbaseEnvironment; import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; import com.couchbase.client.java.query.Index; +import lombok.Getter; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.wait.HttpWaitStrategy; @@ -47,21 +48,24 @@ public class CouchbaseContainer> extends G private String clusterPassword = "password"; - private Boolean keyValue = true; + private boolean keyValue = true; - private Boolean query = true; + @Getter + private boolean query = true; - private Boolean index = true; + @Getter + private boolean index = true; - private Boolean primaryIndex = true; + @Getter + private boolean primaryIndex = true; - private Boolean fts = true; + private boolean fts = true; - private Boolean beerSample = false; + private boolean beerSample = false; - private Boolean travelSample = false; + private boolean travelSample = false; - private Boolean gamesIMSample = false; + private boolean gamesIMSample = false; private CouchbaseEnvironment couchbaseEnvironment; From c361e9b95fe2cc267aa44cd95080df0e97eac4a7 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 18:59:13 +0200 Subject: [PATCH 27/42] Database Lock : Release lock acquired only by the same process --- .../couchmove/service/ChangeLockService.java | 14 ++++++- .../service/ChangeLockServiceTest.java | 41 +++++++++++++++---- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index 41c2ee2..e20924d 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -3,6 +3,7 @@ import com.couchbase.client.java.Bucket; import com.couchbase.client.java.error.CASMismatchException; import com.couchbase.client.java.error.DocumentAlreadyExistsException; +import com.github.couchmove.exception.CouchMoveException; import com.github.couchmove.pojo.ChangeLock; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; @@ -17,7 +18,7 @@ * Service for acquiring a pessimistic lock of a Couchbase {@link Bucket} * * @author ctayeb - * Created on 27/05/2017 + * Created on 27/05/2017 */ public class ChangeLockService { @@ -91,6 +92,17 @@ public boolean isLockAcquired() { * Releases the pessimistic lock of Couchbase {@link Bucket} */ public void releaseLock() { + if (isLockAcquired()) { + forceReleaseLock(); + } else { + throw new CouchMoveException("Unable to release lock acquired by an other process"); + } + } + + /** + * Force release pessimistic lock even if the current instance doesn't hold it + */ + public void forceReleaseLock() { logger.info("Release lock"); repository.delete(LOCK_ID); } diff --git a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java index 3a71551..6091213 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java @@ -1,12 +1,16 @@ package com.github.couchmove.service; import com.github.couchmove.container.AbstractCouchbaseTest; -import org.junit.Assert; +import com.github.couchmove.exception.CouchMoveException; import org.junit.Test; +import static com.github.couchmove.utils.TestUtils.assertThrows; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + /** * @author ctayeb - * Created on 29/05/2017 + * Created on 29/05/2017 */ public class ChangeLockServiceTest extends AbstractCouchbaseTest { @@ -16,16 +20,16 @@ public void should_acquire_and_release_lock() { ChangeLockService changeLockService = new ChangeLockService(getBucket()); // When we tries to acquire lock - Assert.assertTrue(changeLockService.acquireLock()); + assertTrue(changeLockService.acquireLock()); // Then we should get it - Assert.assertTrue(changeLockService.isLockAcquired()); + assertTrue(changeLockService.isLockAcquired()); // When we release the lock changeLockService.releaseLock(); // The it should be released - Assert.assertFalse(changeLockService.isLockAcquired()); + assertFalse(changeLockService.isLockAcquired()); } @Test @@ -38,10 +42,31 @@ public void should_not_acquire_lock_when_already_acquired() { ChangeLockService changeLockService2 = new ChangeLockService(getBucket()); // Then it will fails - Assert.assertFalse(changeLockService2.acquireLock()); - Assert.assertFalse(changeLockService2.isLockAcquired()); + assertFalse(changeLockService2.acquireLock()); + assertFalse(changeLockService2.isLockAcquired()); // And the first service should keep the lock - Assert.assertTrue(changeLockService1.isLockAcquired()); + assertTrue(changeLockService1.isLockAcquired()); + } + + @Test + public void should_not_release_lock_acquired_by_another_process() { + // Given a process holding the lock + ChangeLockService changeLockService1 = new ChangeLockService(getBucket()); + changeLockService1.acquireLock(); + assertTrue(changeLockService1.isLockAcquired()); + + // When an other process tries to release the lock + ChangeLockService changeLockService2 = new ChangeLockService(getBucket()); + + // Then it should fails + assertThrows(changeLockService2::releaseLock, CouchMoveException.class); + + // When an other process force release the lock + changeLockService2.forceReleaseLock(); + + // Then the first should loose the lock + assertFalse(changeLockService1.isLockAcquired()); } + } \ No newline at end of file From 17d00962bb0819b8a2045d4cecca3dedb081fe61 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 11 Jun 2017 21:55:33 +0200 Subject: [PATCH 28/42] Add SonarQube Scanner --- .travis.yml | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 33aee02..cd9dd58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,4 +3,23 @@ sudo: required language: java services: - - docker \ No newline at end of file + - docker + +addons: + sonarqube: + organization: "differentway-github" + token: + secure: "fnzS5t/buOCMWV5xx8sGOcPG+6P3LZwFuyOjOI9efm8s6uaYc2dHaKz9A45W2FB6O74pCohO8hVOh+C610fdnlES4JZ3kEy0V4/NWB9jun6w0dT+kPFRCUPmcyrS1zVZvEtlzuy2dUSPUgKfWQKnufZMex1VxghJge022+bGKbxsSYCcn0/EkOnHKN3hcP/WtjgfMQ7NrGrR+nGzZIblQRDL2bLyhx7skI7aVyo4qv93GyFGk5dIqmJtXlh+p8ylzImrJnM+V74NbRQe+YkgYZbH1VNaAzhCiSCRc8YltrAyJXJ1kLS778rIaQptLu2kn3wsZbC1dgGikg0rhy++on/cMvYWPo8LhQO7hGq31pTIblXI3+l0aU+FrKCXbpofIxbXwzBmZUOLa+StfnB9ANvsC9sn2RZ0A73U7lo/4jGY5EmjjyCze7TcyDonySyA/BrmwvDgnxKXrkAcI5jsY4bK+3Zy8pZkCYhoqilTwMsvs54m5skmLA3qv6l4tdmtNRgZD3EUnNutkjpp86gbrMa6d4k0/b2pxSjnK+MhQWKcpgXbH+Z935gTVTUcWxslu+kPXOhuH2uuiScOCCc0O/R7kPjVbFWakdjylOLFubaGKi9PmCyoYfiAcjfhFGoD7t6pXWjQdo3aRSp1d20qnioKi/c0w66hVQImiSU77Dw=" + branches: + - master + - develop + +script: + # JaCoCo is used to have code coverage, the agent has to be activated + - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar + +cache: + directories: + - '$HOME/.m2/repository' + - '$HOME/.sonar/cache' + From e681e9b58b125121bf54f5622b255abd305b9a11 Mon Sep 17 00:00:00 2001 From: differentway Date: Mon, 12 Jun 2017 14:33:09 +0200 Subject: [PATCH 29/42] Update README --- README.md | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a01e02b..60d7e23 100644 --- a/README.md +++ b/README.md @@ -1 +1,19 @@ -# couchmove \ No newline at end of file + + +### What is Couchmove? + +Couchmove is an open-source Java migration tool for [Couchbase](https://www.couchbase.com/), which is an NoSQL document-oriented database, well know for it's performance and scalability. + +Couchmove can help you *track*, *manage* and *apply changes* in your Couchbase buckets. The concept is very similar to other database migration tools such as [Liquibase](http://www.liquibase.org), [Flyway](http://flywaydb.org), [mongeez](https://github.com/secondmarket/mongeez), [mongobee](http://mongodb-tools.com/tool/mongobee/) ... + +### What's special? + +Couchmove is widely inspired from [Flyway](http://flywaydb.org) : it strongly favors simplicity and convention over configuration : There is *no XML configuration files*. Just **json documents**, **design documents** to import and **n1ql** files to execute. + +### How to use? + +Check out our [wiki](https://github.com/differentway/couchmove/wiki) + + +--- +[![Build Status](https://travis-ci.org/differentway/couchmove.svg?branch=develop)](https://travis-ci.org/differentway/couchmove) [![Licence](https://img.shields.io/hexpm/l/plug.svg)](https://github.com/differentway/couchmove/blob/master/LICENSE) From fce503aa5f49f42c3fe23f85738fa0e9e5b27bd4 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 18 Jun 2017 15:11:20 +0200 Subject: [PATCH 30/42] Fix Jackson no suitable constructor found --- src/main/java/com/github/couchmove/pojo/ChangeLog.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/github/couchmove/pojo/ChangeLog.java b/src/main/java/com/github/couchmove/pojo/ChangeLog.java index b55ec57..09c4d95 100644 --- a/src/main/java/com/github/couchmove/pojo/ChangeLog.java +++ b/src/main/java/com/github/couchmove/pojo/ChangeLog.java @@ -1,19 +1,21 @@ package com.github.couchmove.pojo; import com.couchbase.client.java.Bucket; -import lombok.Builder; -import lombok.Data; -import lombok.EqualsAndHashCode; +import lombok.*; import org.jetbrains.annotations.NotNull; import java.util.Date; +import static lombok.AccessLevel.PRIVATE; + /** * a {@link CouchbaseEntity} representing a change in Couchbase {@link Bucket} * * @author ctayeb - * Created on 27/05/2017 + * Created on 27/05/2017 */ +@AllArgsConstructor +@NoArgsConstructor(access = PRIVATE) @EqualsAndHashCode(callSuper = false) @Builder(toBuilder = true) @Data From 632757f1d53910875e358855e46ec86119674621 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 18 Jun 2017 15:17:53 +0200 Subject: [PATCH 31/42] Update maven groupId --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 22e39a4..03d6e64 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ 2.4.5 - com.github + com.github.differentway couchmove develop-SNAPSHOT From f6e7b6099569e4e2fba66fb33ccbb14cd96907a8 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 18 Jun 2017 15:19:06 +0200 Subject: [PATCH 32/42] Rename CouchMove to Couchmove --- pom.xml | 2 +- .../{CouchMove.java => Couchmove.java} | 30 ++++++------- .../exception/CouchMoveException.java | 15 ------- .../exception/CouchmoveException.java | 15 +++++++ .../repository/CouchbaseRepositoryImpl.java | 14 +++---- .../couchmove/service/ChangeLockService.java | 4 +- .../couchmove/service/ChangeLogDBService.java | 6 +-- .../service/ChangeLogFileService.java | 10 ++--- .../com/github/couchmove/utils/FileUtils.java | 6 +-- ...est.java => CouchmoveIntegrationTest.java} | 20 ++++----- ...{CouchMoveTest.java => CouchmoveTest.java} | 42 +++++++++---------- .../service/ChangeLockServiceTest.java | 4 +- .../service/ChangeLogDBServiceTest.java | 4 +- .../service/ChangeLogFileServiceTest.java | 8 ++-- 14 files changed, 90 insertions(+), 90 deletions(-) rename src/main/java/com/github/couchmove/{CouchMove.java => Couchmove.java} (91%) delete mode 100644 src/main/java/com/github/couchmove/exception/CouchMoveException.java create mode 100644 src/main/java/com/github/couchmove/exception/CouchmoveException.java rename src/test/java/com/github/couchmove/{CouchMoveIntegrationTest.java => CouchmoveIntegrationTest.java} (92%) rename src/test/java/com/github/couchmove/{CouchMoveTest.java => CouchmoveTest.java} (81%) diff --git a/pom.xml b/pom.xml index 03d6e64..28540b3 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - CouchMove + Couchmove Couchbase data migration tool for Java https://github.com/differentway/couchmove diff --git a/src/main/java/com/github/couchmove/CouchMove.java b/src/main/java/com/github/couchmove/Couchmove.java similarity index 91% rename from src/main/java/com/github/couchmove/CouchMove.java rename to src/main/java/com/github/couchmove/Couchmove.java index 12aed4f..e775f3a 100644 --- a/src/main/java/com/github/couchmove/CouchMove.java +++ b/src/main/java/com/github/couchmove/Couchmove.java @@ -3,7 +3,7 @@ import com.couchbase.client.java.Bucket; import com.couchbase.client.java.query.N1qlQuery; import com.couchbase.client.java.view.DesignDocument; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Status; import com.github.couchmove.pojo.Type; @@ -34,11 +34,11 @@ * Created on 03/06/2017 */ @NoArgsConstructor(access = AccessLevel.PACKAGE) -public class CouchMove { +public class Couchmove { public static final String DEFAULT_MIGRATION_PATH = "db/migration"; - private static final Logger logger = LoggerFactory.getLogger(CouchMove.class); + private static final Logger logger = LoggerFactory.getLogger(Couchmove.class); private String bucketName; @@ -50,21 +50,21 @@ public class CouchMove { private ChangeLogFileService fileService; /** - * Initialize a {@link CouchMove} instance with default migration path : {@value DEFAULT_MIGRATION_PATH} + * Initialize a {@link Couchmove} instance with default migration path : {@value DEFAULT_MIGRATION_PATH} * * @param bucket Couchbase {@link Bucket} to execute the migrations on */ - public CouchMove(Bucket bucket) { + public Couchmove(Bucket bucket) { this(bucket, DEFAULT_MIGRATION_PATH); } /** - * Initialize a {@link CouchMove} instance + * Initialize a {@link Couchmove} instance * * @param bucket Couchbase {@link Bucket} to execute the migrations on * @param changePath absolute or relative path of the migration folder containing {@link ChangeLog} */ - public CouchMove(Bucket bucket, String changePath) { + public Couchmove(Bucket bucket, String changePath) { logger.info("Connected to bucket '{}'", bucketName = bucket.name()); lockService = new ChangeLockService(bucket); dbService = new ChangeLogDBService(bucket); @@ -77,7 +77,7 @@ public CouchMove(Bucket bucket, String changePath) { *
  • Tries to acquire Couchbase {@link Bucket} lock *
  • Fetch all {@link ChangeLog}s from migration folder *
  • Fetch corresponding {@link ChangeLog}s from {@link Bucket} - *
  • Execute found {@link ChangeLog}s : {@link CouchMove#executeMigration(List)} + *
  • Execute found {@link ChangeLog}s : {@link Couchmove#executeMigration(List)} * */ public void migrate() { @@ -85,14 +85,14 @@ public void migrate() { try { // Acquire bucket lock if (!lockService.acquireLock()) { - logger.error("CouchMove did not acquire bucket '{}' lock. Exiting", bucketName); - throw new CouchMoveException("Unable to acquire lock"); + logger.error("Couchmove did not acquire bucket '{}' lock. Exiting", bucketName); + throw new CouchmoveException("Unable to acquire lock"); } // Fetching ChangeLogs from migration directory List changeLogs = fileService.fetch(); if (changeLogs.isEmpty()) { - logger.info("CouchMove did not find any migration scripts"); + logger.info("Couchmove did not find any migration scripts"); return; } @@ -102,12 +102,12 @@ public void migrate() { // Executing migration executeMigration(changeLogs); } catch (Exception e) { - throw new CouchMoveException("Unable to migrate", e); + throw new CouchmoveException("Unable to migrate", e); } finally { // Release lock lockService.releaseLock(); } - logger.info("CouchMove has finished his job"); + logger.info("Couchmove has finished his job"); } /** @@ -116,7 +116,7 @@ public void migrate() { *
  • If {@link ChangeLog#version} is lower than last executed one, ignore it and mark it as {@link Status#SKIPPED} *
  • If an {@link Status#EXECUTED} ChangeLog was modified, fail *
  • If an {@link Status#EXECUTED} ChangeLog description was modified, update it - *
  • Otherwise apply the ChangeLog : {@link CouchMove#executeMigration(ChangeLog, int)} + *
  • Otherwise apply the ChangeLog : {@link Couchmove#executeMigration(ChangeLog, int)} * * * @param changeLogs to execute @@ -167,7 +167,7 @@ void executeMigration(List changeLogs) { lastVersion = changeLog.getVersion(); migrationCount++; } else { - throw new CouchMoveException("Migration failed"); + throw new CouchmoveException("Migration failed"); } } if (migrationCount == 0) { diff --git a/src/main/java/com/github/couchmove/exception/CouchMoveException.java b/src/main/java/com/github/couchmove/exception/CouchMoveException.java deleted file mode 100644 index 3fd83f7..0000000 --- a/src/main/java/com/github/couchmove/exception/CouchMoveException.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.github.couchmove.exception; - -/** - * @author ctayeb - * Created on 28/05/2017 - */ -public class CouchMoveException extends RuntimeException { - public CouchMoveException(String message, Throwable cause) { - super(message, cause); - } - - public CouchMoveException(String message) { - super(message); - } -} diff --git a/src/main/java/com/github/couchmove/exception/CouchmoveException.java b/src/main/java/com/github/couchmove/exception/CouchmoveException.java new file mode 100644 index 0000000..4384545 --- /dev/null +++ b/src/main/java/com/github/couchmove/exception/CouchmoveException.java @@ -0,0 +1,15 @@ +package com.github.couchmove.exception; + +/** + * @author ctayeb + * Created on 28/05/2017 + */ +public class CouchmoveException extends RuntimeException { + public CouchmoveException(String message, Throwable cause) { + super(message, cause); + } + + public CouchmoveException(String message) { + super(message); + } +} diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index d8e7bc2..7afce53 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -17,7 +17,7 @@ import com.couchbase.client.java.util.retry.RetryBuilder; import com.couchbase.client.java.util.retry.RetryWhenFunction; import com.couchbase.client.java.view.DesignDocument; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.CouchbaseEntity; import lombok.AccessLevel; import lombok.Getter; @@ -63,7 +63,7 @@ public E save(String id, E entity) { entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { - throw new CouchMoveException("Unable to save document with id " + id, e); + throw new CouchmoveException("Unable to save document with id " + id, e); } } @@ -81,7 +81,7 @@ public E checkAndSave(String id, E entity) { entity.setCas(insertedDocument.cas()); return entity; } catch (JsonProcessingException e) { - throw new CouchMoveException("Unable to save document with id " + id, e); + throw new CouchmoveException("Unable to save document with id " + id, e); } } @@ -105,7 +105,7 @@ public E findOne(String id) { entity.setCas(document.cas()); return entity; } catch (IOException e) { - throw new CouchMoveException("Unable to read document with id " + id, e); + throw new CouchmoveException("Unable to read document with id " + id, e); } } @@ -128,14 +128,14 @@ public void query(String n1qlStatement) { N1qlParams.build().consistency(STATEMENT_PLUS)))); if (!result.parseSuccess()) { logger.info("Invalid N1QL request '{}'", n1qlStatement); - throw new CouchMoveException("Invalid n1ql request"); + throw new CouchmoveException("Invalid n1ql request"); } if (!single(result.finalSuccess())) { logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : ", n1qlStatement, single(result.status()), single(result.errors())); - throw new CouchMoveException("Unable to execute n1ql request"); + throw new CouchmoveException("Unable to execute n1ql request"); } } catch (Exception e) { - throw new CouchMoveException("Unable to execute n1ql request", e); + throw new CouchmoveException("Unable to execute n1ql request", e); } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index e20924d..e0704f0 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -3,7 +3,7 @@ import com.couchbase.client.java.Bucket; import com.couchbase.client.java.error.CASMismatchException; import com.couchbase.client.java.error.DocumentAlreadyExistsException; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLock; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; @@ -95,7 +95,7 @@ public void releaseLock() { if (isLockAcquired()) { forceReleaseLock(); } else { - throw new CouchMoveException("Unable to release lock acquired by an other process"); + throw new CouchmoveException("Unable to release lock acquired by an other process"); } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 28a4be6..4129de9 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -3,7 +3,7 @@ import com.couchbase.client.java.Bucket; import com.couchbase.client.java.query.N1qlQuery; import com.couchbase.client.java.view.DesignDocument; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; @@ -45,7 +45,7 @@ public ChangeLogDBService(Bucket bucket) { * * @param changeLogs to load from database * @return database version of changeLogs - * @throws CouchMoveException if checksum doesn't match + * @throws CouchmoveException if checksum doesn't match */ public List fetchAndCompare(List changeLogs) { logger.info("Fetching changeLogs from bucket '{}'", repository.getBucketName()); @@ -64,7 +64,7 @@ public List fetchAndCompare(List changeLogs) { dbChangeLog.setCas(null); } else if (!dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { logger.error("ChangeLog version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); - throw new CouchMoveException("ChangeLog checksum doesn't match"); + throw new CouchmoveException("ChangeLog checksum doesn't match"); } if (!dbChangeLog.getDescription().equals(changeLog.getDescription())) { logger.warn("ChangeLog version '{}' description updated", changeLog.getDescription()); diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index 7b1fd5e..6a1e25a 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -1,6 +1,6 @@ package com.github.couchmove.service; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Type; import com.github.couchmove.utils.FileUtils; @@ -99,13 +99,13 @@ static File initializeFolder(String changePath) { try { path = FileUtils.getPathFromResource(changePath); } catch (FileNotFoundException e) { - throw new CouchMoveException("The change path '" + changePath + "'doesn't exist"); + throw new CouchmoveException("The change path '" + changePath + "'doesn't exist"); } catch (IOException e) { - throw new CouchMoveException("Unable to get change path '" + changePath + "'", e); + throw new CouchmoveException("Unable to get change path '" + changePath + "'", e); } File file = path.toFile(); if (!file.isDirectory()) { - throw new CouchMoveException("The change path '" + changePath + "' is not a directory"); + throw new CouchmoveException("The change path '" + changePath + "' is not a directory"); } return file; } @@ -133,7 +133,7 @@ static Type getChangeLogType(File file) { return N1QL; } } - throw new CouchMoveException("Unknown ChangeLog type : " + file.getName()); + throw new CouchmoveException("Unknown ChangeLog type : " + file.getName()); } // diff --git a/src/main/java/com/github/couchmove/utils/FileUtils.java b/src/main/java/com/github/couchmove/utils/FileUtils.java index 37db12e..f67bb41 100644 --- a/src/main/java/com/github/couchmove/utils/FileUtils.java +++ b/src/main/java/com/github/couchmove/utils/FileUtils.java @@ -1,6 +1,6 @@ package com.github.couchmove.utils; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FilenameUtils; import org.jetbrains.annotations.NotNull; @@ -77,7 +77,7 @@ public static Path getPathFromResource(String resource) throws IOException { */ public static String calculateChecksum(@NotNull File file, String... extensions) { if (file == null || !file.exists()) { - throw new CouchMoveException("File is null or doesn't exists"); + throw new CouchmoveException("File is null or doesn't exists"); } if (file.isDirectory()) { //noinspection ConstantConditions @@ -96,7 +96,7 @@ public static String calculateChecksum(@NotNull File file, String... extensions) try { return DigestUtils.sha256Hex(toByteArray(file.toURI())); } catch (IOException e) { - throw new CouchMoveException("Unable to calculate file checksum '" + file.getName() + "'"); + throw new CouchmoveException("Unable to calculate file checksum '" + file.getName() + "'"); } } diff --git a/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java similarity index 92% rename from src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java rename to src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java index d44d427..61fdd74 100644 --- a/src/test/java/com/github/couchmove/CouchMoveIntegrationTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java @@ -3,7 +3,7 @@ import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.container.AbstractCouchbaseTest; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Status; import com.github.couchmove.pojo.Type; @@ -30,7 +30,7 @@ * @author ctayeb * Created on 05/06/2017 */ -public class CouchMoveIntegrationTest extends AbstractCouchbaseTest { +public class CouchmoveIntegrationTest extends AbstractCouchbaseTest { public static final String DB_MIGRATION = "db/migration/"; @@ -49,11 +49,11 @@ public static void init() { @Test public void should_migrate_successfully() { - // Given a CouchMove instance configured for success migration folder - CouchMove couchMove = new CouchMove(getBucket(), DB_MIGRATION + "success"); + // Given a Couchmove instance configured for success migration folder + Couchmove couchmove = new Couchmove(getBucket(), DB_MIGRATION + "success"); // When we launch migration - couchMove.migrate(); + couchmove.migrate(); // Then all changeLogs should be inserted in DB List changeLogs = Stream.of("1", "1.1", "2") @@ -110,7 +110,7 @@ public void should_skip_old_migration_version() { .build()); // When we execute migration in skip migration folder - new CouchMove(getBucket(), DB_MIGRATION + "skip").migrate(); + new Couchmove(getBucket(), DB_MIGRATION + "skip").migrate(); // Then the old ChangeLog is marked as skipped assertLike(changeLogRepository.findOne(PREFIX_ID + "1.2"), "1.2", null, "type", DESIGN_DOC, "V1.2__type.json", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", SKIPPED); @@ -118,11 +118,11 @@ public void should_skip_old_migration_version() { @Test public void should_migration_fail_on_exception() { - // Given a CouchMove instance configured for fail migration folder - CouchMove couchMove = new CouchMove(getBucket(), DB_MIGRATION + "fail"); + // Given a Couchmove instance configured for fail migration folder + Couchmove couchmove = new Couchmove(getBucket(), DB_MIGRATION + "fail"); // When we launch migration, then an exception should be raised - assertThrows(couchMove::migrate, CouchMoveException.class); + assertThrows(couchmove::migrate, CouchmoveException.class); // Then new ChangeLog is executed assertLike(changeLogRepository.findOne(PREFIX_ID + "1"), "1", 1, "insert users", N1QL, "V1__insert_users.n1ql", "efcc80f763e48e2a1d5b6689351ad1b4d678c70bebc0c0975a2d19f94e938f18", EXECUTED); @@ -150,7 +150,7 @@ public void should_update_changeLog() { .build()); // When we execute migration in update migration folder - new CouchMove(getBucket(), DB_MIGRATION + "update").migrate(); + new Couchmove(getBucket(), DB_MIGRATION + "update").migrate(); // Then executed changeLog description updated assertLike(changeLogRepository.findOne(PREFIX_ID + "1"), "1", 1, "create index", N1QL, "V1__create_index.n1ql", "69eb9007c910c2b9cac46044a54de5e933b768ae874c6408356372576ab88dbd", EXECUTED); diff --git a/src/test/java/com/github/couchmove/CouchMoveTest.java b/src/test/java/com/github/couchmove/CouchmoveTest.java similarity index 81% rename from src/test/java/com/github/couchmove/CouchMoveTest.java rename to src/test/java/com/github/couchmove/CouchmoveTest.java index 49b9b25..491952b 100644 --- a/src/test/java/com/github/couchmove/CouchMoveTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveTest.java @@ -1,7 +1,7 @@ package com.github.couchmove; import com.couchbase.client.java.Bucket; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; @@ -27,10 +27,10 @@ * Created on 04/06/2017 */ @RunWith(MockitoJUnitRunner.class) -public class CouchMoveTest { +public class CouchmoveTest { @InjectMocks - private CouchMove couchMove = new CouchMove(mockBucket()); + private Couchmove couchmove = new Couchmove(mockBucket()); @Mock private ChangeLockService lockServiceMock; @@ -41,10 +41,10 @@ public class CouchMoveTest { @Mock private ChangeLogFileService fileServiceMock; - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_migration_fail_if_lock_not_acquired() { when(lockServiceMock.acquireLock()).thenReturn(false); - couchMove.migrate(); + couchmove.migrate(); } @Test @@ -52,7 +52,7 @@ public void should_release_lock_after_migration() { when(lockServiceMock.acquireLock()).thenReturn(true); when(fileServiceMock.fetch()).thenReturn(newArrayList(getRandomChangeLog())); when(dbServiceMock.fetchAndCompare(any())).thenReturn(emptyList()); - couchMove.migrate(); + couchmove.migrate(); verify(lockServiceMock).releaseLock(); } @@ -63,7 +63,7 @@ public void should_migration_save_updated_changeLog() { .status(EXECUTED) .order(1) .build(); - couchMove.executeMigration(newArrayList(changeLog)); + couchmove.executeMigration(newArrayList(changeLog)); verify(dbServiceMock).save(changeLog); } @@ -75,7 +75,7 @@ public void should_migration_skip_unmodified_executed_changeLog() { .order(1) .build(); skippedChangeLog.setCas(RANDOM.nextLong()); - couchMove.executeMigration(newArrayList(skippedChangeLog)); + couchmove.executeMigration(newArrayList(skippedChangeLog)); verify(dbServiceMock, never()).save(any()); } @@ -85,7 +85,7 @@ public void should_migration_skip_skipped_changeLogs() { .version("1") .status(SKIPPED) .build(); - couchMove.executeMigration(newArrayList(skippedChangeLog)); + couchmove.executeMigration(newArrayList(skippedChangeLog)); verify(dbServiceMock, never()).save(any()); } @@ -99,15 +99,15 @@ public void should_migration_skip_changeLog_with_old_version() { .order(2) .status(EXECUTED) .build(); - couchMove.executeMigration(newArrayList(changeLogToSkip, executedChangeLog)); + couchmove.executeMigration(newArrayList(changeLogToSkip, executedChangeLog)); verify(dbServiceMock).save(changeLogToSkip); Assert.assertEquals(SKIPPED, changeLogToSkip.getStatus()); } @Test public void should_execute_migrations() { - CouchMove couchMove = spy(CouchMove.class); - couchMove.setDbService(dbServiceMock); + Couchmove couchmove = spy(Couchmove.class); + couchmove.setDbService(dbServiceMock); ChangeLog executedChangeLog = ChangeLog.builder() .version("1") .order(1) @@ -118,20 +118,20 @@ public void should_execute_migrations() { .version("2") .type(DOCUMENTS) .build(); - doReturn(true).when(couchMove).doExecute(changeLog); - couchMove.executeMigration(newArrayList(executedChangeLog, changeLog)); + doReturn(true).when(couchmove).doExecute(changeLog); + couchmove.executeMigration(newArrayList(executedChangeLog, changeLog)); Assert.assertEquals((Integer) 2, changeLog.getOrder()); } - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_throw_exception_if_migration_failed() { - CouchMove couchMove = spy(CouchMove.class); + Couchmove couchmove = spy(Couchmove.class); ChangeLog changeLog = ChangeLog.builder() .version("1") .type(N1QL) .build(); - doReturn(false).when(couchMove).executeMigration(changeLog, 1); - couchMove.executeMigration(newArrayList(changeLog)); + doReturn(false).when(couchmove).executeMigration(changeLog, 1); + couchmove.executeMigration(newArrayList(changeLog)); } @Test @@ -141,7 +141,7 @@ public void should_update_changeLog_on_migration_success() { .description("description") .type(DESIGN_DOC) .build(); - couchMove.executeMigration(changeLog, 1); + couchmove.executeMigration(changeLog, 1); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); @@ -155,8 +155,8 @@ public void should_update_changeLog_on_migration_failure() { .version("1") .type(DOCUMENTS) .build(); - doThrow(CouchMoveException.class).when(dbServiceMock).importDocuments(any()); - couchMove.executeMigration(changeLog, 1); + doThrow(CouchmoveException.class).when(dbServiceMock).importDocuments(any()); + couchmove.executeMigration(changeLog, 1); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); diff --git a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java index 6091213..5efa839 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java @@ -1,7 +1,7 @@ package com.github.couchmove.service; import com.github.couchmove.container.AbstractCouchbaseTest; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import org.junit.Test; import static com.github.couchmove.utils.TestUtils.assertThrows; @@ -60,7 +60,7 @@ public void should_not_release_lock_acquired_by_another_process() { ChangeLockService changeLockService2 = new ChangeLockService(getBucket()); // Then it should fails - assertThrows(changeLockService2::releaseLock, CouchMoveException.class); + assertThrows(changeLockService2::releaseLock, CouchmoveException.class); // When an other process force release the lock changeLockService2.forceReleaseLock(); diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index c127e2f..dd3f3b9 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -1,6 +1,6 @@ package com.github.couchmove.service; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; import org.assertj.core.api.Assertions; @@ -73,7 +73,7 @@ public void should_fetch_return_unchanged_changeLogs() { Assert.assertNotNull(changeLogs.get(1).getCas()); } - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_fetch_fail_when_checksum_does_not_match() { // Given a changeLog stored on DB ChangeLog dbChangeLog = getRandomChangeLog(); diff --git a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java index 9a86289..0947493 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java @@ -1,6 +1,6 @@ package com.github.couchmove.service; -import com.github.couchmove.exception.CouchMoveException; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Type; import org.apache.commons.io.FileUtils; @@ -20,7 +20,7 @@ */ public class ChangeLogFileServiceTest { - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_fail_if_path_does_not_exists() { String folderPath; //noinspection StatementWithEmptyBody @@ -28,7 +28,7 @@ public void should_fail_if_path_does_not_exists() { ChangeLogFileService.initializeFolder(folderPath); } - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_fail_if_path_is_not_directory() throws Exception { File tempFile = File.createTempFile(getRandomString(), ""); tempFile.deleteOnExit(); @@ -47,7 +47,7 @@ public void should_get_right_type_from_file() { Assert.assertEquals(Type.N1QL, ChangeLogFileService.getChangeLogType(new File("toto.N1QL"))); } - @Test(expected = CouchMoveException.class) + @Test(expected = CouchmoveException.class) public void should_throw_exception_when_unknown_file_type() { ChangeLogFileService.getChangeLogType(new File("toto")); } From 4eb2e77abc607c5869652b0d378ad75edfaa7cc7 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 18 Jun 2017 20:08:53 +0200 Subject: [PATCH 33/42] Update logs --- .../java/com/github/couchmove/Couchmove.java | 66 ++++++++++--------- .../repository/CouchbaseRepositoryImpl.java | 22 +++++-- .../couchmove/service/ChangeLockService.java | 12 ++-- .../couchmove/service/ChangeLogDBService.java | 12 ++-- .../service/ChangeLogFileService.java | 4 +- .../com/github/couchmove/utils/Utils.java | 46 ++++++++++++- .../com/github/couchmove/CouchmoveTest.java | 23 ++++--- .../repository/CouchbaseRepositoryTest.java | 20 ++++++ .../com/github/couchmove/utils/UtilsTest.java | 38 +++++++++++ 9 files changed, 182 insertions(+), 61 deletions(-) create mode 100644 src/test/java/com/github/couchmove/utils/UtilsTest.java diff --git a/src/main/java/com/github/couchmove/Couchmove.java b/src/main/java/com/github/couchmove/Couchmove.java index e775f3a..bbbd4b6 100644 --- a/src/main/java/com/github/couchmove/Couchmove.java +++ b/src/main/java/com/github/couchmove/Couchmove.java @@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit; import static com.github.couchmove.pojo.Status.*; +import static com.github.couchmove.utils.Utils.elapsed; +import static java.lang.String.format; /** * Couchmove Runner @@ -79,20 +81,22 @@ public Couchmove(Bucket bucket, String changePath) { *
  • Fetch corresponding {@link ChangeLog}s from {@link Bucket} *
  • Execute found {@link ChangeLog}s : {@link Couchmove#executeMigration(List)} * + * + * @throws CouchmoveException if migration fail */ - public void migrate() { - logger.info("Begin bucket '{}' migration", bucketName); + public void migrate() throws CouchmoveException { + logger.info("Begin bucket '{}' update", bucketName); try { // Acquire bucket lock if (!lockService.acquireLock()) { - logger.error("Couchmove did not acquire bucket '{}' lock. Exiting", bucketName); + logger.error("Couchmove did not acquire bucket '{}' change log lock. Exiting...", bucketName); throw new CouchmoveException("Unable to acquire lock"); } // Fetching ChangeLogs from migration directory List changeLogs = fileService.fetch(); if (changeLogs.isEmpty()) { - logger.info("Couchmove did not find any migration scripts"); + logger.info("Couchmove did not find any change logs"); return; } @@ -102,12 +106,13 @@ public void migrate() { // Executing migration executeMigration(changeLogs); } catch (Exception e) { + logger.error("Couchmove Update failed"); throw new CouchmoveException("Unable to migrate", e); } finally { // Release lock lockService.releaseLock(); } - logger.info("Couchmove has finished his job"); + logger.info("Couchmove Update Successful"); } /** @@ -122,7 +127,7 @@ public void migrate() { * @param changeLogs to execute */ void executeMigration(List changeLogs) { - logger.info("Executing migration scripts..."); + logger.info("Applying change logs..."); int migrationCount = 0; // Get version and order of last executed changeLog String lastVersion = ""; @@ -145,7 +150,7 @@ void executeMigration(List changeLogs) { for (ChangeLog changeLog : changeLogs) { if (changeLog.getStatus() == EXECUTED) { if (changeLog.getCas() == null) { - logger.info("Updating changeLog '{}'", changeLog.getVersion()); + logger.info("Updating change log '{}::{}'", changeLog.getVersion(), changeLog.getDescription()); dbService.save(changeLog); } continue; @@ -156,24 +161,21 @@ void executeMigration(List changeLogs) { } if (lastVersion.compareTo(changeLog.getVersion()) >= 0) { - logger.warn("ChangeLog '{}' version is lower than last executed one '{}'. Skipping", changeLog.getVersion(), lastVersion); + logger.warn("ChangeLog '{}::{}' version is lower than last executed one '{}'. Skipping", changeLog.getVersion(), changeLog.getDescription(), lastVersion); changeLog.setStatus(SKIPPED); dbService.save(changeLog); continue; } - if (executeMigration(changeLog, lastOrder + 1)) { - lastOrder++; - lastVersion = changeLog.getVersion(); - migrationCount++; - } else { - throw new CouchmoveException("Migration failed"); - } + executeMigration(changeLog, lastOrder + 1); + lastOrder++; + lastVersion = changeLog.getVersion(); + migrationCount++; } if (migrationCount == 0) { - logger.info("No new migration scripts found"); + logger.info("No new change logs found"); } else { - logger.info("Executed {} migration scripts", migrationCount); + logger.info("Applied {} change logs", migrationCount); } } @@ -186,24 +188,25 @@ void executeMigration(List changeLogs) { * * @param changeLog {@link ChangeLog} to execute * @param order the order to set if the execution was successful - * @return true if the execution was successful, false otherwise + * @throws CouchmoveException if the execution fail */ - boolean executeMigration(ChangeLog changeLog, int order) { - logger.info("Executing ChangeLog '{}'", changeLog.getVersion()); + void executeMigration(ChangeLog changeLog, int order) { + logger.info("Applying change log '{}::{}'", changeLog.getVersion(), changeLog.getDescription()); Stopwatch sw = Stopwatch.createStarted(); changeLog.setTimestamp(new Date()); changeLog.setRunner(Utils.getUsername()); - if (doExecute(changeLog)) { - logger.info("ChangeLog '{}' successfully executed", changeLog.getVersion()); + try { + doExecute(changeLog); + logger.info("Change log '{}::{}' ran successfully in {}", changeLog.getVersion(), changeLog.getDescription(), elapsed(sw)); changeLog.setOrder(order); changeLog.setStatus(EXECUTED); - } else { - logger.error("Unable to execute ChangeLog '{}'", changeLog.getVersion()); + } catch (CouchmoveException e) { changeLog.setStatus(FAILED); + throw new CouchmoveException(format("Unable to apply change log '%s::%s'", changeLog.getVersion(), changeLog.getDescription()), e); + } finally { + changeLog.setDuration(sw.elapsed(TimeUnit.MILLISECONDS)); + dbService.save(changeLog); } - changeLog.setDuration(sw.elapsed(TimeUnit.MILLISECONDS)); - dbService.save(changeLog); - return changeLog.getStatus() == EXECUTED; } /** @@ -213,10 +216,11 @@ boolean executeMigration(ChangeLog changeLog, int order) { *
  • {@link Type#N1QL} : Execute all {@link N1qlQuery} contained in the {@value Constants#N1QL} file *
  • {@link Type#DESIGN_DOC} : Imports {@link DesignDocument} contained in the {@value Constants#JSON} document * + * * @param changeLog {@link ChangeLog} to apply - * @return true if the execution was successful, false otherwise + * @throws CouchmoveException if the execution fail */ - boolean doExecute(ChangeLog changeLog) { + void doExecute(ChangeLog changeLog) { try { switch (changeLog.getType()) { case DOCUMENTS: @@ -231,10 +235,8 @@ boolean doExecute(ChangeLog changeLog) { default: throw new IllegalArgumentException("Unknown ChangeLog Type '" + changeLog.getType() + "'"); } - return true; } catch (Exception e) { - logger.error("Unable to import " + changeLog.getType().name().toLowerCase().replace("_", " ") + " : '" + changeLog.getScript() + "'", e); - return false; + throw new CouchmoveException("Unable to import " + changeLog.getType().name().toLowerCase().replace("_", " ") + " : '" + changeLog.getScript() + "'", e); } } } diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index 7afce53..bb38d62 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -32,6 +32,7 @@ import java.util.function.Function; import static com.couchbase.client.java.query.consistency.ScanConsistency.STATEMENT_PLUS; +import static lombok.AccessLevel.PACKAGE; import static org.slf4j.LoggerFactory.getLogger; /** @@ -39,7 +40,8 @@ * Created on 27/05/2017 */ // For tests -@NoArgsConstructor(access = AccessLevel.PACKAGE, force = true) +@SuppressWarnings("ConstantConditions") +@NoArgsConstructor(access = PACKAGE, force = true) @RequiredArgsConstructor public class CouchbaseRepositoryImpl implements CouchbaseRepository { @@ -57,6 +59,7 @@ public class CouchbaseRepositoryImpl implements Couch @Override public E save(String id, E entity) { + logger.trace("Save entity '{}' with id '{}'", entity, id); try { String json = getJsonMapper().writeValueAsString(entity); RawJsonDocument insertedDocument = runAsync(bucket -> bucket.upsert(RawJsonDocument.create(id, json))); @@ -69,6 +72,7 @@ public E save(String id, E entity) { @Override public E checkAndSave(String id, E entity) { + logger.trace("Check and save entity '{}' with id '{}'", entity, id); try { String content = getJsonMapper().writeValueAsString(entity); RawJsonDocument insertedDocument; @@ -87,15 +91,17 @@ public E checkAndSave(String id, E entity) { @Override public void delete(String id) { + logger.trace("Remove entity with id '{}'", id); try { runAsync(bucket -> bucket.remove(id)); } catch (DocumentDoesNotExistException e) { - logger.warn("Trying to delete document that does not exist : '{}'", id); + logger.debug("Trying to delete document that does not exist : '{}'", id); } } @Override public E findOne(String id) { + logger.trace("Find entity with id '{}'", id); RawJsonDocument document = runAsync(bucket -> bucket.get(id, RawJsonDocument.class)); if (document == null) { return null; @@ -111,27 +117,29 @@ public E findOne(String id) { @Override public void save(String id, String jsonContent) { + logger.trace("Save document with id '{}' : \n'{}'", id, jsonContent); runAsync(bucket -> bucket.upsert(RawJsonDocument.create(id, jsonContent))); } @Override public void importDesignDoc(String name, String jsonContent) { + logger.trace("Import document : \n'{}'", jsonContent); bucket.bucketManager().upsertDesignDocument(DesignDocument.from(name, JsonObject.fromJson(jsonContent))); - } +} @Override public void query(String n1qlStatement) { - logger.debug("Executing n1ql request : {}", n1qlStatement); + logger.debug("Execute n1ql request : \n{}", n1qlStatement); try { AsyncN1qlQueryResult result = runAsync(bucket -> bucket .query(N1qlQuery.simple(n1qlStatement, N1qlParams.build().consistency(STATEMENT_PLUS)))); if (!result.parseSuccess()) { - logger.info("Invalid N1QL request '{}'", n1qlStatement); + logger.error("Invalid N1QL request '{}' : {}", n1qlStatement, single(result.errors())); throw new CouchmoveException("Invalid n1ql request"); } if (!single(result.finalSuccess())) { - logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : ", n1qlStatement, single(result.status()), single(result.errors())); + logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : {}", n1qlStatement, single(result.status()), single(result.errors())); throw new CouchmoveException("Unable to execute n1ql request"); } } catch (Exception e) { @@ -144,6 +152,7 @@ public String getBucketName() { return bucket.name(); } + // @Nullable private T single(Observable observable) { return observable.toBlocking().singleOrDefault(null); @@ -162,4 +171,5 @@ private static RetryWhenFunction retryStrategy() { .max(3) .build(); } + // } diff --git a/src/main/java/com/github/couchmove/service/ChangeLockService.java b/src/main/java/com/github/couchmove/service/ChangeLockService.java index e0704f0..0b4b8d5 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLockService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLockService.java @@ -40,14 +40,14 @@ public ChangeLockService(Bucket bucket) { * @return true if lock successfully acquired, false otherwise */ public boolean acquireLock() { - logger.info("Trying to acquire bucket '{}' lock...", repository.getBucketName()); + logger.info("Trying to acquire bucket '{}' change log lock...", repository.getBucketName()); // Verify if there is any lock on database ChangeLock lock = repository.findOne(LOCK_ID); // If none, create one if (lock == null) { lock = new ChangeLock(); } else if (lock.isLocked()) { - logger.warn("The database is already locked by '{}'", lock.getRunner()); + logger.warn("The bucket is already locked by '{}'", lock.getRunner()); return false; } // Create Lock information @@ -61,10 +61,10 @@ public boolean acquireLock() { } catch (CASMismatchException | DocumentAlreadyExistsException e) { // In case of exception, this means an other process got the lock, logging its information lock = repository.findOne(LOCK_ID); - logger.warn("The bucket '{}' is already locked by '{}'", repository.getBucketName(), lock.getRunner()); + logger.warn("The bucket is already locked by '{}'", lock.getRunner()); return false; } - logger.info("Lock acquired"); + logger.info("Successfully acquired change log lock"); return true; } @@ -82,7 +82,7 @@ public boolean isLockAcquired() { return false; } if (lock.getUuid() == null || !lock.getUuid().equals(uuid)) { - logger.warn("Lock is acquired by another process"); + logger.warn("Change log lock is acquired by another process"); return false; } return true; @@ -103,7 +103,7 @@ public void releaseLock() { * Force release pessimistic lock even if the current instance doesn't hold it */ public void forceReleaseLock() { - logger.info("Release lock"); repository.delete(LOCK_ID); + logger.info("Successfully released change log lock"); } } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 4129de9..627bf06 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -48,26 +48,26 @@ public ChangeLogDBService(Bucket bucket) { * @throws CouchmoveException if checksum doesn't match */ public List fetchAndCompare(List changeLogs) { - logger.info("Fetching changeLogs from bucket '{}'", repository.getBucketName()); + logger.info("Reading from bucket '{}'", repository.getBucketName()); List result = new ArrayList<>(changeLogs.size()); for (ChangeLog changeLog : changeLogs) { String version = changeLog.getVersion(); ChangeLog dbChangeLog = repository.findOne(PREFIX_ID + version); if (dbChangeLog == null) { - logger.debug("ChangeLog version '{}' not found", version); + logger.debug("Change log version '{}' not found", version); result.add(changeLog); continue; } if (dbChangeLog.getChecksum() == null) { - logger.warn("ChangeLog version '{}' checksum reset"); + logger.warn("Change log version '{}' checksum reset"); dbChangeLog.setChecksum(changeLog.getChecksum()); dbChangeLog.setCas(null); } else if (!dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { - logger.error("ChangeLog version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); + logger.error("Change log version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); throw new CouchmoveException("ChangeLog checksum doesn't match"); } if (!dbChangeLog.getDescription().equals(changeLog.getDescription())) { - logger.warn("ChangeLog version '{}' description updated", changeLog.getDescription()); + logger.warn("Change log version '{}' description updated", changeLog.getDescription()); logger.debug("{} was {}", dbChangeLog, changeLog); dbChangeLog.setDescription(changeLog.getDescription()); dbChangeLog.setScript(changeLog.getScript()); @@ -75,7 +75,7 @@ public List fetchAndCompare(List changeLogs) { } result.add(dbChangeLog); } - logger.info("Fetched {} changeLogs from bucket", result.size()); + logger.info("Fetched {} Change logs from bucket", result.size()); return Collections.unmodifiableList(result); } diff --git a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java index 6a1e25a..ae613f7 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogFileService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogFileService.java @@ -49,7 +49,7 @@ public ChangeLogFileService(String changePath) { * @return An ordered list of {@link ChangeLog}s by {@link ChangeLog#version} */ public List fetch() { - logger.info("Fetching changeLogs from migration folder '{}'", changeFolder.getPath()); + logger.info("Reading from migration folder '{}'", changeFolder.getPath()); SortedSet sortedChangeLogs = new TreeSet<>(); //noinspection ConstantConditions for (File file : changeFolder.listFiles()) { @@ -67,7 +67,7 @@ public List fetch() { sortedChangeLogs.add(changeLog); } } - logger.info("Fetched {} changeLogs from migration folder", sortedChangeLogs.size()); + logger.info("Fetched {} change logs from migration folder", sortedChangeLogs.size()); return Collections.unmodifiableList(new ArrayList<>(sortedChangeLogs)); } diff --git a/src/main/java/com/github/couchmove/utils/Utils.java b/src/main/java/com/github/couchmove/utils/Utils.java index 318367c..657224a 100644 --- a/src/main/java/com/github/couchmove/utils/Utils.java +++ b/src/main/java/com/github/couchmove/utils/Utils.java @@ -1,15 +1,20 @@ package com.github.couchmove.utils; +import com.google.common.base.Stopwatch; import lombok.Getter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; + +import static java.util.concurrent.TimeUnit.*; /** * @author ctayeb - * Created on 04/06/2017 + * Created on 04/06/2017 */ public class Utils { @@ -47,4 +52,43 @@ private static String initializeUserName() { } return "unknown"; } + + /** + * Format duration to human readable + *

    + * Exemple : 188,100,312 {@link TimeUnit#MILLISECONDS} => 2d 4h 15m 15s 312ms + * + * @param duration duration to format + * @param timeUnit source timeUnit + * @return human readable duration format + */ + public static String prettyFormatDuration(long duration, TimeUnit timeUnit) { + StringBuffer sb = new StringBuffer(); + duration = appendUnit(sb, duration, timeUnit, DAYS/* */, "d", timeUnit::toDays); + duration = appendUnit(sb, duration, timeUnit, HOURS/* */, "h", timeUnit::toHours); + duration = appendUnit(sb, duration, timeUnit, MINUTES/* */, "min", timeUnit::toMinutes); + duration = appendUnit(sb, duration, timeUnit, SECONDS/* */, "s", timeUnit::toSeconds); + duration = appendUnit(sb, duration, timeUnit, MILLISECONDS, "ms", timeUnit::toMillis); + duration = appendUnit(sb, duration, timeUnit, MICROSECONDS, "μs", timeUnit::toMicros); + /* */ + appendUnit(sb, duration, timeUnit, NANOSECONDS, "ns", timeUnit::toNanos); + return sb.toString(); + } + + private static long appendUnit(StringBuffer sb, long duration, TimeUnit source, TimeUnit destination, String unit, Function converter) { + long value = converter.apply(duration); + if (value != 0) { + sb.append(value).append(unit); + long remaining = duration - source.convert(value, destination); + if (remaining != 0) { + sb.append(" "); + } + return remaining; + } + return duration; + } + + public static String elapsed(Stopwatch sw) { + return prettyFormatDuration(sw.elapsed(TimeUnit.MILLISECONDS), MILLISECONDS); + } } diff --git a/src/test/java/com/github/couchmove/CouchmoveTest.java b/src/test/java/com/github/couchmove/CouchmoveTest.java index 491952b..16931d2 100644 --- a/src/test/java/com/github/couchmove/CouchmoveTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveTest.java @@ -6,6 +6,7 @@ import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; +import com.github.couchmove.utils.TestUtils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -16,6 +17,7 @@ import static com.github.couchmove.pojo.Status.*; import static com.github.couchmove.pojo.Type.*; import static com.github.couchmove.utils.TestUtils.RANDOM; +import static com.github.couchmove.utils.TestUtils.assertThrows; import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.emptyList; @@ -41,10 +43,10 @@ public class CouchmoveTest { @Mock private ChangeLogFileService fileServiceMock; - @Test(expected = CouchmoveException.class) + @Test public void should_migration_fail_if_lock_not_acquired() { when(lockServiceMock.acquireLock()).thenReturn(false); - couchmove.migrate(); + assertThrows(() -> couchmove.migrate(), CouchmoveException.class); } @Test @@ -60,6 +62,7 @@ public void should_release_lock_after_migration() { public void should_migration_save_updated_changeLog() { ChangeLog changeLog = ChangeLog.builder() .version("1") + .description("update") .status(EXECUTED) .order(1) .build(); @@ -93,9 +96,11 @@ public void should_migration_skip_skipped_changeLogs() { public void should_migration_skip_changeLog_with_old_version() { ChangeLog changeLogToSkip = ChangeLog.builder() .version("1") + .description("old version") .build(); ChangeLog executedChangeLog = ChangeLog.builder() .version("2") + .description("new version") .order(2) .status(EXECUTED) .build(); @@ -116,29 +121,30 @@ public void should_execute_migrations() { executedChangeLog.setCas(RANDOM.nextLong()); ChangeLog changeLog = ChangeLog.builder() .version("2") + .description("valid") .type(DOCUMENTS) .build(); - doReturn(true).when(couchmove).doExecute(changeLog); + doNothing().when(couchmove).doExecute(changeLog); couchmove.executeMigration(newArrayList(executedChangeLog, changeLog)); Assert.assertEquals((Integer) 2, changeLog.getOrder()); } - @Test(expected = CouchmoveException.class) + @Test public void should_throw_exception_if_migration_failed() { Couchmove couchmove = spy(Couchmove.class); ChangeLog changeLog = ChangeLog.builder() .version("1") .type(N1QL) .build(); - doReturn(false).when(couchmove).executeMigration(changeLog, 1); - couchmove.executeMigration(newArrayList(changeLog)); + doThrow(CouchmoveException.class).when(couchmove).executeMigration(changeLog, 1); + assertThrows(() -> couchmove.executeMigration(newArrayList(changeLog)), CouchmoveException.class); } @Test public void should_update_changeLog_on_migration_success() { ChangeLog changeLog = ChangeLog.builder() .version("1") - .description("description") + .description("valid change log") .type(DESIGN_DOC) .build(); couchmove.executeMigration(changeLog, 1); @@ -153,10 +159,11 @@ public void should_update_changeLog_on_migration_success() { public void should_update_changeLog_on_migration_failure() { ChangeLog changeLog = ChangeLog.builder() .version("1") + .description("invalid") .type(DOCUMENTS) .build(); doThrow(CouchmoveException.class).when(dbServiceMock).importDocuments(any()); - couchmove.executeMigration(changeLog, 1); + assertThrows(() -> couchmove.executeMigration(changeLog, 1), CouchmoveException.class); verify(dbServiceMock).save(changeLog); Assert.assertNotNull(changeLog.getTimestamp()); Assert.assertNotNull(changeLog.getDuration()); diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 582155b..d4af691 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -5,6 +5,7 @@ import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.container.AbstractCouchbaseTest; +import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Type; import com.github.couchmove.utils.TestUtils; @@ -16,6 +17,7 @@ import java.util.Random; import java.util.stream.Collectors; +import static com.github.couchmove.utils.TestUtils.assertThrows; import static com.github.couchmove.utils.TestUtils.getRandomString; import static java.lang.String.format; @@ -143,6 +145,24 @@ public void should_execute_n1ql() { Assert.assertEquals(format("`%s`", INDEX_NAME), indexInfo.indexKey().get(0)); } + @Test + public void should_execute_n1ql_parse_fail() { + // Given an invalid request + String request = format("CREATE INDEX `%s`", INDEX_NAME); + + // When we execute the query + assertThrows(() -> repository.query(request), CouchmoveException.class); + } + + @Test + public void should_execute_n1ql_fail() { + // Given an index on invalid bucket + String request = format("CREATE INDEX `%s` on toto(%s)", INDEX_NAME, INDEX_NAME); + + // When we execute the query + assertThrows(() -> repository.query(request), CouchmoveException.class); + } + @Test public void should_save_json_document() { // Given a json document diff --git a/src/test/java/com/github/couchmove/utils/UtilsTest.java b/src/test/java/com/github/couchmove/utils/UtilsTest.java new file mode 100644 index 0000000..686d559 --- /dev/null +++ b/src/test/java/com/github/couchmove/utils/UtilsTest.java @@ -0,0 +1,38 @@ +package com.github.couchmove.utils; + +import com.tngtech.java.junit.dataprovider.DataProvider; +import com.tngtech.java.junit.dataprovider.DataProviderRunner; +import com.tngtech.java.junit.dataprovider.UseDataProvider; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.*; + +/** + * @author ctayeb + * created on 18/06/2017 + */ +@RunWith(DataProviderRunner.class) +public class UtilsTest { + + @DataProvider + public static Object[][] durationProvider() { + return new Object[][]{ + {(((3 * 24) + 5) * 60) + 15, MINUTES, "3d 5h 15min"}, + {((5 * 60) * 60 + 12), SECONDS, "5h 12s"}, + {(((((2 * 24) + 4) * 60) + 15) * 60) * 1000 + 312, MILLISECONDS, "2d 4h 15min 312ms"}, + {25_377_004_023L, MICROSECONDS, "7h 2min 57s 4ms 23μs"}, + {7_380_011_024_014L, NANOSECONDS, "2h 3min 11ms 24μs 14ns"} + }; + } + + @Test + @UseDataProvider("durationProvider") + public void should_pretty_format_duration(long duration, TimeUnit source, String expectedFormat) { + Assert.assertEquals(expectedFormat, Utils.prettyFormatDuration(duration, source)); + } + +} \ No newline at end of file From 10969bfed9c4de4499cc86ae01cbaf61ae1655d5 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Fri, 7 Jul 2017 22:05:25 +0200 Subject: [PATCH 34/42] Externalize Couchbase testcontainer as library --- pom.xml | 6 +- .../couchmove/CouchmoveIntegrationTest.java | 2 +- .../container/AbstractCouchbaseTest.java | 71 ----- .../container/CouchbaseContainer.java | 284 ------------------ .../CouchbaseQueryServiceWaitStrategy.java | 53 ---- .../container/CouchbaseWaitStrategy.java | 159 ---------- .../repository/CouchbaseRepositoryTest.java | 2 +- .../service/ChangeLockServiceTest.java | 2 +- .../service/ChangeLogDBServiceTest.java | 2 +- src/test/resources/logback-test.xml | 29 ++ 10 files changed, 36 insertions(+), 574 deletions(-) delete mode 100644 src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java delete mode 100644 src/test/java/com/github/couchmove/container/CouchbaseContainer.java delete mode 100644 src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java delete mode 100644 src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java create mode 100644 src/test/resources/logback-test.xml diff --git a/pom.xml b/pom.xml index 28540b3..78c7c6a 100644 --- a/pom.xml +++ b/pom.xml @@ -88,9 +88,9 @@ test - org.testcontainers - testcontainers - 1.2.1 + com.github.differentway + couchbase-testcontainer + 1.0 test diff --git a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java index 61fdd74..6978677 100644 --- a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java @@ -2,7 +2,7 @@ import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; -import com.github.couchmove.container.AbstractCouchbaseTest; +import org.testcontainers.couchbase.AbstractCouchbaseTest; import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Status; diff --git a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java b/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java deleted file mode 100644 index ece88b8..0000000 --- a/src/test/java/com/github/couchmove/container/AbstractCouchbaseTest.java +++ /dev/null @@ -1,71 +0,0 @@ -package com.github.couchmove.container; - -import ch.qos.logback.classic.Level; -import ch.qos.logback.classic.Logger; -import com.couchbase.client.java.Bucket; -import com.couchbase.client.java.CouchbaseCluster; -import com.couchbase.client.java.bucket.BucketType; -import com.couchbase.client.java.cluster.DefaultBucketSettings; -import com.couchbase.client.java.query.N1qlParams; -import com.couchbase.client.java.query.N1qlQuery; -import com.couchbase.client.java.query.consistency.ScanConsistency; -import lombok.Getter; -import org.junit.After; -import org.slf4j.LoggerFactory; - -/** - * @author ctayeb - */ -public abstract class AbstractCouchbaseTest { - public static final String CLUSTER_USER = "Administrator"; - public static final String CLUSTER_PASSWORD = "password"; - public static final String DEFAULT_BUCKET = "default"; - - static { - ((Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME)).setLevel(Level.INFO); - } - - @Getter(lazy = true) - private final static CouchbaseContainer couchbaseContainer = initCouchbaseContainer(); - - @Getter(lazy = true) - private final static Bucket bucket = openBucket(DEFAULT_BUCKET); - - @After - public void clear() { - if (getCouchbaseContainer().isIndex() && getCouchbaseContainer().isQuery() && getCouchbaseContainer().isPrimaryIndex()) { - getBucket().query( - N1qlQuery.simple(String.format("DELETE FROM `%s`", getBucket().name()), - N1qlParams.build().consistency(ScanConsistency.STATEMENT_PLUS))); - } else { - getBucket().bucketManager().flush(); - } - } - - private static CouchbaseContainer initCouchbaseContainer() { - CouchbaseContainer couchbaseContainer = new CouchbaseContainer() - .withFTS(false) - .withIndex(true) - .withQuery(true) - .withPrimaryIndex(true) - .withClusterUsername(CLUSTER_USER) - .withClusterPassword(CLUSTER_PASSWORD); - couchbaseContainer.start(); - couchbaseContainer.createBucket(DefaultBucketSettings.builder() - .enableFlush(true) - .name(DEFAULT_BUCKET) - .quota(100) - .replicas(0) - .port(couchbaseContainer.getMappedPort(CouchbaseContainer.BUCKET_PORT)) - .type(BucketType.COUCHBASE) - .build(), couchbaseContainer.isPrimaryIndex()); - return couchbaseContainer; - } - - private static Bucket openBucket(String bucketName) { - CouchbaseCluster cluster = getCouchbaseContainer().getCouchbaseCluster(); - Bucket bucket = cluster.openBucket(bucketName); - Runtime.getRuntime().addShutdownHook(new Thread(bucket::close)); - return bucket; - } -} diff --git a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java b/src/test/java/com/github/couchmove/container/CouchbaseContainer.java deleted file mode 100644 index 567bd9d..0000000 --- a/src/test/java/com/github/couchmove/container/CouchbaseContainer.java +++ /dev/null @@ -1,284 +0,0 @@ -/* - * Copyright (c) 2016 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.github.couchmove.container; - -import com.couchbase.client.core.utils.Base64; -import com.couchbase.client.java.Bucket; -import com.couchbase.client.java.CouchbaseCluster; -import com.couchbase.client.java.cluster.BucketSettings; -import com.couchbase.client.java.env.CouchbaseEnvironment; -import com.couchbase.client.java.env.DefaultCouchbaseEnvironment; -import com.couchbase.client.java.query.Index; -import lombok.Getter; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.containers.wait.HttpWaitStrategy; - -import java.io.DataOutputStream; -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URL; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; - -/** - * @author Laurent Doguin - */ -public class CouchbaseContainer> extends GenericContainer { - - public static final int BUCKET_PORT = 11211; - private String memoryQuota = "400"; - - private String indexMemoryQuota = "400"; - - private String clusterUsername = "Administrator"; - - private String clusterPassword = "password"; - - private boolean keyValue = true; - - @Getter - private boolean query = true; - - @Getter - private boolean index = true; - - @Getter - private boolean primaryIndex = true; - - private boolean fts = true; - - private boolean beerSample = false; - - private boolean travelSample = false; - - private boolean gamesIMSample = false; - - private CouchbaseEnvironment couchbaseEnvironment; - - private CouchbaseCluster couchbaseCluster; - - private List newBuckets = new ArrayList<>(); - - private String urlBase; - - public CouchbaseContainer() { - super("couchbase/server:latest"); - } - - public CouchbaseContainer(String containerName) { - super(containerName); - } - - @Override - protected Integer getLivenessCheckPort() { - return getMappedPort(8091); - } - - @Override - protected void configure() { - // Configurable ports - addExposedPorts(11210, 11207, 8091, 18091, BUCKET_PORT); - - // Non configurable ports - addFixedExposedPort(8092, 8092); - addFixedExposedPort(8093, 8093); - addFixedExposedPort(8094, 8094); - addFixedExposedPort(8095, 8095); - addFixedExposedPort(18092, 18092); - addFixedExposedPort(18093, 18093); - setWaitStrategy(new HttpWaitStrategy().forPath("/ui/index.html#/")); - } - - public CouchbaseEnvironment getCouchbaseEnvironnement() { - if (couchbaseEnvironment == null) { - initCluster(); - couchbaseEnvironment = DefaultCouchbaseEnvironment.builder() - .bootstrapCarrierDirectPort(getMappedPort(11210)) - .bootstrapCarrierSslPort(getMappedPort(11207)) - .bootstrapHttpDirectPort(getMappedPort(8091)) - .bootstrapHttpSslPort(getMappedPort(18091)) - .build(); - } - return couchbaseEnvironment; - } - - public CouchbaseCluster getCouchbaseCluster() { - if (couchbaseCluster == null) { - couchbaseCluster = CouchbaseCluster.create(getCouchbaseEnvironnement(), getContainerIpAddress()); - } - return couchbaseCluster; - } - - public SELF withClusterUsername(String username) { - this.clusterUsername = username; - return self(); - } - - public SELF withClusterPassword(String password) { - this.clusterPassword = password; - return self(); - } - - public SELF withMemoryQuota(String memoryQuota) { - this.memoryQuota = memoryQuota; - return self(); - } - - public SELF withIndexMemoryQuota(String indexMemoryQuota) { - this.indexMemoryQuota = indexMemoryQuota; - return self(); - } - - public SELF withKeyValue(Boolean withKV) { - this.keyValue = withKV; - return self(); - } - - public SELF withIndex(Boolean withIndex) { - this.index = withIndex; - return self(); - } - - public SELF withPrimaryIndex(Boolean primaryIndex) { - this.primaryIndex = primaryIndex; - return self(); - } - - public SELF withQuery(Boolean withQuery) { - this.query = withQuery; - return self(); - } - - public SELF withFTS(Boolean withFTS) { - this.fts = withFTS; - return self(); - } - - public SELF withTravelSample(Boolean withTravelSample) { - this.travelSample = withTravelSample; - return self(); - } - - public SELF withBeerSample(Boolean withBeerSample) { - this.beerSample = withBeerSample; - return self(); - } - - public SELF withGamesIMSample(Boolean withGamesIMSample) { - this.gamesIMSample = withGamesIMSample; - return self(); - } - - public SELF withNewBucket(BucketSettings bucketSettings) { - newBuckets.add(bucketSettings); - return self(); - } - - - public void initCluster() { - urlBase = String.format("http://%s:%s", getContainerIpAddress(), getMappedPort(8091)); - try { - String poolURL = "/pools/default"; - String poolPayload = "memoryQuota=" + URLEncoder.encode(memoryQuota, "UTF-8") + "&indexMemoryQuota=" + URLEncoder.encode(indexMemoryQuota, "UTF-8"); - - String setupServicesURL = "/node/controller/setupServices"; - StringBuilder servicePayloadBuilder = new StringBuilder(); - if (keyValue) { - servicePayloadBuilder.append("kv,"); - } - if (query) { - servicePayloadBuilder.append("n1ql,"); - } - if (index) { - servicePayloadBuilder.append("index,"); - } - if (fts) { - servicePayloadBuilder.append("fts,"); - } - String setupServiceContent = "services=" + URLEncoder.encode(servicePayloadBuilder.toString(), "UTF-8"); - - String webSettingsURL = "/settings/web"; - String webSettingsContent = "username=" + URLEncoder.encode(clusterUsername, "UTF-8") + "&password=" + URLEncoder.encode(clusterPassword, "UTF-8") + "&port=8091"; - - String bucketURL = "/sampleBuckets/install"; - - StringBuilder sampleBucketPayloadBuilder = new StringBuilder(); - sampleBucketPayloadBuilder.append('['); - if (travelSample) { - sampleBucketPayloadBuilder.append("\"travel-sample\","); - } - if (beerSample) { - sampleBucketPayloadBuilder.append("\"beer-sample\","); - } - if (gamesIMSample) { - sampleBucketPayloadBuilder.append("\"gamesim-sample\","); - } - sampleBucketPayloadBuilder.append(']'); - - callCouchbaseRestAPI(poolURL, poolPayload); - callCouchbaseRestAPI(setupServicesURL, setupServiceContent); - callCouchbaseRestAPI(webSettingsURL, webSettingsContent); - callCouchbaseRestAPI(bucketURL, sampleBucketPayloadBuilder.toString()); - - CouchbaseWaitStrategy s = new CouchbaseWaitStrategy(); - s.withBasicCredentials(clusterUsername, clusterPassword); - s.waitUntilReady(this); - callCouchbaseRestAPI("/settings/indexes", "indexerThreads=0&logLevel=info&maxRollbackPoints=5&storageMode=memory_optimized"); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - public void createBucket(BucketSettings bucketSetting, Boolean primaryIndex) { - BucketSettings bucketSettings = getCouchbaseCluster().clusterManager(clusterUsername, clusterPassword).insertBucket(bucketSetting); - if (index) { - Bucket bucket = getCouchbaseCluster().openBucket(bucketSettings.name()); - new CouchbaseQueryServiceWaitStrategy(bucket).waitUntilReady(this); - if (primaryIndex) { - bucket.query(Index.createPrimaryIndex().on(bucketSetting.name())); - } - } - } - - public void callCouchbaseRestAPI(String url, String payload) throws IOException { - String fullUrl = urlBase + url; - HttpURLConnection httpConnection = (HttpURLConnection) ((new URL(fullUrl).openConnection())); - httpConnection.setDoOutput(true); - httpConnection.setRequestMethod("POST"); - httpConnection.setRequestProperty("Content-Type", - "application/x-www-form-urlencoded"); - String encoded = Base64.encode((clusterUsername + ":" + clusterPassword).getBytes("UTF-8")); - httpConnection.setRequestProperty("Authorization", "Basic " + encoded); - DataOutputStream out = new DataOutputStream(httpConnection.getOutputStream()); - out.writeBytes(payload); - out.flush(); - out.close(); - httpConnection.getResponseCode(); - httpConnection.disconnect(); - } - - @Override - public void start() { - super.start(); - if (!newBuckets.isEmpty()) { - for (BucketSettings bucketSetting : newBuckets) { - createBucket(bucketSetting, primaryIndex); - } - } - } - -} diff --git a/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java b/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java deleted file mode 100644 index fe950d5..0000000 --- a/src/test/java/com/github/couchmove/container/CouchbaseQueryServiceWaitStrategy.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.github.couchmove.container; - -import com.couchbase.client.core.message.cluster.GetClusterConfigRequest; -import com.couchbase.client.core.message.cluster.GetClusterConfigResponse; -import com.couchbase.client.core.service.ServiceType; -import com.couchbase.client.java.Bucket; -import org.rnorth.ducttape.TimeoutException; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; - -import java.time.Duration; -import java.util.concurrent.TimeUnit; - -import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; - -/** - * @author ctayeb - * Created on 06/06/2017 - */ -public class CouchbaseQueryServiceWaitStrategy extends GenericContainer.AbstractWaitStrategy { - - private final Bucket bucket; - - public CouchbaseQueryServiceWaitStrategy(Bucket bucket) { - this.bucket = bucket; - startupTimeout = Duration.ofSeconds(120); - } - - @Override - protected void waitUntilReady() { - logger().info("Waiting for {} seconds for QUERY service", startupTimeout.getSeconds()); - - // try to connect to the URL - try { - retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { - getRateLimiter().doWhenReady(() -> { - GetClusterConfigResponse clusterConfig = bucket.core() - .send(new GetClusterConfigRequest()) - .toBlocking().single(); - boolean queryServiceEnabled = clusterConfig.config() - .bucketConfig(bucket.name()) - .serviceEnabled(ServiceType.QUERY); - if (!queryServiceEnabled) { - throw new RuntimeException("Query service not ready yet"); - } - }); - return true; - }); - } catch (TimeoutException e) { - throw new ContainerLaunchException("Timed out waiting for QUERY service"); - } - } -} diff --git a/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java b/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java deleted file mode 100644 index 024407d..0000000 --- a/src/test/java/com/github/couchmove/container/CouchbaseWaitStrategy.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (c) 2016 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.github.couchmove.container; - -import com.couchbase.client.deps.com.fasterxml.jackson.databind.JsonNode; -import com.couchbase.client.deps.com.fasterxml.jackson.databind.ObjectMapper; -import org.rnorth.ducttape.TimeoutException; -import org.testcontainers.containers.ContainerLaunchException; -import org.testcontainers.containers.GenericContainer; -import org.testcontainers.shaded.com.google.common.base.Strings; -import org.testcontainers.shaded.com.google.common.io.BaseEncoding; - -import java.io.IOException; -import java.net.HttpURLConnection; -import java.net.URI; -import java.net.URL; -import java.util.concurrent.TimeUnit; - -import static org.rnorth.ducttape.unreliables.Unreliables.retryUntilSuccess; - -/** - * Created by ldoguin on 18/07/16. - */ -public class CouchbaseWaitStrategy extends GenericContainer.AbstractWaitStrategy { - /** - * Authorization HTTP header. - */ - private static final String HEADER_AUTHORIZATION = "Authorization"; - - /** - * Basic Authorization scheme prefix. - */ - private static final String AUTH_BASIC = "Basic "; - - private String path = "/pools/default/"; - private int statusCode = HttpURLConnection.HTTP_OK; - private boolean tlsEnabled; - private String username; - private String password; - private ObjectMapper om = new ObjectMapper(); - - /** - * Indicates that the status check should use HTTPS. - * - * @return this - */ - public CouchbaseWaitStrategy usingTls() { - this.tlsEnabled = true; - return this; - } - - /** - * Authenticate with HTTP Basic Authorization credentials. - * - * @param username the username - * @param password the password - * @return this - */ - public CouchbaseWaitStrategy withBasicCredentials(String username, String password) { - this.username = username; - this.password = password; - return this; - } - - @Override - protected void waitUntilReady() { - final Integer livenessCheckPort = getLivenessCheckPort(); - if (null == livenessCheckPort) { - logger().warn("No exposed ports or mapped ports - cannot wait for status"); - return; - } - - final String uri = buildLivenessUri(livenessCheckPort).toString(); - logger().info("Waiting for {} seconds for URL: {}", startupTimeout.getSeconds(), uri); - - // try to connect to the URL - try { - retryUntilSuccess((int) startupTimeout.getSeconds(), TimeUnit.SECONDS, () -> { - getRateLimiter().doWhenReady(() -> { - try { - final HttpURLConnection connection = (HttpURLConnection) new URL(uri).openConnection(); - - // authenticate - if (!Strings.isNullOrEmpty(username)) { - connection.setRequestProperty(HEADER_AUTHORIZATION, buildAuthString(username, password)); - connection.setUseCaches(false); - } - - connection.setRequestMethod("GET"); - connection.connect(); - - if (statusCode != connection.getResponseCode()) { - throw new RuntimeException(String.format("HTTP response code was: %s", - connection.getResponseCode())); - } - - // Specific Couchbase wait strategy to be sure the node is online and healthy - JsonNode node = om.readTree(connection.getInputStream()); - JsonNode statusNode = node.at("/nodes/0/status"); - String status = statusNode.asText(); - if (!"healthy".equals(status)){ - throw new RuntimeException(String.format("Couchbase Node status was: %s", status)); - } - - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - return true; - }); - - } catch (TimeoutException e) { - throw new ContainerLaunchException(String.format( - "Timed out waiting for URL to be accessible (%s should return HTTP %s)", uri, statusCode)); - } - } - - /** - * Build the URI on which to check if the container is ready. - * - * @param livenessCheckPort the liveness port - * @return the liveness URI - */ - private URI buildLivenessUri(int livenessCheckPort) { - final String scheme = (tlsEnabled ? "https" : "http") + "://"; - final String host = container.getContainerIpAddress(); - - final String portSuffix; - if ((tlsEnabled && 443 == livenessCheckPort) || (!tlsEnabled && 80 == livenessCheckPort)) { - portSuffix = ""; - } else { - portSuffix = ":" + String.valueOf(livenessCheckPort); - } - - return URI.create(scheme + host + portSuffix + path); - } - - /** - * @param username the username - * @param password the password - * @return a basic authentication string for the given credentials - */ - private String buildAuthString(String username, String password) { - return AUTH_BASIC + BaseEncoding.base64().encode((username + ":" + password).getBytes()); - } -} \ No newline at end of file diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index d4af691..1fe320e 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -4,7 +4,7 @@ import com.couchbase.client.java.error.DocumentAlreadyExistsException; import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; -import com.github.couchmove.container.AbstractCouchbaseTest; +import org.testcontainers.couchbase.AbstractCouchbaseTest; import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Type; diff --git a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java index 5efa839..de214ec 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLockServiceTest.java @@ -1,6 +1,6 @@ package com.github.couchmove.service; -import com.github.couchmove.container.AbstractCouchbaseTest; +import org.testcontainers.couchbase.AbstractCouchbaseTest; import com.github.couchmove.exception.CouchmoveException; import org.junit.Test; diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index dd3f3b9..db15383 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -3,6 +3,7 @@ import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.repository.CouchbaseRepository; +import com.google.common.collect.Lists; import org.assertj.core.api.Assertions; import org.junit.Assert; import org.junit.Before; @@ -12,7 +13,6 @@ import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.runners.MockitoJUnitRunner; -import org.testcontainers.shaded.com.google.common.collect.Lists; import java.util.ArrayList; import java.util.List; diff --git a/src/test/resources/logback-test.xml b/src/test/resources/logback-test.xml new file mode 100644 index 0000000..fe273fe --- /dev/null +++ b/src/test/resources/logback-test.xml @@ -0,0 +1,29 @@ + + + + + + %d{HH:mm:ss.SSS} %-5level %logger - %msg%n + + + + + + + + + + + + + + + + + + + PROFILER + DENY + + \ No newline at end of file From b275f113e5c15117507ba42555a506a4223da58a Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 8 Jul 2017 00:38:36 +0200 Subject: [PATCH 35/42] Allow execution of updated failed change logs --- .../couchmove/service/ChangeLogDBService.java | 11 +++++-- .../couchmove/CouchmoveIntegrationTest.java | 27 ++++++++++++++-- .../com/github/couchmove/CouchmoveTest.java | 19 ++++++++--- .../service/ChangeLogDBServiceTest.java | 32 ++++++++++++++++++- .../com/github/couchmove/utils/TestUtils.java | 1 - .../fixed-fail/V1__insert_users.n1ql | 8 +++++ .../fixed-fail/V2__invalid_request.n1ql | 8 +++++ 7 files changed, 94 insertions(+), 12 deletions(-) create mode 100644 src/test/resources/db/migration/fixed-fail/V1__insert_users.n1ql create mode 100644 src/test/resources/db/migration/fixed-fail/V2__invalid_request.n1ql diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index 627bf06..e5af3e5 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -5,6 +5,7 @@ import com.couchbase.client.java.view.DesignDocument; import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; +import com.github.couchmove.pojo.Status; import com.github.couchmove.repository.CouchbaseRepository; import com.github.couchmove.repository.CouchbaseRepositoryImpl; import org.apache.commons.io.FilenameUtils; @@ -18,7 +19,7 @@ * Service for fetching and executing {@link ChangeLog}s * * @author ctayeb - * Created on 03/06/2017 + * Created on 03/06/2017 */ public class ChangeLogDBService { @@ -63,8 +64,12 @@ public List fetchAndCompare(List changeLogs) { dbChangeLog.setChecksum(changeLog.getChecksum()); dbChangeLog.setCas(null); } else if (!dbChangeLog.getChecksum().equals(changeLog.getChecksum())) { - logger.error("Change log version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); - throw new CouchmoveException("ChangeLog checksum doesn't match"); + if (dbChangeLog.getStatus() != Status.FAILED) { + logger.error("Change log version '{}' checksum doesn't match, please verify if the script '{}' content was modified", changeLog.getVersion(), changeLog.getScript()); + throw new CouchmoveException("ChangeLog checksum doesn't match"); + } + dbChangeLog.setStatus(null); + dbChangeLog.setChecksum(changeLog.getChecksum()); } if (!dbChangeLog.getDescription().equals(changeLog.getDescription())) { logger.warn("Change log version '{}' description updated", changeLog.getDescription()); diff --git a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java index 6978677..f68513b 100644 --- a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java @@ -2,7 +2,6 @@ import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; -import org.testcontainers.couchbase.AbstractCouchbaseTest; import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Status; @@ -13,6 +12,7 @@ import com.github.couchmove.service.ChangeLogDBService; import org.junit.BeforeClass; import org.junit.Test; +import org.testcontainers.couchbase.AbstractCouchbaseTest; import java.util.Date; import java.util.List; @@ -28,7 +28,7 @@ /** * @author ctayeb - * Created on 05/06/2017 + * Created on 05/06/2017 */ public class CouchmoveIntegrationTest extends AbstractCouchbaseTest { @@ -133,6 +133,29 @@ public void should_migration_fail_on_exception() { assertLike(changeLogRepository.findOne(PREFIX_ID + "2"), "2", null, "invalid request", N1QL, "V2__invalid_request.n1ql", "890c7bac55666a3073059c57f34e358f817e275eb68932e946ca35e9dcd428fe", FAILED); } + @Test + public void should_fixed_failed_migration_pass() { + // Given a Couchmove instance configured for fail migration folder + Couchmove couchmove = new Couchmove(getBucket(), DB_MIGRATION + "fail"); + + // When we launch migration, then an exception should be raised + assertThrows(couchmove::migrate, CouchmoveException.class); + + // Given a Couchmove instance configured for fixed-fail migration folder + couchmove = new Couchmove(getBucket(), DB_MIGRATION + "fixed-fail"); + + // When we relaunch migration + couchmove.migrate(); + + // Then it should be success + assertLike(changeLogRepository.findOne(PREFIX_ID + "1"), "1", 1, "insert users", N1QL, "V1__insert_users.n1ql", "efcc80f763e48e2a1d5b6689351ad1b4d678c70bebc0c0975a2d19f94e938f18", EXECUTED); + + assertLike(changeLogRepository.findOne(PREFIX_ID + "2"), "2", 2, "invalid request", N1QL, "V2__invalid_request.n1ql", + "778c69b64c030eec8b33eb6ebf955954a3dfa20cab489021a2b71d445d5c3e54", EXECUTED); + + assertEquals(new User("user", "toto", "06/03/1997"), userRepository.findOne("user::toto")); + } + @Test public void should_update_changeLog() { // Given an executed changeLog diff --git a/src/test/java/com/github/couchmove/CouchmoveTest.java b/src/test/java/com/github/couchmove/CouchmoveTest.java index 16931d2..285092f 100644 --- a/src/test/java/com/github/couchmove/CouchmoveTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveTest.java @@ -6,7 +6,6 @@ import com.github.couchmove.service.ChangeLockService; import com.github.couchmove.service.ChangeLogDBService; import com.github.couchmove.service.ChangeLogFileService; -import com.github.couchmove.utils.TestUtils; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -16,9 +15,7 @@ import static com.github.couchmove.pojo.Status.*; import static com.github.couchmove.pojo.Type.*; -import static com.github.couchmove.utils.TestUtils.RANDOM; -import static com.github.couchmove.utils.TestUtils.assertThrows; -import static com.github.couchmove.utils.TestUtils.getRandomChangeLog; +import static com.github.couchmove.utils.TestUtils.*; import static com.google.common.collect.Lists.newArrayList; import static java.util.Collections.emptyList; import static org.mockito.Matchers.any; @@ -26,7 +23,7 @@ /** * @author ctayeb - * Created on 04/06/2017 + * Created on 04/06/2017 */ @RunWith(MockitoJUnitRunner.class) public class CouchmoveTest { @@ -171,6 +168,18 @@ public void should_update_changeLog_on_migration_failure() { Assert.assertEquals(FAILED, changeLog.getStatus()); } + @Test + public void should_execute_failed_changeLog_if_updated() { + Couchmove couchmove = spy(Couchmove.class); + couchmove.setDbService(dbServiceMock); + ChangeLog changeLog = getRandomChangeLog().toBuilder() + .status(FAILED).build(); + doNothing().when(couchmove).doExecute(changeLog); + couchmove.executeMigration(newArrayList(changeLog)); + Assert.assertEquals((Integer) 1, changeLog.getOrder()); + Assert.assertEquals(EXECUTED, changeLog.getStatus()); + } + private static Bucket mockBucket() { Bucket mockedBucket = mock(Bucket.class); when(mockedBucket.name()).thenReturn("default"); diff --git a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java index db15383..1fb5329 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogDBServiceTest.java @@ -17,6 +17,7 @@ import java.util.ArrayList; import java.util.List; +import static com.github.couchmove.pojo.Status.FAILED; import static com.github.couchmove.service.ChangeLogDBService.PREFIX_ID; import static com.github.couchmove.service.ChangeLogDBService.extractRequests; import static com.github.couchmove.utils.TestUtils.*; @@ -24,7 +25,7 @@ /** * @author ctayeb - * Created on 03/06/2017 + * Created on 03/06/2017 */ @RunWith(MockitoJUnitRunner.class) public class ChangeLogDBServiceTest { @@ -112,6 +113,35 @@ public void should_return_updated_changeLog_checksum_with_cas_reset_if_checksum_ Assert.assertNull(changeLog.getCas()); } + @Test + public void should_return_failed_changeLog_with_cas_reset_if_checksum_reset() { + // Given a failed changeLog stored on DB + ChangeLog dbChangeLog = getRandomChangeLog(); + dbChangeLog.setStatus(FAILED); + dbChangeLog.setCas(RANDOM.nextLong()); + when(repository.findOne(PREFIX_ID + dbChangeLog.getVersion())).thenReturn(dbChangeLog); + + // And a changeLog with different checksum + String newChecksum = getRandomString(); + ChangeLog changeLog = dbChangeLog.toBuilder() + .checksum(newChecksum) + .build(); + + // When we call service with the later + List results = service.fetchAndCompare(Lists.newArrayList(changeLog)); + + // Then it should be returned with status reset + Assertions.assertThat(results).hasSize(1); + ChangeLog result = results.get(0); + Assert.assertNull("status", result.getStatus()); + Assert.assertNotNull("cas", result.getCas()); + Assert.assertEquals("description", dbChangeLog.getDescription(), result.getDescription()); + Assert.assertEquals("version", dbChangeLog.getVersion(), result.getVersion()); + Assert.assertEquals("type", dbChangeLog.getType(), result.getType()); + Assert.assertEquals("script", dbChangeLog.getScript(), result.getScript()); + Assert.assertEquals("checksum", newChecksum, result.getChecksum()); + } + @Test public void should_return_updated_changeLog_with_cas_reset_if_description_changed() { // Given a changeLog stored on DB diff --git a/src/test/java/com/github/couchmove/utils/TestUtils.java b/src/test/java/com/github/couchmove/utils/TestUtils.java index 6a5ed3a..6596f1a 100644 --- a/src/test/java/com/github/couchmove/utils/TestUtils.java +++ b/src/test/java/com/github/couchmove/utils/TestUtils.java @@ -36,7 +36,6 @@ public static ChangeLog getRandomChangeLog() { .script("V" + version + "__" + description + (!type.getExtension().isEmpty() ? "." + type.getExtension() : "")) .duration(RANDOM.nextLong()) .checksum(getRandomString()) - .status(Status.values()[Math.abs(RANDOM.nextInt(Status.values().length))]) .build(); } diff --git a/src/test/resources/db/migration/fixed-fail/V1__insert_users.n1ql b/src/test/resources/db/migration/fixed-fail/V1__insert_users.n1ql new file mode 100644 index 0000000..9f0a2be --- /dev/null +++ b/src/test/resources/db/migration/fixed-fail/V1__insert_users.n1ql @@ -0,0 +1,8 @@ +INSERT INTO default (KEY, VALUE) + VALUES + ("user::Administrator", + { + "type": "admin", + "username": "Administrator", + "birthday": "01/09/1998" + }); \ No newline at end of file diff --git a/src/test/resources/db/migration/fixed-fail/V2__invalid_request.n1ql b/src/test/resources/db/migration/fixed-fail/V2__invalid_request.n1ql new file mode 100644 index 0000000..42fddd5 --- /dev/null +++ b/src/test/resources/db/migration/fixed-fail/V2__invalid_request.n1ql @@ -0,0 +1,8 @@ +INSERT INTO default (KEY, VALUE) + VALUES + ("user::toto", + { + "type": "user", + "username": "toto", + "birthday": "06/03/1997" + }); \ No newline at end of file From 4f7d81df96b029f4845c24648727476ac751fe22 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 8 Jul 2017 13:31:35 +0200 Subject: [PATCH 36/42] Inject bucket name parameter to N1QL requests --- pom.xml | 6 ++++++ .../repository/CouchbaseRepositoryImpl.java | 19 ++++++++++++++----- .../couchmove/CouchmoveIntegrationTest.java | 2 +- .../repository/CouchbaseRepositoryTest.java | 11 +++++++++-- .../service/ChangeLogFileServiceTest.java | 2 +- .../github/couchmove/utils/FileUtilsTest.java | 2 +- .../migration/success/V1__create_index.n1ql | 2 +- 7 files changed, 33 insertions(+), 11 deletions(-) diff --git a/pom.xml b/pom.xml index 78c7c6a..083eb1a 100644 --- a/pom.xml +++ b/pom.xml @@ -75,6 +75,11 @@ guava 22.0 + + commons-lang + commons-lang + 2.6 + junit junit @@ -103,6 +108,7 @@ org.assertj assertj-core 3.1.0 + test ch.qos.logback diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java index bb38d62..339d9aa 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepositoryImpl.java @@ -23,6 +23,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; +import org.apache.commons.lang.text.StrSubstitutor; import org.slf4j.Logger; import rx.Observable; @@ -32,6 +33,7 @@ import java.util.function.Function; import static com.couchbase.client.java.query.consistency.ScanConsistency.STATEMENT_PLUS; +import static com.google.common.collect.ImmutableMap.of; import static lombok.AccessLevel.PACKAGE; import static org.slf4j.LoggerFactory.getLogger; @@ -50,6 +52,8 @@ public class CouchbaseRepositoryImpl implements Couch @Getter(lazy = true) private static final ObjectMapper jsonMapper = new ObjectMapper(); + public static final String BUCKET_PARAM = "bucket"; + private final Bucket bucket; private final Class entityClass; @@ -125,21 +129,22 @@ public void save(String id, String jsonContent) { public void importDesignDoc(String name, String jsonContent) { logger.trace("Import document : \n'{}'", jsonContent); bucket.bucketManager().upsertDesignDocument(DesignDocument.from(name, JsonObject.fromJson(jsonContent))); -} + } @Override public void query(String n1qlStatement) { - logger.debug("Execute n1ql request : \n{}", n1qlStatement); + String parametrizedStatement = injectParameters(n1qlStatement); + logger.debug("Execute n1ql request : \n{}", parametrizedStatement); try { AsyncN1qlQueryResult result = runAsync(bucket -> bucket - .query(N1qlQuery.simple(n1qlStatement, + .query(N1qlQuery.simple(parametrizedStatement, N1qlParams.build().consistency(STATEMENT_PLUS)))); if (!result.parseSuccess()) { - logger.error("Invalid N1QL request '{}' : {}", n1qlStatement, single(result.errors())); + logger.error("Invalid N1QL request '{}' : {}", parametrizedStatement, single(result.errors())); throw new CouchmoveException("Invalid n1ql request"); } if (!single(result.finalSuccess())) { - logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : {}", n1qlStatement, single(result.status()), single(result.errors())); + logger.error("Unable to execute n1ql request '{}'. Status : {}, errors : {}", parametrizedStatement, single(result.status()), single(result.errors())); throw new CouchmoveException("Unable to execute n1ql request"); } } catch (Exception e) { @@ -147,6 +152,10 @@ public void query(String n1qlStatement) { } } + String injectParameters(String statement) { + return StrSubstitutor.replace(statement, of(BUCKET_PARAM, getBucketName())); + } + @Override public String getBucketName() { return bucket.name(); diff --git a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java index f68513b..928d9ad 100644 --- a/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java +++ b/src/test/java/com/github/couchmove/CouchmoveIntegrationTest.java @@ -64,7 +64,7 @@ public void should_migrate_successfully() { assertEquals(3, changeLogs.size()); assertLike(changeLogs.get(0), "1", 1, "create index", N1QL, "V1__create_index.n1ql", - "eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5", + "1a417b9f5787e52a46bc65bcd801e8f3f096e63ebcf4b0a17410b16458124af3", EXECUTED); assertLike(changeLogs.get(1), "1.1", 2, "insert users", DOCUMENTS, "V1.1__insert_users", diff --git a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java index 1fe320e..8be9151 100644 --- a/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java +++ b/src/test/java/com/github/couchmove/repository/CouchbaseRepositoryTest.java @@ -4,7 +4,6 @@ import com.couchbase.client.java.error.DocumentAlreadyExistsException; import com.couchbase.client.java.query.util.IndexInfo; import com.couchbase.client.java.view.DesignDocument; -import org.testcontainers.couchbase.AbstractCouchbaseTest; import com.github.couchmove.exception.CouchmoveException; import com.github.couchmove.pojo.ChangeLog; import com.github.couchmove.pojo.Type; @@ -12,6 +11,7 @@ import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.testcontainers.couchbase.AbstractCouchbaseTest; import java.util.List; import java.util.Random; @@ -127,10 +127,17 @@ public void should_import_design_doc() { Assert.assertNotNull(designDocument); } + @Test + public void should_inject_bucket_name() { + String format = "SELECT * FROM `%s`"; + String statement = format(format, "${bucket}"); + Assert.assertEquals(format(format, getBucket().name()), ((CouchbaseRepositoryImpl) repository).injectParameters(statement)); + } + @Test public void should_execute_n1ql() { // Given a primary index request - String request = format("CREATE INDEX `%s` ON `%s`(`%s`)", INDEX_NAME, getBucket().name(), INDEX_NAME); + String request = format("CREATE INDEX `%s` ON `${bucket}`(`%s`)", INDEX_NAME, INDEX_NAME); // When we execute the query repository.query(request); diff --git a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java index 0947493..4a68eff 100644 --- a/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java +++ b/src/test/java/com/github/couchmove/service/ChangeLogFileServiceTest.java @@ -60,7 +60,7 @@ public void should_fetch_changeLogs() { .script("V1__create_index.n1ql") .version("1") .description("create index") - .checksum("eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5") + .checksum("1a417b9f5787e52a46bc65bcd801e8f3f096e63ebcf4b0a17410b16458124af3") .build(), ChangeLog.builder() .type(Type.DOCUMENTS) diff --git a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java index 88c46a4..d0b9809 100644 --- a/src/test/java/com/github/couchmove/utils/FileUtilsTest.java +++ b/src/test/java/com/github/couchmove/utils/FileUtilsTest.java @@ -48,7 +48,7 @@ public void should_get_folder_path_from_resource() throws Exception { public static Object[][] fileProvider() { return new Object[][]{ {DB_MIGRATION_PATH + "V1.1__insert_users", "99a4aaf12e7505286afe2a5b074f7ebabd496f3ea8c4093116efd3d096c430a8"}, - {DB_MIGRATION_PATH + "V1__create_index.n1ql", "eb4ed634d72ea0af9da0b990e0ebc81f6c09264109078e18d3d7b77cb64f28a5"}, + {DB_MIGRATION_PATH + "V1__create_index.n1ql", "1a417b9f5787e52a46bc65bcd801e8f3f096e63ebcf4b0a17410b16458124af3"}, {DB_MIGRATION_PATH + "V2__user.json", "22df7f8496c21a3e1f3fbd241592628ad6a07797ea5d501df8ab6c65c94dbb79"} }; } diff --git a/src/test/resources/db/migration/success/V1__create_index.n1ql b/src/test/resources/db/migration/success/V1__create_index.n1ql index 89065a6..909de19 100644 --- a/src/test/resources/db/migration/success/V1__create_index.n1ql +++ b/src/test/resources/db/migration/success/V1__create_index.n1ql @@ -1,3 +1,3 @@ -- create Index -CREATE INDEX user_index ON default(username) +CREATE INDEX user_index ON `${bucket}`(username) WHERE type = 'user'; \ No newline at end of file From 717ab2d95e63431b58542c8c4a8588ab23ff9137 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 8 Jul 2017 13:35:00 +0200 Subject: [PATCH 37/42] Bump version 1.0-SNAPSHOT --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 083eb1a..a0a40d0 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.github.differentway couchmove - develop-SNAPSHOT + 1.0-SNAPSHOT From 4f453c8801a6090d62d9533f4f2d48f82a9a8ae1 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 8 Jul 2017 13:38:51 +0200 Subject: [PATCH 38/42] Add support for automatic release with Travis --- .travis.yml | 19 ++++++++ pom.xml | 85 ++++++++++++++++++++++++++++++++++++ release/.gitignore | 1 + release/codesigning.asc.enc | Bin 0 -> 9648 bytes release/deploy.sh | 6 +++ release/settings.xml | 22 ++++++++++ 6 files changed, 133 insertions(+) create mode 100644 release/.gitignore create mode 100644 release/codesigning.asc.enc create mode 100755 release/deploy.sh create mode 100644 release/settings.xml diff --git a/.travis.yml b/.travis.yml index cd9dd58..daba379 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,25 @@ script: # JaCoCo is used to have code coverage, the agent has to be activated - mvn clean org.jacoco:jacoco-maven-plugin:prepare-agent package sonar:sonar +deploy: + provider: script + skip_cleanup: true + script: release/deploy.sh + on: + tags: true + branch: master + +env: + global: + # OSSRH_JIRA_USERNAME + - secure: "caws3wHcWWqniMDyBq0TejNpJdgZYogiLWsWNMRAbnKvvwtwP21OQDAs/fIW8/7R85U1gT/WY9J/W375EzMHLn7kCqd8H15DMvWbr06WPhs/oxcWIaRaijv9YmVxX/aqPc/31B0YsEY/f5bDtZcu8guVKvhGpBsZMVTK1pHshUbskYvx/0NLDkJgC/KhW0taYfYoQ+6aKS6s15kqUyC+kXMrf7qNiiajWPbgARWAAPHci/XWzlqn9QD+kUq4YK2xgtq6ris5fudfrA1/z5EG+7E5qdsZ7UZaJfs3PGnBv1tpLTKWt2KxcgMVI+P5nEsjxpZ//RJDW7g9wKvJFbwgfB+2b44dAMzer4xOjzK+PeueiOsgXP42MzyUXDlTSAks7+W2u48qntQVYCOv9pMb0rKjUk5LL6SaGQuiZPgNs13jZXiPH8EVxwsf67qjz/wf0KNpejoDxhwa5mnlqOLyTc/+NufIf0Zea1t3Et5YwXZR7i9DvS+N8j01eeHYhIPjMtSAidO7lj+3oRMTw4O+hrnplj3khUJd3J585I6QEPYxcPYH/2gUOmrl7rXuPC0CHa7oiXsHcpBZsGdDxzkkv1s4qTEH6Y3uujb+rXV3CF1cUMGqXwXztyed0WO9KSdXvTjYK/8jQussSVJanbVd3XGUFSYqhJxOWMpxBIqDgQk=" + # OSSRH_JIRA_PASSWORD + - secure: "cYNSycG67mhSUHrOZRcsXeuxMJJ2laScbXT9BNXTThu8GGHat2H/uHIBWvmUl8XVzNGTcQb6L0iI2EGYDAXwqmSWD2qhBStcRBmwSEPvEHmExEiBy8vL33YVH/YWOg50f39A3AZNZAWarrqEZV1UAMiC3Eqgsmn0xkl+C+KJ5WWPMOQItYfV+jq3Gl+MCO6zKF0CLJWfkm4KFbvGkUeQDAgg7+F9kqTAN8Tj/fTxV5PY2MqsqNKVbi91ObhYM3ChW1ZrA/4CTbUAoooAXyc8yJGDgHNVqeOfC4Pe6+BZyl231438x0jf1M2SHA3izx9OHLirSWAOFyseOImxYrTUw+4yB0u43p/2EtY7Njls0dTTVXDjTadskIKYnM/yFVjlw7IGULSahOyICLA/t5bWnl8CMHsPv83ezdRnloMha6My0oB9k4Qk8JHDv4ZP7o/FSKQpngbA/0KEunclLIqlkjqAJ+5vyQrdCyp2KbAGFQEDv6/3U7P8KxM6HSgy21I8r1M3QBrvTHfbw9UbTNBZeDRy0LPwBGo6A7UH6SALW3gYkX8RGan2otEZxUFKX/ZGDnTK9sWiNyUYPlBfMTKCdQlcTKtRorK/4ypQdvcm29jV+44mxKvmvikddAi92sLUgvhfo7nSVgQ5JJYGVLyZEhK2SDzqimJTDq5gsJe+KwA=" + # GPG_KEY_NAME + - secure: "amjeDAgcasXubwZ9hIYf1SfuwmYg9/CQxYwR5vDEzxXbKGzXJWUWy5aDWv/8b6NZ6lOm+wiaGutEvmF/M43SYhc+7NUQZfNBJQQJMZrILsUhFN6hCVeQKSZQ8932fvHnlXzGMrleiUpW/5m5YajS9GyCkV3+Z7hSxW8MuLsYeFGkmCU8S0idUGjt17EklBn4ZNYN1UQr6VJsRgf1ZLzdmdULeQaJBToZNdrP3HH7nfwACa/l7Hq1b1wN87du9QwWcPONc/7JMeZwXpX2kYdQgAaGTk91Z0Kycp7yjMw5SQKm349y3cvensWz0EAmq9ElEUJStafHSat57NjcHXR+fuynfwde0wRkVa9pfJb126lQCnv0E5rT3pJn7GeRXTcUKFKV8pzITEZbU4Xv6j0WFgLF5Ms9Tw01bpCVASq3e1ebO2yjGxLYPhCTOfSw3N5Pubo5DsTDVvFZcZ9aH1/E6wbJ+b1J4lvzyH+xWVw3aFa/hoksfu+vMfOe24YNymaOeOc9V++bi71Ddf8+cWuOHcESPa7M4YEBO6PpfGKqXPK/7W8DrKjfupAfH4qIMUI5KTDPBtjvujyQcQU3949B0QcJcvHGyjk4P0F6L47xUhE4CnyVRV3FkIZita25atZJFc9o3ZiBCJIQ8Jau2u/H/lgGc9EkS7pirmJlwWWcOEY=" + # GPG_PASSPHRASE + - secure: "DR4U1DbQBQyVfsKay6QHlQ//sZmQpvx/N0vkECtQqUtZuROBsoIwQj/s5Sc79VIIWjFQGLOaOaOKd8nC0s1Om0RTB2a1DdaUIrBlsLP7vfVeGwCE4FsJdrIY3mDfg7V+gQenbzvhqtupx7/+l2lQcXQb84Jqd9cECjfRUpNy9l441oTlnmV9p8NiM6hkFI2t2SLZAmTjkzIaaOZO16jBKgoALTd+OpGvmYLUYDlNXwpSI0sdyzWMIEudREDAWUsJRGiOFUdy4AyfhZpfGHV2PhKo4wBNIZLH6l5Ifb3KKVtOeavgN4rQ5u+tNPI/PoRnyoTbHHBFe5ICjvZewwhxJPxlw7UOF4iPURlla+3QTvQCV1ptQHmJ7E/fZZ1jZbDIEb8se4XtLOk5Nxcfp6FTcKvVBAoo3mezONDxBSC2w3LnNtwPLRVtfH3BaqnEQwZ4yb5JGjFwgQ4mbMzc1fF6w+L26LgsoayUoykulKwaKvqk1BSyIVvC0/To6bIPUsqWACAIXFmlqhGt3aCQhoAAxF2uItGPBbMW/+KepOS0PRsM13TrZDdqc8TWiSsW8KdfsvswR1dT7BDfsNtwNCCSx2m2qFxdxwAd81IcVtVgcpQiWj6R9378SUnTr4KtM2UCMumV5xcpn5/rZ2pqQSOFVl5M6xiiOK4Q9It7whapZUc=" + cache: directories: - '$HOME/.m2/repository' diff --git a/pom.xml b/pom.xml index a0a40d0..410f1a4 100644 --- a/pom.xml +++ b/pom.xml @@ -118,6 +118,13 @@ + + + ossrh + https://oss.sonatype.org/content/repositories/snapshots + + + @@ -129,6 +136,84 @@ ${java.version} + + maven-scm-plugin + 1.9.4 + + ${project.version} + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.6 + true + + ossrh + https://oss.sonatype.org/ + true + + + + + + + sign + + + + org.apache.maven.plugins + maven-gpg-plugin + 1.6 + + + sign-artifacts + verify + + sign + + + + + + + + + + + build-extras + + + + org.apache.maven.plugins + maven-source-plugin + 2.4 + + + attach-sources + + jar-no-fork + + + + + + org.apache.maven.plugins + maven-javadoc-plugin + 2.10.3 + + + attach-javadocs + + jar + + + + + + + + \ No newline at end of file diff --git a/release/.gitignore b/release/.gitignore new file mode 100644 index 0000000..be046be --- /dev/null +++ b/release/.gitignore @@ -0,0 +1 @@ +codesigning.asc \ No newline at end of file diff --git a/release/codesigning.asc.enc b/release/codesigning.asc.enc new file mode 100644 index 0000000000000000000000000000000000000000..3e99506f2072fe5d56abfdc4d2d45a99710e27a0 GIT binary patch literal 9648 zcmV;hB~RL|t)7?E2(cz-_O@(#{SxKKkRlG&Joq4Pd6#CIcrND}t96f27}Jy2FEBhketc@hzWuX=lC0es+(*k3z+ zz(8#4@$mpV=qv0XOpy1#4{Ky=a1Pr9|@@rAraz;28tUm=Z+7qRH z%rlucR$2rj+LvZ-H?_*5l@z{j^pydPJCu_KXYE^mNIoK1DZI4Wj3!7fl_K2BG0x)8 zmfkGvspH5!VcRVPE5gklEp!~DQz3~{*V$COtZEg`fs}6|=M$cCDP11+2nlp6q#BQ* zRDOLwMn!cqi6u{hS=+PXz~tE58G6y36xzRXi?n!HKW5pdDNFOh8y&XGme+-7?gZt7 zvtURB zemff~xo~l}>FDs*Ba|Cz9p4ckf?+ObPZ7l)6OEb5gNCV|Y_Z3*jq2l~KZ=rR12pI{ zMT?}>k@Ywj2o{M9NrG>Y&_%tZgl6uP(P)K4IgqSN6roe-=0Oqc97y>D-aa^lJO}Gu@LBI3_%A!(vGXFaev<8U zR+O4Ot*Z&J004N=aqDF30Lo?Xb4{02p~@xE(#xy5j6Nd+%Rj?XJ%Dw~NbL|axo-z| z!YE1O(ejK`OB;85Hen2P$wnL9l}HB*pn z5j;=!E-C-;&D+bk8iImNUF4rg6q(--y5;paNVn&@$Ysp z0o2q+kH1|$bgz%u!lI|ruX(U^eF|R(_7*xo4tivEPDS!cGG2oX1tWa4t^7H6Tn&#V zj>|3X3mM5D*OV`ejDl@bwN|Mh=K>izhpUPC95bT=ke7U}G26Kvm$rTRAasa9hVnx+0EkuhiNib}A zTpjg9u^gBbHdtAN-;*K4;1|;M?nTw6ZBvZ6ZHhzQgacT@#B^uHv)&WBdKQghUY4~0 z7;y0{cN$`9iF-QKEdma{q>2ISwydoe3ATr6%I`J{wa`{ttF_iad!mqz0BU9!?Q$go z&^e9bxnMG_@`+3So$ID8B>tu-Ma{EXI|5muWMxMcz$zxdOAgjT?7>9cAz+|I2npPA z@O{!fN0w(QWVS&F%>`tc69;qW*-$!Mz!H($!@3(AkA{vt4}mL<`~ZDDa7dzxq==L0rd= zJ$w7)r%5XMRms@$Y)>K)bDy{_Be3}npFo6VVc&HYbiwy(f-g^X9{IT}<@i$GCiKnf z1P^YG`F|nHS1N^?SV7@NExC%np2HVr_i30R(%(jPg+u%qm79GVl4i zp=Sx>X$7(Y0j3+2(X5Q&5fu&w-dorN=hn-AX|VykV)PHRH#Hx1g+28KccF--a>`LV zmVj1ZGKWb^o@O~3l=eqTUXB^MOj{Y0RN~$K0)^ce>vyWWvuh;K-$uXmW*_TWpo6U; zIb{ISz=!>Y3bJC%kN~t~x`tSp8antCrv`6;YVj%3o(_BzQh=Dp4$;$dk>z$SMoV7~SuqjH`&yp>Qw>{d z;m#-rwwJJTCBL!%4dclM7e$-sUs$Eh9WVJ^;^tPV}>p>?CK-d zJPgV!ze$W6)1v-YXMQLI5kydRH8SENSb&V|nK(JuQdE zndISds4~<-1#kzl6IgM-+R5vFu4%iJF2bLg%UnZ9&Qz;t8nt{Wj7ow&m686>>=Z<@bd#=m6rc z&QlVy%Q#qXl4kC4y8DAFiY8?iDO}Hbt6B1}{0)AQXsD5k|FaGva|4LFvS$0CwyTxK zbB<9(Mxz0ObTALa_&{-4t6Ide?OsVnsbL)97mna5_JmRUvL&LryFhGzK$QYtQ$ZM5 zQo8kd>i;!e3Zx#Y+4LITlves7*xN$hlgBH?<#=jBS^;~O$2%uc&G>-ShmK@Ul7Bu} z^?IxVSB_7`n>gy4)tRA4zb03XWjg*KK;xs%K@T&K7v-fpGz}VhL#N^yzknjcg&Da{ zi*RjdVv~G}oF?i_-IDoH0G*NDL9^yZa&fC~`LpFc|5d<04$XfAbMW@;Z@PH&MM+Rhxsl|FV=Ybg#^R1m&>JS zxj#A~(t;;Zo4EHoU3$FYK7j|@7qzTGl`V>tIGk8qBNAm&?EmHcmn@#co`8Abvar5LXcKjcM*4H9iE(6)Pp?8M zW5pNVJ&vvd=Lx@1bra+nr$;6GI-o-5esZ8>D7S^rNd3<|%etvh19&4;_SYp>2c(zm z{GDg0E2J^)py%gsX9eIxRG}2YcJ%yg@$zSZ;&|pJ4dq*5J_v#Q!)Sf6;*29DU;B<@ zMHWm12ahjm zw2?aJnq%&ydfeDnR1cR*FXPbr+OdD??J5~KcWMl4@e?6VGk^uWbODHcuiEEg@I4&~ zMW~){kvcrNjpkR@-|QirtKkUG33rLAK&V4iH;rElNVT2dZ#zoi3eCo}rf>z_?%ET! zFCqPZW4iSV$AeI60lpBKO-jNzby@W0U~Hsf2DbK+-<%ZOj|jNBwrYjMe9nF`0|(AG znP!w&)E`*NaqPMZKT!g+kl>NIxb4Nf3=r|C8`xNt`9cIC5JJ2a=S-=qi^0ILW|>nFh$h-<5{cb~pQ zzy;dQPrl9|DfuTJhQHSiv5p?&6=_qnR75G0`b@4WvM0$Lo>p1P$P{;KSjkY2X0zQ8 zc)j($XDg`iGHRmwmyW`tD6y_XO6;!DtxR1Zb!}Tk55^3j5CiMHe=h4>{-nB*W!alE zSI?;v^yt-^TqQ8^Gd~GJm-&MZy#~xPoLW!Q(~cg%I^a$^xodYLt10$HMVI+s%eRHm)TjR+7sPd0YBx8#0^-!|FyDG}K9)**Ir??Eo%@2`27jBs1FE(K z>qdVDxvICHGPzlo#rRWbpJ%7ANAs06xQh2EE#S%&4ksA$gqHlb!i&uMVNAywZSD7| z_`ljtqx9{>@*lOOKZuUtAj6G>C46%ZQ!P`_&Y(v(YXyD5T{T{n0GZd!>nJc_e2*g- z#)EJEgOvz-!8H9NBn}_s^bo3?g@8a*Pityr&a5I{NB=X7Y(C;BA>^RF^*7HI^+bM+ zht%r&!JGl@8K|w@{$Y*Wuo!9SwB*pu1)YPD`azo%ua&=k~3R zM;|C8REdF6bt(cpQV;HSJN6smVq`=bl1$_b!oolbhhvS77H4oelOW$fzUH$LX;UP% z0UkwYv$NuBs&eSE&S%aHh0UiETULzCct_N=EClC~%x}S8BA5s{Jo5|_vEe|h0he0NjPT0TuU%5&00*}~HhS;oU z#JW2EC%njpsx-j)ov;;ozn4*dY)MX_W(P=ye!(bW4fEEFt^jeBf&IJAxFqepmx&;q zK(k?@qL_O>SQ_A=f`q^azl|4~D|akEqS%J|4;HMTQ<=ica9{>ha(^CofF|&t)c02v zFerQ!^&$9cYyD(=1{)M*PR-g`i-S~CF3s$0@Gzjep9*m9LH8OWsO?}K=`}7>M7%PYXC+a@o z8aF$J%7>C0roq>FVV1x+7&dY?14Eq?e#WjU1*003h4b-rW7Ay3Z12H?k$4l)e*C1+ z+FUEjs`=}T*Knc|yt-8M{K6kyWI7j5_6mvDSHD*Utw+y=^&e*v& zjkwRLMWuu?OtBz`Tzv0ytUj!b5L{LsVWJjA-)$wCqNa9mfnS~E6g{&S1gHq;!JY&# zEd8r(P>|7j@)VRxhXiMA$iefS%1^%***b@g_xMA?|7!nxPbT_APN~c^(VD>_=C0Ow z7`ZztsC$fne5BMA8AK|T*@pCn(!fjrdAu=&5|TRJR>V(nM+cX-28 z>5%?7&?{r>^%BIf>$zIP5`3)KQDy zMC$+p^cxU$1Qv;ZYDbB844MaW$=PbvH#1Cx6I|X`gK7C~usd5T(dbek<)uJ4W<-&_lm!1i}mZ6$a zJ)e*Do1SVNIl*F%alSD*2w1XDwN+*?E*oE?&{@CFenl$j=D$ z%gM=gm$oRX{-7frH$vOn(WY2aEs94!kab4wPtx3sZ4=gcOhz$?KLNgHk0q<)W~&`1 z;JKv8@vAVBYU~J3&Y_rVb+eUA6c!@g0fTIMA%gpp#$s=J8F~So&cLm;w0G6X-xo-I z?KsCP+`%5VjpPm$49J^Pp=q{3LE$joBxoQ3lSUFu>_I^tBV~}-m6)QUwd4w|0_2S@ ztRlqSn!LeXXH|_kDdY^b3Xy?n0f+-eub196vW^1jb7TQE)4^$V*Yv-h>rN0k6^>#F zhKLTR_={Hf1~Ze8@kv5dbMWu^(EO!$lEj| zpZ?I97*!Nw41-^>O1i&kr+%U%xo@5u1hcqF=gX}%MeeuROjPwo{GcV+h0vyTdVR1O z;$v!c$Pxhw7zXYM)rP!2qTl(@L4u%88o<*i^IF7%#QJ;1 z-AdM7`c^nF)>tX_@YPH!uQ~<6RlH#EL_IlYtJf3Tv7Mze2B+&4Pd}~7e0yO?(*T^B zSuz$ZPtt7vp)oj`N9Zg^5i8(O^fP>zxVAzooL`0*MyJD%4Ui>DW3rpAenBOm8KM|0 zLa#kLA(FHP;T|&fS2=tc{O@k-ZDcWrJnCXkK{~4Xc0w`B$YY$Sa-)z@$vZI)KA`pn z#ru%36o%*uhc^an@<_7-0+s#vwGdD4-9LVf$14gZoGRSqPTqqMlTJ;rKpIe8AQrba z9ZK_yUX;Qpord&Y-z3GUZmEJLNV_oO@!=a@j6T`fpu~J5Z*PkLze&+Mk}WcL>-x%^UPJ_1bL z#DcyFU+|~9o#QK|pKIAVI7?jC0@~wcV4eW#x`5i3#k8b;=Mtkf!R;M(aX6lleHIIW z#ml)C2}EIJ3c#^O82)J<6&d>9AKN7jC!J!mr%L>i{3~BiA4Rd7*?upO!tL#!&jvbi zrHd*+T5HGuG!fxHon)0EEZNl1NEdF?=muDb#`4Z&G}Q^k_?CcfbJ6gJrDhI+Tc$W5 zJ?y>4=PnJo8WmS5eRxtD8KNYsEiZ)|El9nM&SR}m%f6T*rSc?!O;LxH={&QCr6i6B zXIPjaB(1(&-y!k9DJMSCUOF|DSC0S1DgI;!I|W%KYr9JrFriIs0!h~$L{mCB3k=T9 zE%DE?+<^a@BtWsIlQnf6t`QIiHQ9z55CD-P+kMJT=a%=d z*>{KeWkS6EDlr1ELdo@xnx6@{QQuPgQReyP5!5D^gsTa|?;6u)fj|Mt%FsQ_4cXK9 z7MfVejDlKuy(%f3Ut|3MCndWHGyi?WPpA$0mw7=g<L%go@3IN>{^-#k$&%p1(A zygI9(SY4Tu-v`($^OfE+wP#W!{Yw{Ex?((1rZ*Eq_IprnH&s~`V0u>6Eh}^kM-}kQ zqEt(}s4cVM$+ZJudF{e8wx>o(_A4VL?$@rCpGP5rKVJR;#_TNIdqV7Xz9erCXz4X! zpe8+v9eaec4V&0t1U!^0KKe3c2bZa?z<8Ja`r|?&MVYeAyt-LP20TX`DqOxZPQmku zC&#UIH^9wM2XY3ZsLD>Uql6BrZ$jm(v9-D*C3Yl*VYKX@a|+lK zhOiI;C){|uE>(tfWPeg z0krqUher9N*ZksmM~l>Z5m@8Pi98;|(S4MQdP6`Sh)->-L+)cu&rl zq2rcqM~tf*snB%AAUK^`S^Xf;l<;4B)$_`$)^ zqLi4~ilHf^2ni+?3yUHIe2Dgk(V~1farh+JVc%euk0$2fhgL@ARj*z67OPM(X)*Iy zv72}e*QLSnz$RN@L5X?`QSC65C;HRw|oiVYHqpcZ5xY>rG^Vfsg3J+g3E<9gDd7sv-EeP^NfKV=*ZJUjh* zHmp9547Zh)v+C94(6VdZQ<%doKg&E*aWZot!tz3nQzZ;JB%c2!audzbV|gGAB0S*& zn&c~6T2TMJVo#nkp_8BtR&+b!=foTuQ4eZ9=Vj-$4)$ha7QP*j-SDS@Jrlc8F7maHD(? z6|rQ8_KA6Z& zoCQb_`ANx44;XTe+`+{U(x?jCJf+@WfD#_F$uBlPFmjWDRv`Zy0LWRl+ip0uMN_$V z%)}$u$R^W`kVtuNS%4UpyI&_w#*h#eX5CaHDHinL5qcrWPHvDh4#6>eCWM#XR(GMO z-rJsk66pm6Q#O=;jT^y9qWZ7fTC3M^^(-trpU?0zlc4@rg^=i&G_?@MWv07l4$7C# z9y>*G!?1SuPdrGxBe}&60^}!>DSRD2&Hqr4J$Wp3Q{F%8o7|k09E_rQhgyRIh`DzW z5AVkX8`(X-4V=V{k1Qf@0R-MH?u=qQ@k)V;)2tj<1RF#2KtpFo=)R3c-@Qu^a}hCsDh0B$bAHz*Ar>#KWRINYGI?FyPZ-74;cG7AuovVX#|l0E zS}bs(x`mAM%o=h!$KqU8|A`WGVR-(3(y{+n z&&{9>|9bR%Ok#6K%IYEs`>t}`f~iSN!VC5uGE930mLC$c6hVUs+%HCb6qA*$dRULFoeeZfb$}vT6I~_B&H_? zuiTW*IgZJad*5qxeTe;n0Q*7=!1*5dicZ#}^3XjYU z*&C?L8XtX^-c{pZ`Uyy~5#;=?U5d7UIju-ZRTJ%EO3fcRlq7m)wMHKO;W1qsm|Fv- zVqM<%;ta@hN9i(mjL^cy&4mwhU48xH@MbW|c_<+S|1FYs6%UvjeNIdXdq$U=65UP2 zIE#9ESS@28#tYDG-fVe7l{g(63(0%^>PXdw^+p}YCFExvR)O-@2&(sE9V;>g8Us?k z+soP9x~D-+PDAxmBHtGGA2}KNJn0J}KKzVSk(g+9;~xC71WivMRid3bXg#e@J~#QQ7Z()?r5HBEb8*ve|J-<~-p~{*GguODoYoY>hq61OloT-0{Sh zp}dgc$!F!Hj}$)mFM_5}u6j3~Ioj-W*@>8mt9c0fF=&2#G7PF{-a7QBBdsuV z_`CYDOr~SjT#DRW+f?BDbz*63LVKd$ufY*wr-EQzzz_l-5UuBw7#rcub&RzuJ0W|M zdeP8xt%FZ|0c*7=aJXT#OH<~!EdenV&@JV++`4we#v7A@sBM<-J@Rft0cd2R(k&Kl z6G_V1!Qmvgc?f4x4J+bP&7mh}%cZ>|A~Ae;vqOIXkt&62B0E>Ko%p0>bO?BIkw_OE zyRNd{79w+jZ?+dmn+NeF|dN8A6l?l2BOGUbQ8_sW9m@~h;o2qX)81%QiDX7KEVD> zO}9bkwfKz{#N+fp5KmJjsZyVQ!qIO-o>kdXZ=IjHClznV88PI8B_+W{c4-(+n4?Z` z$v7}W2=DgER!!;)ZZ=!?(^_PIz@hLizNKDGB zE^Kq`2#*_J7V?83!b>e`i)Jf)NAmBsQH|%xy~8`;)Pp?o3U!=6s&-SGxP64lGK`ku zSW+f-)UJbr>#CFNo4iQqaQ&?V3FNFe`GpM=>JE`}0-)!LrN^-_Ux6?f2so8`STxI% zgGgtZN{Ke}Y)|gO%7=~ANxCbICHO={9hoCXsDsjdL7ND~JQ`$`DxeUr3>`_>hEACe z9TXI`YcXUMo(A;Nt+Sc!e6uK0ekuA*SZ|qmVO`&h1^`VCYITx1Di8a(945lc_mOj6 zQiHc(*Smfp4CFkuP{1>C%A}UvgvYIzE8x;9?zU(mti*%83xO|!nWH$oo^f@a6}>`g mhJ{Fq&u1(VrN429*e+)?W(R@)s%2z`%N(@IY!hBBM*ffKl&B>D literal 0 HcmV?d00001 diff --git a/release/deploy.sh b/release/deploy.sh new file mode 100755 index 0000000..6b26d5f --- /dev/null +++ b/release/deploy.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +openssl aes-256-cbc -K $encrypted_467352795a68_key -iv $encrypted_467352795a68_iv -in release/codesigning.asc.enc -out release/codesigning.asc -d +gpg --fast-import release/codesigning.asc + +mvn deploy -P sign,build-extras -DskipTests --settings release/settings.xml \ No newline at end of file diff --git a/release/settings.xml b/release/settings.xml new file mode 100644 index 0000000..cecf892 --- /dev/null +++ b/release/settings.xml @@ -0,0 +1,22 @@ + + + + ossrh + ${env.OSSRH_JIRA_USERNAME} + ${env.OSSRH_JIRA_PASSWORD} + + + + + + ossrh + + true + + + ${env.GPG_KEY_NAME} + ${env.GPG_PASSPHRASE} + + + + \ No newline at end of file From a85b48eeac413dfcf670027ec69c041ba84514bc Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sat, 8 Jul 2017 13:59:20 +0200 Subject: [PATCH 39/42] Fix travis sonar branch config --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd9dd58..7a15585 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,9 +10,9 @@ addons: organization: "differentway-github" token: secure: "fnzS5t/buOCMWV5xx8sGOcPG+6P3LZwFuyOjOI9efm8s6uaYc2dHaKz9A45W2FB6O74pCohO8hVOh+C610fdnlES4JZ3kEy0V4/NWB9jun6w0dT+kPFRCUPmcyrS1zVZvEtlzuy2dUSPUgKfWQKnufZMex1VxghJge022+bGKbxsSYCcn0/EkOnHKN3hcP/WtjgfMQ7NrGrR+nGzZIblQRDL2bLyhx7skI7aVyo4qv93GyFGk5dIqmJtXlh+p8ylzImrJnM+V74NbRQe+YkgYZbH1VNaAzhCiSCRc8YltrAyJXJ1kLS778rIaQptLu2kn3wsZbC1dgGikg0rhy++on/cMvYWPo8LhQO7hGq31pTIblXI3+l0aU+FrKCXbpofIxbXwzBmZUOLa+StfnB9ANvsC9sn2RZ0A73U7lo/4jGY5EmjjyCze7TcyDonySyA/BrmwvDgnxKXrkAcI5jsY4bK+3Zy8pZkCYhoqilTwMsvs54m5skmLA3qv6l4tdmtNRgZD3EUnNutkjpp86gbrMa6d4k0/b2pxSjnK+MhQWKcpgXbH+Z935gTVTUcWxslu+kPXOhuH2uuiScOCCc0O/R7kPjVbFWakdjylOLFubaGKi9PmCyoYfiAcjfhFGoD7t6pXWjQdo3aRSp1d20qnioKi/c0w66hVQImiSU77Dw=" - branches: - - master - - develop + branches: + - master + - develop script: # JaCoCo is used to have code coverage, the agent has to be activated From eb0d1d862a46f53909749b900a500b3d061702a6 Mon Sep 17 00:00:00 2001 From: differentway Date: Sun, 9 Jul 2017 21:29:38 +0200 Subject: [PATCH 40/42] Update README.md Add Sonar quality gate badge --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 60d7e23..270dad3 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,5 @@ Couchmove is widely inspired from [Flyway](http://flywaydb.org) : it strongly fa Check out our [wiki](https://github.com/differentway/couchmove/wiki) - --- -[![Build Status](https://travis-ci.org/differentway/couchmove.svg?branch=develop)](https://travis-ci.org/differentway/couchmove) [![Licence](https://img.shields.io/hexpm/l/plug.svg)](https://github.com/differentway/couchmove/blob/master/LICENSE) +[![Build Status](https://travis-ci.org/differentway/couchmove.svg?branch=develop)](https://travis-ci.org/differentway/couchmove) [![Quality Gate](https://sonarcloud.io/api/badges/gate?key=com.github.differentway:couchmove:develop)](https://sonarqube.com/dashboard/index/com.github.differentway:couchmove:develop) [![Licence](https://img.shields.io/hexpm/l/plug.svg)](https://github.com/differentway/couchmove/blob/master/LICENSE) From 1d0b423d23c427c18e5c99c4c75a53f50a415852 Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 23 Jul 2017 16:58:34 +0200 Subject: [PATCH 41/42] Fix javadoc --- .../com/github/couchmove/repository/CouchbaseRepository.java | 3 +-- .../java/com/github/couchmove/service/ChangeLogDBService.java | 4 ++-- src/main/java/com/github/couchmove/utils/Utils.java | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java index a8049d4..a3f47ae 100644 --- a/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java +++ b/src/main/java/com/github/couchmove/repository/CouchbaseRepository.java @@ -46,11 +46,10 @@ public interface CouchbaseRepository { /** * Retrieves a document from Couchbase {@link Bucket} by its ID. *

    - *

    * - If the document exists, convert it to {@link CouchbaseEntity} with CAS set (Check And Swap for optimistic concurrency) *
    * - Otherwise it return null - * + *

    * @param id the id of the document * @return the found and converted {@link CouchbaseEntity} with CAS set, or null if absent */ diff --git a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java index e5af3e5..34f6013 100644 --- a/src/main/java/com/github/couchmove/service/ChangeLogDBService.java +++ b/src/main/java/com/github/couchmove/service/ChangeLogDBService.java @@ -36,10 +36,10 @@ public ChangeLogDBService(Bucket bucket) { /** * Get corresponding ChangeLogs from Couchbase bucket *
      - *
    • if a {@link ChangeLog} doesn't exist => return it as it its + *
    • if a {@link ChangeLog} doesn't exist → return it as it its *
    • else : *
        - *
      • if checksum ({@link ChangeLog#checksum}) is reset (set to null), or description ({@link ChangeLog#description}) updated => reset {@link ChangeLog#cas} + *
      • if checksum ({@link ChangeLog#checksum}) is reset (set to null), or description ({@link ChangeLog#description}) updated → reset {@link ChangeLog#cas} *
      • return database version *
      *
    diff --git a/src/main/java/com/github/couchmove/utils/Utils.java b/src/main/java/com/github/couchmove/utils/Utils.java index 657224a..8b4c1bd 100644 --- a/src/main/java/com/github/couchmove/utils/Utils.java +++ b/src/main/java/com/github/couchmove/utils/Utils.java @@ -56,7 +56,7 @@ private static String initializeUserName() { /** * Format duration to human readable *

    - * Exemple : 188,100,312 {@link TimeUnit#MILLISECONDS} => 2d 4h 15m 15s 312ms + * Exemple : 188,100,312 {@link TimeUnit#MILLISECONDS} → 2d 4h 15m 15s 312ms * * @param duration duration to format * @param timeUnit source timeUnit From 18dec4bdeaf5ae7cd6f9452db32210cf91afd1fc Mon Sep 17 00:00:00 2001 From: ctayeb Date: Sun, 23 Jul 2017 16:27:09 +0200 Subject: [PATCH 42/42] Bump version 1.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 410f1a4..ebbe56b 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.github.differentway couchmove - 1.0-SNAPSHOT + 1.0