alignement with Esop 1.0.5, release of 1.0.6

pom.xml

@@ -5,15 +5,15 @@
 
     <groupId>com.instaclustr</groupId>
     <artifactId>icarus</artifactId>
-    <version>1.0.5</version>
+    <version>1.0.6</version>
 
     <name>instaclustr-icarus</name>
     <description>Sidecar for Apache Cassandra</description>
     <url>https://github.com/instaclustr/instaclustr-icarus</url>
 
     <properties>
         <instaclustr.sidecar.common.version>1.3.13</instaclustr.sidecar.common.version>
-        <instaclustr.esop.version>1.0.4</instaclustr.esop.version>
+        <instaclustr.esop.version>1.0.5</instaclustr.esop.version>
 
         <testng.version>6.14.3</testng.version>
         <mockito.version>2.23.4</mockito.version>

src/main/java/com/instaclustr/icarus/Icarus.java

@@ -16,6 +16,8 @@
 import com.instaclustr.esop.impl.backup.BackupModules;
 import com.instaclustr.esop.impl.backup.BackupModules.BackupModule;
 import com.instaclustr.esop.impl.backup.BackupModules.CommitlogBackupModule;
+import com.instaclustr.esop.impl.hash.HashModule;
+import com.instaclustr.esop.impl.hash.HashSpec;
 import com.instaclustr.esop.impl.restore.RestoreModules;
 import com.instaclustr.esop.impl.restore.RestoreModules.RestorationStrategyModule;
 import com.instaclustr.esop.impl.restore.RestoreModules.RestoreCommitlogModule;
@@ -65,6 +67,9 @@ public final class Icarus extends CLIApplication implements Callable<Void> {
     @Mixin
     public CassandraJMXSpec jmxSpec;
 
+    @Mixin
+    public HashSpec hashSpec;
+
     @Spec
     private CommandSpec commandSpec;
 
@@ -94,7 +99,7 @@ public Void call() throws Exception {
         logCommandVersionInformation(commandSpec);
 
         // production binds singletons as eager by default
-        final Injector injector = createInjector(PRODUCTION, getModules(icarusSpec, jmxSpec, enableTruncateOperation));
+        final Injector injector = createInjector(PRODUCTION, getModules(icarusSpec, jmxSpec, hashSpec, enableTruncateOperation));
 
         return injector.getInstance(Application.class).call();
     }
@@ -104,12 +109,13 @@ public String getImplementationTitle() {
         return "instaclustr-icarus";
     }
 
-    public List<AbstractModule> getModules(SidecarSpec icarusSpec,
-                                           CassandraJMXSpec jmxSpec,
+    public List<AbstractModule> getModules(final SidecarSpec icarusSpec,
+                                           final CassandraJMXSpec jmxSpec,
+                                           final HashSpec hashSpec,
                                            final boolean enableTruncateOperation) throws Exception {
         List<AbstractModule> modules = new ArrayList<>();
 
-        modules.addAll(backupRestoreModules());
+        modules.addAll(backupRestoreModules(hashSpec));
         modules.addAll(operationModules());
         modules.addAll(icarusModules(icarusSpec, jmxSpec));
 
@@ -154,7 +160,7 @@ protected void configure() {
         }};
     }
 
-    public static List<AbstractModule> backupRestoreModules() {
+    public static List<AbstractModule> backupRestoreModules(final HashSpec hashSpec) {
         return new ArrayList<AbstractModule>() {{
             add(new StorageModules());
             add(new BackupModule());
@@ -164,6 +170,7 @@ public static List<AbstractModule> backupRestoreModules() {
             add(new RestorationStrategyModule());
             add(new BackupModules.UploadingModule());
             add(new RestoreModules.DownloadingModule());
+            add(new HashModule(hashSpec));
         }};
     }
 

src/main/java/com/instaclustr/icarus/coordination/IcarusBackupOperationCoordinator.java

@@ -19,13 +19,15 @@
 import com.instaclustr.cassandra.CassandraVersion;
 import com.instaclustr.esop.guice.BackuperFactory;
 import com.instaclustr.esop.guice.BucketServiceFactory;
+import com.instaclustr.esop.impl.CassandraData;
 import com.instaclustr.esop.impl.KeyspaceTable;
 import com.instaclustr.esop.impl.StorageLocation;
 import com.instaclustr.esop.impl.backup.BackupOperation;
 import com.instaclustr.esop.impl.backup.BackupOperationRequest;
 import com.instaclustr.esop.impl.backup.Backuper;
 import com.instaclustr.esop.impl.backup.UploadTracker;
 import com.instaclustr.esop.impl.backup.coordination.BaseBackupOperationCoordinator;
+import com.instaclustr.esop.impl.hash.HashSpec;
 import com.instaclustr.esop.topology.CassandraClusterTopology;
 import com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology;
 import com.instaclustr.icarus.rest.IcarusClient;
@@ -55,8 +57,9 @@ public IcarusBackupOperationCoordinator(final CassandraJMXService cassandraJMXSe
                                             final SidecarSpec icarusSpec,
                                             final ExecutorServiceSupplier executorServiceSupplier,
                                             final ObjectMapper objectMapper,
-                                            final UploadTracker uploadTracker) {
-        super(cassandraJMXService, cassandraVersionProvider, backuperFactoryMap, bucketServiceFactoryMap, objectMapper, uploadTracker);
+                                            final UploadTracker uploadTracker,
+                                            final HashSpec hashSpec) {
+        super(cassandraJMXService, cassandraVersionProvider, backuperFactoryMap, bucketServiceFactoryMap, objectMapper, uploadTracker, hashSpec);
         this.icarusSpec = icarusSpec;
         this.executorServiceSupplier = executorServiceSupplier;
     }
@@ -70,7 +73,8 @@ public void coordinate(final Operation<BackupOperationRequest> operation) {
         }
 
         try {
-            KeyspaceTable.checkEntitiesToProcess(operation.request.cassandraDirectory.resolve("data"), operation.request.entities);
+            CassandraData data = CassandraData.parse(operation.request.cassandraDirectory.resolve("data"));
+            data.setDatabaseEntitiesFromRequest(operation.request.entities);
         } catch (final Exception ex) {
             logger.error(ex.getMessage());
             operation.addError(Operation.Error.from(ex));

src/main/java/com/instaclustr/icarus/coordination/IcarusRestoreOperationCoordinator.java

@@ -29,6 +29,7 @@
 import com.instaclustr.esop.guice.BucketServiceFactory;
 import com.instaclustr.esop.guice.RestorerFactory;
 import com.instaclustr.esop.impl.BucketService;
+import com.instaclustr.esop.impl.CassandraData;
 import com.instaclustr.esop.impl.StorageLocation;
 import com.instaclustr.esop.impl.restore.RestorationPhase.RestorationPhaseType;
 import com.instaclustr.esop.impl.restore.RestorationStrategyResolver;
@@ -89,8 +90,8 @@ public void coordinate(final Operation<RestoreOperationRequest> operation) throw
 
         /*
          * I receive a request
-         *  If it is a global request, I will be coordinator
-         *  otherwise just execute that request
+         * If it is a global request, I will be coordinator
+         * otherwise just execute that request
          */
 
         // if it is not global request, there might be at most one global request running
@@ -155,22 +156,39 @@ public void coordinate(final Operation<RestoreOperationRequest> operation) throw
             throw new IllegalStateException("ID of a running operation does not equal to ID of this restore operation!");
         }
 
-        if (operation.request.restorationPhase != DOWNLOAD) {
-            throw new IllegalStateException(format("Restoration coordination has to start with %s phase.", DOWNLOAD));
+        if (!operation.request.singlePhase && operation.request.restorationPhase != INIT) {
+            throw new IllegalStateException("If global request is true and singlePhase is false, restorationPhase has to be INIT");
+        }
+
+        try {
+            CassandraData data = CassandraData.parse(operation.request.cassandraDirectory.resolve("data"));
+            data.setDatabaseEntitiesFromRequest(operation.request.entities);
+            data.setRenamedEntitiesFromRequest(operation.request.rename);
+        } catch (final Exception ex) {
+            throw new IllegalStateException(ex.getMessage());
         }
 
         try (final IcarusWrapper icarusWrapper = new IcarusWrapper(getSidecarClients())) {
             final IcarusWrapper oneClient = getOneClient(icarusWrapper);
 
-            final GlobalOperationProgressTracker progressTracker = new GlobalOperationProgressTracker(operation, numberOfOperations(icarusWrapper));
-
-            final ResultSupplier[] resultSuppliers = new ResultSupplier[]{
-                    () -> executePhase(new InitPhasePreparation(), operation, oneClient, progressTracker),
-                    () -> executePhase(new DownloadPhasePreparation(), operation, icarusWrapper, progressTracker),
-                    () -> executePhase(new TruncatePhasePreparation(), operation, oneClient, progressTracker),
-                    () -> executePhase(new ImportingPhasePreparation(), operation, icarusWrapper, progressTracker),
-                    () -> executePhase(new CleaningPhasePreparation(), operation, icarusWrapper, progressTracker),
-            };
+            final GlobalOperationProgressTracker progressTracker = new GlobalOperationProgressTracker(operation,
+                                                                                                      numberOfOperations(icarusWrapper, operation.request.singlePhase));
+
+            ResultSupplier[] resultSuppliers;
+
+            if (operation.request.singlePhase) {
+                resultSuppliers = new ResultSupplier[]{
+                        () -> executePhase(getPhase(operation.request.restorationPhase), operation, icarusWrapper, progressTracker)
+                };
+            } else {
+                resultSuppliers = new ResultSupplier[]{
+                        () -> executePhase(new InitPhasePreparation(), operation, icarusWrapper, progressTracker),
+                        () -> executePhase(new DownloadPhasePreparation(), operation, icarusWrapper, progressTracker),
+                        () -> executePhase(new TruncatePhasePreparation(), operation, oneClient, progressTracker),
+                        () -> executePhase(new ImportingPhasePreparation(), operation, icarusWrapper, progressTracker),
+                        () -> executePhase(new CleaningPhasePreparation(), operation, icarusWrapper, progressTracker),
+                };
+            }
 
             for (final ResultSupplier supplier : resultSuppliers) {
                 supplier.getWithEx();
@@ -187,13 +205,13 @@ public void coordinate(final Operation<RestoreOperationRequest> operation) throw
         }
     }
 
-    private int numberOfOperations(final IcarusWrapper icarusWrapper) {
-        int operationsPerPhase = icarusWrapper.icarusClients.size();
-
-        // download, import and cleanup will be done on each node = num of clients * 3
-        // 1 node for init phase + 1 node for truncate phase = 2
-
-        return operationsPerPhase * 3 + 2;
+    private int numberOfOperations(final IcarusWrapper icarusWrapper, final boolean singlePhase) {
+        final int numberOfClients = icarusWrapper.icarusClients.size();
+        if (singlePhase) {
+            return numberOfClients;
+        }
+        // init, download, import and cleanup will be done on each node = num of clients * 4 + 1 node for truncate phase
+        return numberOfClients * 4 + 1;
     }
 
     public static class IcarusWrapper implements Closeable {
@@ -273,6 +291,23 @@ void prepareBasics(final RestoreOperationRequest request, final IcarusClient cli
         }
     }
 
+    public static PhasePreparation getPhase(final RestorationPhaseType phaseType) {
+        switch (phaseType) {
+            case DOWNLOAD:
+                return new DownloadPhasePreparation();
+            case TRUNCATE:
+                return new TruncatePhasePreparation();
+            case IMPORT:
+                return new ImportingPhasePreparation();
+            case CLEANUP:
+                return new CleaningPhasePreparation();
+            case INIT:
+                return new InitPhasePreparation();
+            default:
+                throw new IllegalStateException("Unable to resolve phase preparation from " + phaseType);
+        }
+    }
+
     private static final class DownloadPhasePreparation extends PhasePreparation {
         @Override
         public RestorationPhaseType getPhaseType() {

src/test/java/com/instaclustr/icarus/embedded/AbstractCassandraIcarusTest.java

@@ -24,6 +24,7 @@
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
+import com.instaclustr.esop.impl.hash.HashSpec;
 import com.instaclustr.icarus.Icarus;
 import com.instaclustr.icarus.rest.IcarusClient;
 import com.instaclustr.io.FileUtils;
@@ -192,7 +193,9 @@ protected IcarusHolder sidecar(String jmxAddress, Integer jmxPort, String httpAd
         icarusSpec.httpServerAddress = new HttpServerInetSocketAddressTypeConverter().convert(httpAdddress + ":" + httpPort.toString());
         icarusSpec.operationsExpirationPeriod = new Time(1L, HOURS);
 
-        List<AbstractModule> modules = new Icarus().getModules(icarusSpec, cassandraJMXSpec, true);
+        HashSpec hashSpec = new HashSpec();
+
+        List<AbstractModule> modules = new Icarus().getModules(icarusSpec, cassandraJMXSpec, hashSpec, true);
 
         Injector injector = Guice.createInjector(modules);
 

src/test/java/com/instaclustr/icarus/embedded/singlenode/AbstractSingleNodeBackupFromScratchRestoreTest.java

@@ -1,5 +1,6 @@
 package com.instaclustr.icarus.embedded.singlenode;
 
+import static com.instaclustr.esop.impl.restore.RestorationPhase.RestorationPhaseType.INIT;
 import static com.instaclustr.esop.impl.restore.RestorationStrategy.RestorationStrategyType.IN_PLACE;
 import static com.instaclustr.io.FileUtils.createDirectory;
 import static com.instaclustr.io.FileUtils.deleteDirectory;
@@ -11,7 +12,9 @@
 
 import com.instaclustr.esop.impl.StorageLocation;
 import com.instaclustr.esop.impl.backup.BackupOperationRequest;
+import com.instaclustr.esop.impl.restore.RestorationPhase;
 import com.instaclustr.esop.impl.restore.RestoreOperationRequest;
+import com.instaclustr.esop.impl.retry.RetrySpec;
 import com.instaclustr.esop.topology.CassandraClusterTopology;
 import com.instaclustr.esop.topology.CassandraClusterTopology.ClusterTopology;
 import com.instaclustr.icarus.embedded.AbstractCassandraIcarusTest;
@@ -85,7 +88,8 @@ private BackupOperationRequest createBackupRequest(final String cloud,
                 false, // skip bucket verification
                 null, // schema version
                 false, // topology file, even it is false, global request does not care, it will upload it anyway
-                null // proxy settings
+                null, // proxy settings
+                new RetrySpec(10, RetrySpec.RetryStrategy.EXPONENTIAL, 3, true) // retry
         );
     }
 
@@ -132,7 +136,10 @@ private RestoreOperationRequest createRestoreOperationRequest(final String cloud
                 false, // insecure
                 false, // new cluster
                 false, // skip bucket verification
-                null // proxy settings
+                null, // proxy settings
+                null, // rename
+                new RetrySpec(10, RetrySpec.RetryStrategy.EXPONENTIAL, 3, true), // retry
+                false
         );
     }
 

src/test/java/com/instaclustr/icarus/embedded/singlenode/BackupRestoreOperationTest.java

@@ -49,7 +49,8 @@ public void backupTest() throws Exception {
                     false, // skip bucket verification
                     null, // schemaVersion,
                     false, // upload topology
-                    null // proxy
+                    null, // proxy
+                    null // retry
             );
 
             final OperationResult<BackupOperation> result = icarusHolder.icarusClient.backup(backupOperationRequest);
@@ -105,7 +106,10 @@ public void backupTest() throws Exception {
                     false, // insecure
                     false, // newCluster
                     false, // skipBucketVerification
-                    null // proxy
+                    null, // proxy
+                    null, // rename
+                    null, // retry
+                    false // single
             );
 
             icarusHolder.icarusClient.waitForCompleted(icarusHolder.icarusClient.restore(restoreOperationRequest));

src/test/java/com/instaclustr/icarus/embedded/singlenode/SingleNodeDistributedBackupRestoreTest.java

@@ -1,6 +1,6 @@
 package com.instaclustr.icarus.embedded.singlenode;
 
-import static com.instaclustr.esop.impl.restore.RestorationPhase.RestorationPhaseType.DOWNLOAD;
+import static com.instaclustr.esop.impl.restore.RestorationPhase.RestorationPhaseType.INIT;
 import static com.instaclustr.esop.impl.restore.RestorationStrategy.RestorationStrategyType.HARDLINKS;
 import static com.instaclustr.io.FileUtils.deleteDirectory;
 
@@ -15,9 +15,9 @@
 import com.instaclustr.esop.impl.backup.BackupOperationRequest;
 import com.instaclustr.esop.impl.restore.RestoreOperationRequest;
 import com.instaclustr.esop.impl.truncate.TruncateOperationRequest;
+import com.instaclustr.icarus.embedded.AbstractCassandraIcarusTest;
 import com.instaclustr.icarus.rest.IcarusClient;
 import com.instaclustr.measure.DataRate;
-import com.instaclustr.icarus.embedded.AbstractCassandraIcarusTest;
 import org.testng.annotations.Test;
 
 /**
@@ -47,7 +47,8 @@ private BackupOperationRequest createBackupRequest(String snapshotName) {
                 false, // skip bucket verification
                 null, // schema version
                 false, // topology file, even it is false, global request does not care, it will upload it anyway
-                null // proxy
+                null, // proxy
+                null // retry
         );
     }
 
@@ -64,7 +65,7 @@ private RestoreOperationRequest createRestoreOperationRequest(String schemaVersi
                 DatabaseEntities.parse(keyspaceName), // entities
                 false, // update cassandra yaml
                 HARDLINKS, // restoration strategy
-                DOWNLOAD, // restoration phase
+                INIT, // restoration phase
                 new ImportOperationRequest(null, null, downloadDir), // import
                 false, // noDeleteTruncates
                 false, // noDeleteDownload
@@ -79,7 +80,10 @@ private RestoreOperationRequest createRestoreOperationRequest(String schemaVersi
                 false, // insecure
                 false, // newCluster
                 false, // skip bucket verification
-                null // proxy
+                null, // proxy
+                null, // rename
+                null,
+                false
         );
     }