From 4e12265087f85b579a44ae5d66633c2d9a80492f Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 13:56:57 +0100 Subject: [PATCH] fix: fixed bugs --- ...alidationSecretsApplicationConverter.java} | 4 +- ...ValidationSecretsApplicationConverter.java | 34 ++++++ .../dto/ContentCompoundUnitDto.java | 2 +- .../dto/FolderContentUnitDto.java | 23 ++++ .../RepositoryContentApplicationUnitDto.java | 28 +++++ .../entity/common/PropertiesEntity.java | 3 + .../repository/ContentRepository.java | 49 ++++++++ .../repository/facade/RepositoryFacade.java | 61 +++++++++- .../integration/backup/BackupService.java | 109 +++++++++++++++++- .../TemporateStorageService.java | 7 +- .../service/processor/ProcessorService.java | 44 ------- .../service/telemetry/TelemetryService.java | 19 +++ .../telemetry/binding/TelemetryBinding.java | 8 +- .../service/workspace/WorkspaceService.java | 39 +++++++ .../common/WorkspaceConfigurationHelper.java | 17 +++ .../workspace/facade/WorkspaceFacade.java | 40 ++++--- api-server/src/main/openapi/openapi.yml | 1 + .../src/main/resources/application.properties | 3 + 18 files changed, 416 insertions(+), 75 deletions(-) rename api-server/src/main/java/com/objectstorage/converter/{ContentCompoundUnitToValidationSecretsApplicationConverter.java => ContentCompoundUnitsToValidationSecretsApplicationConverter.java} (87%) create mode 100644 api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java create mode 100644 api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java create mode 100644 api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java create mode 100644 api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java diff --git a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java similarity index 87% rename from api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java rename to api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java index 84ec9c2..180c75b 100644 --- a/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitToValidationSecretsApplicationConverter.java +++ b/api-server/src/main/java/com/objectstorage/converter/ContentCompoundUnitsToValidationSecretsApplicationConverter.java @@ -10,7 +10,7 @@ /** * Represents content compounds units to validation secrets application converter; */ -public class ContentCompoundUnitToValidationSecretsApplicationConverter { +public class ContentCompoundUnitsToValidationSecretsApplicationConverter { /** * Converts given content compound units to validation secrets application. @@ -25,7 +25,7 @@ public static ValidationSecretsApplication convert(List element -> validationSecretsUnits.add( ValidationSecretsUnit.of( element.getProvider(), - element.getCredentialsFieldsFull()))); + element.getCredentials()))); return ValidationSecretsApplication.of(validationSecretsUnits); } diff --git a/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java new file mode 100644 index 0000000..8826ad1 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/converter/RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter.java @@ -0,0 +1,34 @@ +package com.objectstorage.converter; + +import com.objectstorage.dto.ContentCompoundUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.model.ValidationSecretsUnit; + +import java.util.ArrayList; +import java.util.List; + +/** + * Represents repository content application units to validation secrets application converter; + */ +public class RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter { + + /** + * Converts given content compound units to validation secrets application. + * + * @param repositoryContentApplicationUnits given repository content application units. + * @return converted validation secrets application. + */ + public static ValidationSecretsApplication convert( + List repositoryContentApplicationUnits) { + List validationSecretsUnits = new ArrayList<>(); + + repositoryContentApplicationUnits.forEach( + element -> validationSecretsUnits.add( + ValidationSecretsUnit.of( + element.getProvider(), + element.getCredentials()))); + + return ValidationSecretsApplication.of(validationSecretsUnits); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java index 6a944ac..d4643f6 100644 --- a/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java +++ b/api-server/src/main/java/com/objectstorage/dto/ContentCompoundUnitDto.java @@ -25,5 +25,5 @@ public class ContentCompoundUnitDto { /** * Represents full credentials fields. */ - private CredentialsFieldsFull credentialsFieldsFull; + private CredentialsFieldsFull credentials; } diff --git a/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java new file mode 100644 index 0000000..691f727 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/FolderContentUnitDto.java @@ -0,0 +1,23 @@ +package com.objectstorage.dto; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +import java.io.InputStream; + +/** + * Represents folder content unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class FolderContentUnitDto { + /** + * Represents folder entity name. + */ + private String location; + + /** + * Represents folder entity content. + */ + private byte[] content; +} diff --git a/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java new file mode 100644 index 0000000..2750cac --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/dto/RepositoryContentApplicationUnitDto.java @@ -0,0 +1,28 @@ +package com.objectstorage.dto; + +import com.objectstorage.model.CredentialsFieldsFull; +import com.objectstorage.model.Provider; +import lombok.AllArgsConstructor; +import lombok.Getter; + +/** + * Represents repository content application unit. + */ +@Getter +@AllArgsConstructor(staticName = "of") +public class RepositoryContentApplicationUnitDto { + /** + * Represents root location for internal file system. + */ + private String root; + + /** + * Represents provider. + */ + private Provider provider; + + /** + * Represents full credentials fields. + */ + private CredentialsFieldsFull credentials; +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java index 4cf79a1..ec3e7a9 100644 --- a/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java +++ b/api-server/src/main/java/com/objectstorage/entity/common/PropertiesEntity.java @@ -68,6 +68,9 @@ public class PropertiesEntity { @ConfigProperty(name = "workspace.content.backup.directory") String workspaceContentBackupDirectory; + @ConfigProperty(name = "workspace.content.backup.unit") + String workspaceContentBackupUnit; + @ConfigProperty(name = "workspace.compression.file.name") String workspaceCompressionFileName; diff --git a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java index 8d4fda1..32412d8 100644 --- a/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java +++ b/api-server/src/main/java/com/objectstorage/repository/ContentRepository.java @@ -102,6 +102,55 @@ public ContentEntity findByProviderAndSecret(Integer provider, Integer secret) t return ContentEntity.of(id, provider, secret, root); } + /** + * Retrieves all the persisted content entities. + * + * @return retrieved content entities. + * @throws RepositoryOperationFailureException if repository operation fails. + */ + public List findAll() throws RepositoryOperationFailureException { + ResultSet resultSet; + + try { + resultSet = + repositoryExecutor.performQueryWithResult( + String.format( + "SELECT t.id, t.root, t.provider, t.secret FROM %s as t", + properties.getDatabaseContentTableName())); + + } catch (QueryExecutionFailureException | QueryEmptyResultException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + List result = new ArrayList<>(); + + Integer id; + String root; + Integer provider; + Integer secret; + + try { + while (resultSet.next()) { + id = resultSet.getInt("id"); + root = resultSet.getString("root"); + provider = resultSet.getInt("provider"); + secret = resultSet.getInt("secret"); + + result.add(ContentEntity.of(id, provider, secret, root)); + } + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + try { + resultSet.close(); + } catch (SQLException e) { + throw new RepositoryOperationFailureException(e.getMessage()); + } + + return result; + } + /** * Deletes all entities with the given provider and secret from content table. * diff --git a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java index 6bb351e..370a41e 100644 --- a/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java +++ b/api-server/src/main/java/com/objectstorage/repository/facade/RepositoryFacade.java @@ -1,9 +1,6 @@ package com.objectstorage.repository.facade; -import com.objectstorage.dto.ContentCompoundUnitDto; -import com.objectstorage.dto.RepositoryContentUnitDto; -import com.objectstorage.dto.EarliestTemporateContentDto; -import com.objectstorage.dto.TemporateContentUnitDto; +import com.objectstorage.dto.*; import com.objectstorage.entity.repository.ContentEntity; import com.objectstorage.entity.repository.ProviderEntity; import com.objectstorage.entity.repository.SecretEntity; @@ -160,7 +157,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem throw new TemporateContentRetrievalFailureException(e.getMessage()); } - CredentialsFieldsFull secrets = + CredentialsFieldsFull credentials = repositoryConfigurationHelper.convertRawSecretsToContentCredentials( provider, secret.getSession(), @@ -179,7 +176,7 @@ public EarliestTemporateContentDto retrieveEarliestTemporateContent() throws Tem RepositoryContentUnitDto.of( contentEntity.getRoot()), provider, - secrets)); + credentials)); } return EarliestTemporateContentDto.of( @@ -300,6 +297,58 @@ public RepositoryContentUnitDto retrieveContentApplication(ValidationSecretsUnit return RepositoryContentUnitDto.of(contentEntity.getRoot()); } + /** + * Retrieves all content applications from the content repository. + * + * @return retrieved all content applications. + * @throws ContentApplicationRetrievalFailureException if content applications retrieval fails. + */ + public List retrieveAllContentApplications() + throws ContentApplicationRetrievalFailureException { + List contentEntities; + + try { + contentEntities = contentRepository.findAll(); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + List repositoryContentApplicationUnits = new ArrayList<>(); + + for (ContentEntity content : contentEntities) { + ProviderEntity rawProvider; + + try { + rawProvider = providerRepository.findById(content.getProvider()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + Provider provider = + repositoryConfigurationHelper.convertRawProviderToContentProvider(rawProvider.getName()); + + SecretEntity secret; + + try { + secret = secretRepository.findById(content.getSecret()); + } catch (RepositoryOperationFailureException e) { + throw new ContentApplicationRetrievalFailureException(e.getMessage()); + } + + CredentialsFieldsFull credentials = + repositoryConfigurationHelper.convertRawSecretsToContentCredentials( + provider, + secret.getSession(), + secret.getCredentials()); + + repositoryContentApplicationUnits.add( + RepositoryContentApplicationUnitDto.of( + content.getRoot(), provider, credentials)); + } + + return repositoryContentApplicationUnits; + } + /** * Removes temporate content from the temporate repository with the given hash. * diff --git a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java index ed572ce..453fc2a 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/backup/BackupService.java @@ -1,18 +1,31 @@ package com.objectstorage.service.integration.backup; import com.objectstorage.converter.CronExpressionConverter; -import com.objectstorage.exception.BackupPeriodRetrievalFailureException; -import com.objectstorage.exception.CronExpressionException; +import com.objectstorage.converter.RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter; +import com.objectstorage.dto.FolderContentUnitDto; +import com.objectstorage.dto.RepositoryContentApplicationUnitDto; +import com.objectstorage.entity.common.PropertiesEntity; +import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalProviderUnit; +import com.objectstorage.model.ValidationSecretsApplication; +import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.state.StateService; +import com.objectstorage.service.telemetry.TelemetryService; import com.objectstorage.service.vendor.VendorFacade; +import com.objectstorage.service.vendor.common.VendorConfigurationHelper; import com.objectstorage.service.workspace.facade.WorkspaceFacade; +import io.quarkus.runtime.Startup; import jakarta.annotation.PostConstruct; import jakarta.annotation.PreDestroy; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import com.objectstorage.service.config.ConfigService; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -20,8 +33,14 @@ /** * Provides backup configuration, which will create a local backup of uploaded files. */ +@Startup(value = 900) @ApplicationScoped public class BackupService { + private static final Logger logger = LogManager.getLogger(BackupService.class); + + @Inject + PropertiesEntity properties; + @Inject ConfigService configService; @@ -31,6 +50,12 @@ public class BackupService { @Inject VendorFacade vendorFacade; + @Inject + RepositoryFacade repositoryFacade; + + @Inject + TelemetryService telemetryService; + private final ScheduledExecutorService scheduledOperationExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -53,6 +78,86 @@ public void process() throws BackupPeriodRetrievalFailureException { scheduledOperationExecutorService.scheduleWithFixedDelay(() -> { StateService.getBackupProcessorGuard().lock(); + List repositoryContentApplicationUnits; + + try { + repositoryContentApplicationUnits = repositoryFacade.retrieveAllContentApplications(); + } catch (ContentApplicationRetrievalFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + ValidationSecretsApplication validationSecretsApplication = + RepositoryContentApplicationUnitsToValidationSecretsApplicationConverter + .convert(repositoryContentApplicationUnits); + + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + + for (RepositoryContentApplicationUnitDto repositoryContentApplicationUnit : + repositoryContentApplicationUnits) { + List contentRetrievalProviderUnits; + + try { + contentRetrievalProviderUnits = vendorFacade.listAllObjectsFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot())); + } catch (SecretsConversionException | BucketObjectRetrievalFailureException | VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + List folderContentUnits = new ArrayList<>(); + + for (ContentRetrievalProviderUnit contentRetrievalProviderUnit : contentRetrievalProviderUnits) { + byte[] content; + + try { + content = vendorFacade.retrieveObjectFromBucket( + repositoryContentApplicationUnit.getProvider(), + repositoryContentApplicationUnit.getCredentials().getExternal(), + VendorConfigurationHelper.createBucketName( + repositoryContentApplicationUnit.getRoot()), + contentRetrievalProviderUnit.getLocation() + ); + } catch ( + SecretsConversionException | + BucketObjectRetrievalFailureException | + VendorOperationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + folderContentUnits.add(FolderContentUnitDto.of( + contentRetrievalProviderUnit.getLocation(), content)); + } + + try { + workspaceFacade.addBackupFile( + workspaceUnitKey, + workspaceFacade.createFileUnitKey(properties.getWorkspaceContentBackupUnit()), + folderContentUnits); + } catch (FileCreationFailureException e) { + StateService.getBackupProcessorGuard().unlock(); + + logger.error(e.getMessage()); + + return; + } + + telemetryService.increaseCurrentBackupsAmount(); + } + StateService.getBackupProcessorGuard().unlock(); }, 0, period, TimeUnit.MILLISECONDS); } diff --git a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java index 57d4807..30251c4 100644 --- a/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java +++ b/api-server/src/main/java/com/objectstorage/service/integration/temporatestorage/TemporateStorageService.java @@ -1,12 +1,11 @@ package com.objectstorage.service.integration.temporatestorage; -import com.objectstorage.converter.ContentCompoundUnitToValidationSecretsApplicationConverter; +import com.objectstorage.converter.ContentCompoundUnitsToValidationSecretsApplicationConverter; import com.objectstorage.converter.CronExpressionConverter; import com.objectstorage.dto.ContentCompoundUnitDto; import com.objectstorage.dto.EarliestTemporateContentDto; import com.objectstorage.exception.*; import com.objectstorage.model.ValidationSecretsApplication; -import com.objectstorage.model.ValidationSecretsUnit; import com.objectstorage.repository.executor.RepositoryExecutor; import com.objectstorage.repository.facade.RepositoryFacade; import com.objectstorage.service.config.ConfigService; @@ -155,7 +154,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { } ValidationSecretsApplication validationSecretsApplication = - ContentCompoundUnitToValidationSecretsApplicationConverter.convert( + ContentCompoundUnitsToValidationSecretsApplicationConverter.convert( temporateContentDto.getContentCompoundUnits()); String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); @@ -192,7 +191,7 @@ public void process() throws TemporateStoragePeriodRetrievalFailureException { try { vendorFacade.uploadObjectToBucket( contentCompoundUnit.getProvider(), - contentCompoundUnit.getCredentialsFieldsFull().getExternal(), + contentCompoundUnit.getCredentials().getExternal(), VendorConfigurationHelper.createBucketName( contentCompoundUnit.getRepositoryContentUnitDto().getRoot()), temporateContentDto.getLocation(), diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 0afea18..5b4bedb 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -188,24 +188,6 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) } for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { - RepositoryContentUnitDto repositoryContentLocationUnitDto; - - try { - repositoryContentLocationUnitDto = repositoryFacade.retrieveContentApplication(validationSecretsUnit); - } catch (ContentApplicationRetrievalFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } - try { repositoryFacade.withdraw(validationSecretsUnit); } catch (RepositoryContentDestructionFailureException e1) { @@ -221,32 +203,6 @@ public void withdraw(ValidationSecretsApplication validationSecretsApplication) throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); } - - try { - if (vendorFacade.isBucketPresent( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot()))) { - vendorFacade.removeBucket( - validationSecretsUnit.getProvider(), - validationSecretsUnit.getCredentials().getExternal(), - VendorConfigurationHelper.createBucketName( - repositoryContentLocationUnitDto.getRoot())); - } - } catch (SecretsConversionException | VendorOperationFailureException e1) { - try { - repositoryExecutor.rollbackTransaction(); - } catch (TransactionRollbackFailureException e2) { - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e2.getMessage()); - } - - StateService.getTransactionProcessorGuard().unlock(); - - throw new ProcessorContentWithdrawalFailureException(e1.getMessage()); - } } try { diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java index be88cf3..b8c7f50 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/TelemetryService.java @@ -35,6 +35,8 @@ public class TelemetryService { private final ConcurrentLinkedQueue averageUploadFileSizeQueue = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue currentBackupsAmountQueue = new ConcurrentLinkedQueue<>(); + private final static ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(0, Thread.ofVirtual().factory()); @@ -66,6 +68,12 @@ private void configure() { averageUploadFileSizeQueue.poll().run(); } }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); + + scheduledExecutorService.scheduleWithFixedDelay(() -> { + if (!currentBackupsAmountQueue.isEmpty()) { + currentBackupsAmountQueue.poll().run(); + } + }, 0, properties.getDiagnosticsScrapeDelay(), TimeUnit.MILLISECONDS); } /** @@ -113,4 +121,15 @@ public void setAverageUploadFileSizeQueue(Double value) { () -> telemetryBinding.getAverageUploadFileSize().set(value)); } } + + /** + * Increases performed cloud service backup operations. + */ + public void increaseCurrentBackupsAmount() { + if (configService.getConfig().getDiagnostics().getEnabled()) { + currentBackupsAmountQueue.add( + () -> telemetryBinding.getCurrentBackupsAmount().set( + telemetryBinding.getCurrentBackupsAmount().get() + 1)); + } + } } diff --git a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java index 6c6b8f4..7b51ec1 100644 --- a/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java +++ b/api-server/src/main/java/com/objectstorage/service/telemetry/binding/TelemetryBinding.java @@ -28,6 +28,8 @@ public class TelemetryBinding implements MeterBinder { private final AtomicDouble averageUploadFileSize = new AtomicDouble(); + private final AtomicInteger currentBackupsAmount = new AtomicInteger(); + /** * @see MeterBinder */ @@ -38,7 +40,7 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { .register(meterRegistry); Gauge.builder("general.current_cloud_service_uploads_amount", currentCloudServiceUploadsAmount, AtomicInteger::get) - .description("Represents amount of uploads to different cloud services") + .description("Represents amount of uploads to different cloud services in the current session") .register(meterRegistry); Gauge.builder("general.cloud_service_upload_retries", cloudServiceUploadRetries, AtomicInteger::get) @@ -52,5 +54,9 @@ public void bindTo(@NotNull MeterRegistry meterRegistry) { Gauge.builder("general.average_upload_file_size", averageUploadFileSize, AtomicDouble::get) .description("Represents average upload file size in ObjectStorage Temporate Storage") .register(meterRegistry); + + Gauge.builder("general.current_backups_amount", currentBackupsAmount, AtomicInteger::get) + .description("Represents amount of performed cloud service backup operation in the current session") + .register(meterRegistry); } } diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java index a9da9a8..fa79435 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java @@ -1,8 +1,10 @@ package com.objectstorage.service.workspace; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; import com.objectstorage.exception.FileNotFoundException; +import com.objectstorage.service.workspace.common.WorkspaceConfigurationHelper; import jakarta.enterprise.context.ApplicationScoped; import jakarta.inject.Inject; import jakarta.xml.bind.DatatypeConverter; @@ -19,6 +21,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Stream; +import java.util.zip.Deflater; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; @@ -316,6 +319,42 @@ public byte[] compressFile(InputStream inputStream) throws return result.toByteArray(); } + /** + * Compresses given folder entities of the given type. This will act as a non-compressed folder, which has + * previously compressed with zip files. + * + * @param folderContentUnits given folder input entities. + * @param type given folder type. + * @return compressed folder input. + * @throws InputCompressionFailureException if input compression fails. + */ + public byte[] compressFolder(List folderContentUnits, String type) throws + InputCompressionFailureException { + ByteArrayOutputStream result = new ByteArrayOutputStream(); + + try (ZipOutputStream writer = new ZipOutputStream(result)) { + writer.setMethod(ZipOutputStream.DEFLATED); + writer.setLevel(Deflater.NO_COMPRESSION); + + writer.putNextEntry(new ZipEntry(WorkspaceConfigurationHelper.getZipFolderDefinition(type))); + + for (FolderContentUnitDto folderContentUnit : folderContentUnits) { + writer.putNextEntry(new ZipEntry(folderContentUnit.getLocation())); + + writer.write(folderContentUnit.getContent()); + } + + writer.flush(); + + writer.finish(); + + } catch (IOException e) { + throw new InputCompressionFailureException(e.getMessage()); + } + + return result.toByteArray(); + } + /** * Adds new file of the given type to the workspace with the given workspace unit key as the compressed input stream. * diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java new file mode 100644 index 0000000..41cc530 --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java @@ -0,0 +1,17 @@ +package com.objectstorage.service.workspace.common; + +/** + * Contains helpful tools used for workspace configuration. + */ +public class WorkspaceConfigurationHelper { + + /** + * Creates folder definition with the help of the given folder name for ZIP achieve. + * + * @param name given folder name value. + * @return wrapped token. + */ + public static String getZipFolderDefinition(String name) { + return String.format("%s/", name); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 35e010a..48a8759 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -1,5 +1,6 @@ package com.objectstorage.service.workspace.facade; +import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; import com.objectstorage.model.ValidationSecretsApplication; @@ -10,6 +11,7 @@ import java.io.*; import java.time.Instant; +import java.util.List; import java.util.stream.Collectors; /** @@ -55,7 +57,12 @@ public String createWorkspaceUnitKey(ValidationSecretsApplication validationSecr * @return created file unit key. */ public String createFileUnitKey(String name) { - return workspaceService.createUnitKey(name, Instant.now().toString()); + Instant timestamp = Instant.now(); + + String fileUnit = + workspaceService.createUnitKey(name, Instant.now().toString()); + + return String.format("%s-%s-%d", name, fileUnit, timestamp.toEpochMilli()); } /** @@ -75,13 +82,26 @@ public void addObjectFile(String workspaceUnitKey, String name, InputStream inpu * Adds new backup file to the workspace with the given workspace unit key as the compressed input stream. * * @param workspaceUnitKey given user workspace unit key. - * @param name given content name. - * @param inputStream given input. + * @param name given file name. + * @param folderContentUnits given folder content units. * @throws FileCreationFailureException if file creation operation failed. */ - public void addBackupFile(String workspaceUnitKey, String name, InputStream inputStream) + public void addBackupFile(String workspaceUnitKey, String name, List folderContentUnits) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name, inputStream); + byte[] content; + + try { + content = + workspaceService.compressFolder(folderContentUnits, properties.getWorkspaceContentBackupDirectory()); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentBackupDirectory(), + name, + new ByteArrayInputStream(content)); Integer amount; @@ -166,16 +186,6 @@ public void removeObjectFile(String workspaceUnitKey, String name) throws FileRe workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name); } - /** - * Removes backup file with the given name from the workspace with the help of the given workspace unit key. - * - * @param workspaceUnitKey given user workspace unit key. - * @throws FileRemovalFailureException if file removal operation failed. - */ - public void removeBackupFile(String workspaceUnitKey, String name) throws FileRemovalFailureException { - workspaceService.removeContentFile(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory(), name); - } - /** * Removes all the files from the workspace with the help of the given workspace unit key. * diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index b89e1f0..15a7eb9 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -264,6 +264,7 @@ components: required: - pending - uploaded + - backups properties: pending: type: array diff --git a/api-server/src/main/resources/application.properties b/api-server/src/main/resources/application.properties index f7dd05d..d9213f4 100644 --- a/api-server/src/main/resources/application.properties +++ b/api-server/src/main/resources/application.properties @@ -74,6 +74,9 @@ workspace.content.object.directory=object # Describes location of backup content directory. workspace.content.backup.directory=backup +# Describes location of backup content unit. +workspace.content.backup.unit=backup + # Describes name of the file used for compression operation. workspace.compression.file.name=objectstorage-file