From a54a5deff1e4a778210470706324c03a4a5d3ea0 Mon Sep 17 00:00:00 2001 From: Yaroslav Svitlytskyi Date: Fri, 29 Nov 2024 15:54:29 +0100 Subject: [PATCH] fix: fixed bugs --- ...itsLocationsRetrievalFailureException.java | 21 --- .../FileUnitsRetrievalFailureException.java | 21 +++ .../service/processor/ProcessorService.java | 12 +- .../service/workspace/WorkspaceService.java | 56 ++++++-- .../common/WorkspaceConfigurationHelper.java | 10 ++ .../workspace/facade/WorkspaceFacade.java | 32 ++++- api-server/src/main/openapi/openapi.yml | 10 ++ config/grafana/dashboards/diagnostics.tmpl | 124 ++++++++++++++++-- 8 files changed, 237 insertions(+), 49 deletions(-) delete mode 100644 api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java create mode 100644 api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java deleted file mode 100644 index 310dc1c..0000000 --- a/api-server/src/main/java/com/objectstorage/exception/FileUnitsLocationsRetrievalFailureException.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.objectstorage.exception; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Formatter; - -/** - * Represents exception used when file units locations retrieval operation fails. - */ -public class FileUnitsLocationsRetrievalFailureException extends IOException { - public FileUnitsLocationsRetrievalFailureException() { - this(""); - } - - public FileUnitsLocationsRetrievalFailureException(Object... message) { - super( - new Formatter() - .format("File units locations retrieval operation failed: %s", Arrays.stream(message).toArray()) - .toString()); - } -} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java new file mode 100644 index 0000000..ba8de4e --- /dev/null +++ b/api-server/src/main/java/com/objectstorage/exception/FileUnitsRetrievalFailureException.java @@ -0,0 +1,21 @@ +package com.objectstorage.exception; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Formatter; + +/** + * Represents exception used when file units retrieval operation fails. + */ +public class FileUnitsRetrievalFailureException extends IOException { + public FileUnitsRetrievalFailureException() { + this(""); + } + + public FileUnitsRetrievalFailureException(Object... message) { + super( + new Formatter() + .format("File units retrieval operation failed: %s", Arrays.stream(message).toArray()) + .toString()); + } +} \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java index 5b4bedb..00d0f39 100644 --- a/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java +++ b/api-server/src/main/java/com/objectstorage/service/processor/ProcessorService.java @@ -55,6 +55,8 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid throws ProcessorContentRetrievalFailureException { List compounds = new ArrayList<>(); + String workspaceUnitKey = workspaceFacade.createWorkspaceUnitKey(validationSecretsApplication); + for (ValidationSecretsUnit validationSecretsUnit : validationSecretsApplication.getSecrets()) { RepositoryContentUnitDto repositoryContentLocationUnitDto; @@ -84,11 +86,19 @@ public ContentRetrievalResult retrieveContent(ValidationSecretsApplication valid VendorOperationFailureException ignored) { } + List backups; + + try { + backups = workspaceFacade.getBackupUnits(workspaceUnitKey); + } catch (FileUnitsRetrievalFailureException e) { + throw new ProcessorContentRetrievalFailureException(e.getMessage()); + } + compounds.add( ContentRetrievalCompound.of( repositoryContentLocationUnitDto.getRoot(), validationSecretsUnit.getProvider().toString(), - List.of(ContentRetrievalUnits.of(pending, uploaded)))); + List.of(ContentRetrievalUnits.of(pending, uploaded, backups)))); } return ContentRetrievalResult.of(compounds); diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java index fa79435..b59b925 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/WorkspaceService.java @@ -23,7 +23,6 @@ import java.util.stream.Stream; import java.util.zip.Deflater; import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; import java.util.zip.ZipOutputStream; import lombok.SneakyThrows; @@ -95,7 +94,7 @@ public void createContentDirectory(String workspaceUnitDirectory, String type) t * @param key given workspace unit directory. * @return result of the check. */ - public Boolean isUnitDirectoryExist(String key) { + private Boolean isUnitDirectoryExist(String key) { return Files.exists(Paths.get(properties.getWorkspaceDirectory(), key)); } @@ -106,7 +105,7 @@ public Boolean isUnitDirectoryExist(String key) { * @param type given content directory type. * @return result of the check. */ - public Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { + private Boolean isContentDirectoryExist(String workspaceUnitDirectory, String type) { return Files.exists(Paths.get(workspaceUnitDirectory, type)); } @@ -339,7 +338,11 @@ public byte[] compressFolder(List folderContentUnits, Stri writer.putNextEntry(new ZipEntry(WorkspaceConfigurationHelper.getZipFolderDefinition(type))); for (FolderContentUnitDto folderContentUnit : folderContentUnits) { - writer.putNextEntry(new ZipEntry(folderContentUnit.getLocation())); + writer.putNextEntry(new ZipEntry( + Path.of( + properties.getWorkspaceContentBackupDirectory(), + WorkspaceConfigurationHelper.getZipFile( + folderContentUnit.getLocation())).toString())); writer.write(folderContentUnit.getContent()); } @@ -361,10 +364,10 @@ public byte[] compressFolder(List folderContentUnits, Stri * @param workspaceUnitKey given user workspace unit key. * @param type given content type. * @param name given content name. - * @param inputStream given input. + * @param content given content. * @throws FileCreationFailureException if file creation operation failed. */ - public void addContentFile(String workspaceUnitKey, String type, String name, InputStream inputStream) + public void addContentFile(String workspaceUnitKey, String type, String name, byte[] content) throws FileCreationFailureException { if (!isUnitDirectoryExist(workspaceUnitKey)) { try { @@ -394,14 +397,6 @@ public void addContentFile(String workspaceUnitKey, String type, String name, In throw new FileCreationFailureException(); } - byte[] content; - - try { - content = compressFile(inputStream); - } catch (InputCompressionFailureException e) { - throw new FileCreationFailureException(e.getMessage()); - } - try { createFile(workspaceUnitDirectory, type, name, content); } catch (FileWriteFailureException e) { @@ -435,6 +430,39 @@ public Boolean isContentFilePresent(String workspaceUnitKey, String type, String return false; } + /** + * Retrieves content units in the workspace with the given workspace unit key and of the given type. + * + * @param workspaceUnitKey given user workspace unit key. + * @param type given file type. + * @return retrieves additional content units. + * @throws FileUnitsRetrievalFailureException if content units retrieval failed. + */ + public List getContentUnits(String workspaceUnitKey, String type) throws + FileUnitsRetrievalFailureException { + List result = new ArrayList<>(); + + if (isUnitDirectoryExist(workspaceUnitKey)) { + String workspaceUnitDirectory; + + try { + workspaceUnitDirectory = getUnitDirectory(workspaceUnitKey); + } catch (WorkspaceUnitDirectoryNotFoundException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + + if (isContentDirectoryExist(workspaceUnitDirectory, type)) { + try { + result = getFilesLocations(workspaceUnitDirectory, type); + } catch (FilesLocationsRetrievalFailureException e) { + throw new FileUnitsRetrievalFailureException(e.getMessage()); + } + } + } + + return result; + } + /** * Retrieves file from the workspace with the given workspace unit key as compressed byte array. * diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java index 41cc530..f95e0de 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/common/WorkspaceConfigurationHelper.java @@ -14,4 +14,14 @@ public class WorkspaceConfigurationHelper { public static String getZipFolderDefinition(String name) { return String.format("%s/", name); } + + /** + * Creates zip file definition with the help of the given file name. + * + * @param name given zip file name. + * @return wrapped zip file. + */ + public static String getZipFile(String name) { + return String.format("%s.zip", name); + } } \ No newline at end of file diff --git a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java index 48a8759..6d90ec5 100644 --- a/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java +++ b/api-server/src/main/java/com/objectstorage/service/workspace/facade/WorkspaceFacade.java @@ -3,6 +3,7 @@ import com.objectstorage.dto.FolderContentUnitDto; import com.objectstorage.entity.common.PropertiesEntity; import com.objectstorage.exception.*; +import com.objectstorage.model.ContentRetrievalBackupUnit; import com.objectstorage.model.ValidationSecretsApplication; import com.objectstorage.service.config.ConfigService; import com.objectstorage.service.workspace.WorkspaceService; @@ -75,7 +76,19 @@ public String createFileUnitKey(String name) { */ public void addObjectFile(String workspaceUnitKey, String name, InputStream inputStream) throws FileCreationFailureException { - workspaceService.addContentFile(workspaceUnitKey, properties.getWorkspaceContentObjectDirectory(), name, inputStream); + byte[] content; + + try { + content = workspaceService.compressFile(inputStream); + } catch (InputCompressionFailureException e) { + throw new FileCreationFailureException(e.getMessage()); + } + + workspaceService.addContentFile( + workspaceUnitKey, + properties.getWorkspaceContentObjectDirectory(), + name, + content); } /** @@ -101,7 +114,7 @@ public void addBackupFile(String workspaceUnitKey, String name, List getBackupUnits(String workspaceUnitKey) throws FileUnitsRetrievalFailureException { + return workspaceService + .getContentUnits(workspaceUnitKey, properties.getWorkspaceContentBackupDirectory()) + .stream() + .map(ContentRetrievalBackupUnit::of) + .toList(); + } + /** * Retrieves object file with the given name and of the given type from the workspace with the given workspace * unit key as compressed byte array. diff --git a/api-server/src/main/openapi/openapi.yml b/api-server/src/main/openapi/openapi.yml index 15a7eb9..f6092e3 100644 --- a/api-server/src/main/openapi/openapi.yml +++ b/api-server/src/main/openapi/openapi.yml @@ -274,6 +274,10 @@ components: type: array items: $ref: "#/components/schemas/ContentRetrievalProviderUnit" + backups: + type: array + items: + $ref: "#/components/schemas/ContentRetrievalBackupUnit" ContentRetrievalProviderUnit: required: - location @@ -284,6 +288,12 @@ components: created_at: type: integer format: int64 + ContentRetrievalBackupUnit: + required: + - location + properties: + location: + type: string ContentApplication: required: - root diff --git a/config/grafana/dashboards/diagnostics.tmpl b/config/grafana/dashboards/diagnostics.tmpl index 44d81a0..a9e7da5 100644 --- a/config/grafana/dashboards/diagnostics.tmpl +++ b/config/grafana/dashboards/diagnostics.tmpl @@ -440,6 +440,108 @@ "title": "ObjectStorage API Server average upload file size", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "P21B111CBFE6E8FCA" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 38, + "interval": "0.1", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "P21B111CBFE6E8FCA" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "general_current_backups_amount", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "ObjectStorage API Server current backups amount", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", @@ -466,7 +568,7 @@ "gridPos": { "h": 8, "w": 12, - "x": 0, + "x": 12, "y": 17 }, "id": 32, @@ -538,8 +640,8 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 17 + "x": 0, + "y": 25 }, "id": 35, "options": { @@ -587,7 +689,7 @@ "h": 1, "w": 24, "x": 0, - "y": 25 + "y": 33 }, "id": 28, "panels": [], @@ -636,7 +738,7 @@ "h": 7, "w": 3, "x": 0, - "y": 26 + "y": 34 }, "id": 15, "maxDataPoints": 100, @@ -721,7 +823,7 @@ "h": 6, "w": 6, "x": 3, - "y": 26 + "y": 34 }, "id": 6, "maxDataPoints": 100, @@ -807,7 +909,7 @@ "h": 6, "w": 6, "x": 9, - "y": 26 + "y": 34 }, "id": 4, "maxDataPoints": 100, @@ -845,7 +947,7 @@ "type": "gauge" } ], - "refresh": "5s", + "refresh": "", "schemaVersion": 39, "tags": [ "docker", @@ -1021,8 +1123,8 @@ ] }, "time": { - "from": "now-5m", - "to": "now" + "from": "2024-11-29T14:23:10.911Z", + "to": "2024-11-29T14:25:03.115Z" }, "timepicker": { "refresh_intervals": [ @@ -1052,6 +1154,6 @@ "timezone": "browser", "title": "ObjectStorage Diagnostics", "uid": "64nrElFmk", - "version": 7, + "version": 10, "weekStart": "" } \ No newline at end of file