From 32b8767ed52a2578ea032db7084adf4bebb19d6e Mon Sep 17 00:00:00 2001 From: aktoboy Date: Tue, 20 Feb 2024 14:16:58 +0530 Subject: [PATCH 01/11] Added changes to support loading templates without internet --- .../akto/listener/InitializerListener.java | 75 +++++++------------ .../com/akto/utils/GithubAccountTask.java | 59 +++++++++++++++ .../java/com/akto/dto/ByteArrayWrapper.java | 14 ++++ 3 files changed, 99 insertions(+), 49 deletions(-) create mode 100644 apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java create mode 100644 libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 1d53f483b7..9abcc957d8 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -70,11 +70,8 @@ import com.akto.util.enums.GlobalEnums.TestCategory; import com.akto.util.enums.GlobalEnums.YamlTemplateSource; import com.akto.util.tasks.OrganizationTask; -import com.akto.utils.Auth0; +import com.akto.utils.*; import com.akto.util.DashboardMode; -import com.akto.utils.GithubSync; -import com.akto.utils.HttpUtils; -import com.akto.utils.RedactSampleData; import com.akto.utils.crons.SyncCron; import com.akto.utils.crons.UpdateSensitiveInfoInApiInfo; import com.akto.utils.billing.OrganizationUtils; @@ -1190,17 +1187,17 @@ public static void addAktoDataTypes(BackwardCompatibility backwardCompatibility) ); } } - public static void loadTemplateFilesFromDirectory(BackwardCompatibility backwardCompatibility) { - if (backwardCompatibility.getLoadTemplateFilesFromDirectory() == 0) { - String resourceName = "/tests-library-master.zip"; + public static byte[] loadTemplateFilesFromDirectory() { + String resourceName = "/tests-library-master.zip"; - loggerMaker.infoAndAddToDb("Loading template files from directory", LogDb.DASHBOARD); + loggerMaker.infoAndAddToDb("Loading template files from directory", LogDb.DASHBOARD); - try (InputStream is = InitializerListener.class.getResourceAsStream(resourceName); - ByteArrayOutputStream baos = new ByteArrayOutputStream()) { + try (InputStream is = InitializerListener.class.getResourceAsStream(resourceName); + ByteArrayOutputStream baos = new ByteArrayOutputStream()) { if (is == null) { loggerMaker.errorAndAddToDb("Resource not found: " + resourceName, LogDb.DASHBOARD); + return null; } else { // Read the contents of the .zip file into a byte array byte[] buffer = new byte[1024]; @@ -1209,17 +1206,12 @@ public static void loadTemplateFilesFromDirectory(BackwardCompatibility backward baos.write(buffer, 0, bytesRead); } - processTemplateFilesZip(baos.toByteArray(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); - } - } catch (Exception ex) { - loggerMaker.errorAndAddToDb(ex, String.format("Error while loading templates files from directory. Error: %s", ex.getMessage()), LogDb.DASHBOARD); + return baos.toByteArray(); } - - BackwardCompatibilityDao.instance.updateOne( - Filters.eq("_id", backwardCompatibility.getId()), - Updates.set(BackwardCompatibility.LOAD_TEMPLATES_FILES_FROM_DIRECTORY, Context.now()) - ); + } catch (Exception ex) { + loggerMaker.errorAndAddToDb(ex, String.format("Error while loading templates files from directory. Error: %s", ex.getMessage()), LogDb.DASHBOARD); } + return null; } public static void setAktoDefaultNewUI(BackwardCompatibility backwardCompatibility){ @@ -1759,7 +1751,6 @@ public static void setBackwardCompatibilities(BackwardCompatibility backwardComp deleteAccessListFromApiToken(backwardCompatibility); deleteNullSubCategoryIssues(backwardCompatibility); enableNewMerging(backwardCompatibility); - loadTemplateFilesFromDirectory(backwardCompatibility); if (DashboardMode.isMetered()) { initializeOrganizationAccountBelongsTo(backwardCompatibility); } @@ -1865,34 +1856,20 @@ private static String getUpdateDeploymentStatusUrl() { public final static String _AKTO = "AKTO"; public void setUpTestEditorTemplatesScheduler() { - GithubSync githubSync = new GithubSync(); - byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); - - if (repoZip != null) { - scheduler.scheduleAtFixedRate(new Runnable() { - public void run() { - AccountTask.instance.executeTask(new Consumer() { - @Override - public void accept(Account t) { - try { - int accountId = t.getId(); - loggerMaker.infoAndAddToDb( - String.format("Updating Akto test templates for account: %d", accountId), - LogDb.DASHBOARD); - processTemplateFilesZip(repoZip, _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); - } catch (Exception e) { - cacheLoggerMaker.errorAndAddToDb(e, - String.format("Error while updating Test Editor Files %s", e.toString()), - LogDb.DASHBOARD); - } - } - }, "update-test-editor-templates-github"); - } - }, 0, 4, TimeUnit.HOURS); - } else { - loggerMaker.errorAndAddToDb("Unable to update test templates - test templates zip could not be downloaded", LogDb.DASHBOARD); - } - + scheduler.scheduleAtFixedRate(new Runnable() { + public void run() { + GithubAccountTask.instance.executeTask((consumer) -> { + try { + loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + consumer.getFirst(), LogDb.DASHBOARD); + processTemplateFilesZip(consumer.getSecond().getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + } catch (Exception e) { + cacheLoggerMaker.errorAndAddToDb(e, + String.format("Error while updating Test Editor Files %s", e.toString()), + LogDb.DASHBOARD); + } + }, "update-test-editor-templates-github"); + } + }, 0, 4, TimeUnit.HOURS); } public static void processTemplateFilesZip(byte[] zipFile, String author, String source, String repositoryUrl) { @@ -1967,7 +1944,7 @@ public static void processTemplateFilesZip(byte[] zipFile, String author, String Updates.set(YamlTemplate.HASH, templateContent.hashCode()), Updates.set(YamlTemplate.CONTENT, templateContent), Updates.set(YamlTemplate.INFO, testConfig.getInfo()))); - + try { Object inactiveObject = TestConfigYamlParser.getFieldIfExists(templateContent, YamlTemplate.INACTIVE); diff --git a/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java b/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java new file mode 100644 index 0000000000..b37a751cbe --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java @@ -0,0 +1,59 @@ +package com.akto.utils; + +import com.akto.dao.AccountsDao; +import com.akto.dao.context.Context; +import com.akto.dto.Account; +import com.akto.dto.ByteArrayWrapper; +import com.akto.util.AccountTask; +import com.akto.util.Pair; +import com.mongodb.client.model.Filters; +import org.bson.conversions.Bson; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.function.Consumer; + +import static com.akto.listener.InitializerListener.loadTemplateFilesFromDirectory; + +public class GithubAccountTask { + + private static final Logger logger = LoggerFactory.getLogger(AccountTask.class); + public static final GithubAccountTask instance = new GithubAccountTask(); + + public void executeTask(Consumer> consumeAccount, String taskName) { + + Bson activeFilter = Filters.or( + Filters.exists(Account.INACTIVE_STR, false), + Filters.eq(Account.INACTIVE_STR, false) + ); + + GithubSync githubSync = new GithubSync(); + byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); + if(repoZip == null) { + logger.info("Failed to load test templates from github, trying to load from local directory"); + repoZip = loadTemplateFilesFromDirectory(); + if(repoZip == null) { + logger.error("Failed to load test templates from github or local directory"); + return; + } else { + logger.info("Loaded test templates from local directory"); + } + } else { + logger.info("Loaded test templates from github"); + } + + ByteArrayWrapper baw = new ByteArrayWrapper(repoZip); + + List activeAccounts = AccountsDao.instance.findAll(activeFilter); + for(Account account: activeAccounts) { + try { + Context.accountId.set(account.getId()); + consumeAccount.accept(new Pair<>(account, baw)); + } catch (Exception e) { + String msgString = String.format("Error in executing task %s for account %d", taskName, account.getId()); + logger.error(msgString, e); + } + } + } +} diff --git a/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java b/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java new file mode 100644 index 0000000000..3209c13ad4 --- /dev/null +++ b/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java @@ -0,0 +1,14 @@ +package com.akto.dto; + +public class ByteArrayWrapper { + + private byte[] data; + + public ByteArrayWrapper(byte[] data) { + this.data = data; + } + + public byte[] getData() { + return data; + } +} From 6ac372b48385277a471730bf9be5c73b1a64efb9 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Tue, 20 Feb 2024 16:39:13 +0530 Subject: [PATCH 02/11] Changed other usage of fetching templates from github + improved logging --- .../java/com/akto/action/AccountAction.java | 10 +++-- .../akto/listener/InitializerListener.java | 6 +-- .../com/akto/utils/GithubAccountTask.java | 43 +++++++++++-------- .../main/java/com/akto/log/LoggerMaker.java | 5 +++ 4 files changed, 41 insertions(+), 23 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java index fb489f5cae..4afe41b9ed 100644 --- a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java @@ -12,6 +12,7 @@ import com.akto.runtime.Main; import com.akto.util.enums.GlobalEnums.YamlTemplateSource; import com.akto.util.DashboardMode; +import com.akto.utils.GithubAccountTask; import com.akto.utils.GithubSync; import com.akto.utils.billing.OrganizationUtils; import com.akto.utils.cloud.Utils; @@ -313,10 +314,13 @@ public void run() { } try { - GithubSync githubSync = new GithubSync(); - byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); + ByteArrayWrapper testingTemplates = GithubAccountTask.getTestingTemplates(); + if(testingTemplates == null){ + loggerMaker.errorAndAddToDb("Failed to load test templates", LogDb.DASHBOARD); + return; + } loggerMaker.infoAndAddToDb(String.format("Updating akto test templates for new account: %d", newAccountId), LogDb.DASHBOARD); - InitializerListener.processTemplateFilesZip(repoZip, InitializerListener._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + InitializerListener.processTemplateFilesZip(testingTemplates.getData(), InitializerListener._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); } catch (Exception e) { loggerMaker.errorAndAddToDb(e,String.format("Error while adding test editor templates for new account %d, Error: %s", newAccountId, e.getMessage()), LogDb.DASHBOARD); } diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 9abcc957d8..e293a8bad8 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1858,10 +1858,10 @@ private static String getUpdateDeploymentStatusUrl() { public void setUpTestEditorTemplatesScheduler() { scheduler.scheduleAtFixedRate(new Runnable() { public void run() { - GithubAccountTask.instance.executeTask((consumer) -> { + GithubAccountTask.instance.executeTask((data) -> { try { - loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + consumer.getFirst(), LogDb.DASHBOARD); - processTemplateFilesZip(consumer.getSecond().getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + data.getFirst(), LogDb.DASHBOARD); + processTemplateFilesZip(data.getSecond().getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); } catch (Exception e) { cacheLoggerMaker.errorAndAddToDb(e, String.format("Error while updating Test Editor Files %s", e.toString()), diff --git a/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java b/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java index b37a751cbe..47e2a7dc78 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java +++ b/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java @@ -4,6 +4,7 @@ import com.akto.dao.context.Context; import com.akto.dto.Account; import com.akto.dto.ByteArrayWrapper; +import com.akto.log.LoggerMaker; import com.akto.util.AccountTask; import com.akto.util.Pair; import com.mongodb.client.model.Filters; @@ -18,7 +19,7 @@ public class GithubAccountTask { - private static final Logger logger = LoggerFactory.getLogger(AccountTask.class); + private static final LoggerMaker loggerMaker = new LoggerMaker(GithubAccountTask.class, LoggerMaker.LogDb.DASHBOARD); public static final GithubAccountTask instance = new GithubAccountTask(); public void executeTask(Consumer> consumeAccount, String taskName) { @@ -28,23 +29,12 @@ public void executeTask(Consumer> consumeAccount Filters.eq(Account.INACTIVE_STR, false) ); - GithubSync githubSync = new GithubSync(); - byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); - if(repoZip == null) { - logger.info("Failed to load test templates from github, trying to load from local directory"); - repoZip = loadTemplateFilesFromDirectory(); - if(repoZip == null) { - logger.error("Failed to load test templates from github or local directory"); - return; - } else { - logger.info("Loaded test templates from local directory"); - } - } else { - logger.info("Loaded test templates from github"); + ByteArrayWrapper baw = getTestingTemplates(); + if (baw == null) { + loggerMaker.errorAndAddToDb("Failed to load test templates"); + return; } - ByteArrayWrapper baw = new ByteArrayWrapper(repoZip); - List activeAccounts = AccountsDao.instance.findAll(activeFilter); for(Account account: activeAccounts) { try { @@ -52,8 +42,27 @@ public void executeTask(Consumer> consumeAccount consumeAccount.accept(new Pair<>(account, baw)); } catch (Exception e) { String msgString = String.format("Error in executing task %s for account %d", taskName, account.getId()); - logger.error(msgString, e); + loggerMaker.errorAndAddToDb(e, msgString); } } } + + public static ByteArrayWrapper getTestingTemplates() { + GithubSync githubSync = new GithubSync(); + byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); + if(repoZip == null) { + loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory"); + repoZip = loadTemplateFilesFromDirectory(); + if(repoZip == null) { + loggerMaker.errorAndAddToDb("Failed to load test templates from github or local directory"); + return null; + } else { + loggerMaker.infoAndAddToDb("Loaded test templates from local directory"); + } + } else { + loggerMaker.infoAndAddToDb("Loaded test templates from github"); + } + return new ByteArrayWrapper(repoZip); + } + } diff --git a/libs/utils/src/main/java/com/akto/log/LoggerMaker.java b/libs/utils/src/main/java/com/akto/log/LoggerMaker.java index 8f2f0ca6e4..2eb1850b83 100644 --- a/libs/utils/src/main/java/com/akto/log/LoggerMaker.java +++ b/libs/utils/src/main/java/com/akto/log/LoggerMaker.java @@ -64,6 +64,7 @@ public enum LogDb { TESTING,RUNTIME,DASHBOARD,BILLING } + @Deprecated public LoggerMaker(Class c) { aClass = c; logger = LoggerFactory.getLogger(c); @@ -117,6 +118,10 @@ public void errorAndAddToDb(String err, LogDb db) { } } + public void errorAndAddToDb(Exception e, String err) { + errorAndAddToDb(e, err, this.db); + } + public void errorAndAddToDb(Exception e, String err, LogDb db) { try { if (e != null && e.getStackTrace() != null && e.getStackTrace().length > 0) { From 702737b9e67d46fbadf23d796ae365761c52aeeb Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 14:56:47 +0530 Subject: [PATCH 03/11] Added changes to remove dependency from github --- .github/workflows/prod.yml | 9 +- .../akto/listener/InitializerListener.java | 166 +++++++++++------- 2 files changed, 110 insertions(+), 65 deletions(-) diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 18cac5828f..7109a8be90 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -28,9 +28,14 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '17' - - name: Download Akto templates zip + - name: Download Akto templates zip and PII files working-directory: ./apps/dashboard/src/main/resources - run: wget -O test-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip + run: | + wget -O test-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip + wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json + wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json + wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json + - name: Prepare Dashboard UI working-directory: ./apps/dashboard/ run: npm install && export RELEASE_VERSION=${{github.event.inputs.release_version}} && npm run build diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index e293a8bad8..10a2cfb323 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -137,6 +137,8 @@ public class InitializerListener implements ServletContextListener { private static String domain = null; public static String subdomain = "https://app.akto.io"; + + private static Map piiFileMap; Crons crons = new Crons(); public static String getDomain() { @@ -447,89 +449,120 @@ private static boolean isSimilar(String param, List commonPayloads) { return false; } + private static String loadPIIFileFromResources(String fileUrl){ + String fileName = piiFileMap.get(fileUrl); + if(fileName == null){ + loggerMaker.errorAndAddToDb("Unable to find file locally: " + fileUrl, LogDb.DASHBOARD); + return null; + } + try { + return convertStreamToString(InitializerListener.class.getResourceAsStream("/" + fileName)); + } catch (Exception e) { + loggerMaker.errorAndAddToDb(e, "Exception while reading content locally: " + fileUrl, LogDb.DASHBOARD); + return null; + } + } + + private static String fetchPIIFile(PIISource piiSource){ + String fileUrl = piiSource.getFileUrl(); + String id = piiSource.getId(); + if (fileUrl.startsWith("http")) { + String tempFileUrl = "temp_" + id; + if(downloadFileCheck(tempFileUrl)){ + try { + FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); + } catch (IOException e) { + loggerMaker.errorAndAddToDb(e, String.format("failed to read file %s", piiSource.getFileUrl()), LogDb.DASHBOARD); + return loadPIIFileFromResources(piiSource.getFileUrl()); + } + } + fileUrl = tempFileUrl; + try { + return FileUtils.readFileToString(new File(fileUrl), StandardCharsets.UTF_8); + } catch (IOException e){ + loggerMaker.errorAndAddToDb(e, String.format("failed to read file %s", piiSource.getFileUrl()), LogDb.DASHBOARD); + return loadPIIFileFromResources(piiSource.getFileUrl()); + } + } else { + return loadPIIFileFromResources(piiSource.getFileUrl()); + } + } + public static void executePIISourceFetch() { List piiSources = PIISourceDao.instance.findAll("active", true); for (PIISource piiSource : piiSources) { - String fileUrl = piiSource.getFileUrl(); String id = piiSource.getId(); Map currTypes = piiSource.getMapNameToPIIType(); if (currTypes == null) { currTypes = new HashMap<>(); } - try { - if (fileUrl.startsWith("http")) { - String tempFileUrl = "temp_" + id; - if(downloadFileCheck(tempFileUrl)){ - FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); - } - fileUrl = tempFileUrl; - } - String fileContent = FileUtils.readFileToString(new File(fileUrl), StandardCharsets.UTF_8); - BasicDBObject fileObj = BasicDBObject.parse(fileContent); - BasicDBList dataTypes = (BasicDBList) (fileObj.get("types")); - Bson findQ = Filters.eq("_id", id); - - List customDataTypes = CustomDataTypeDao.instance.findAll(new BasicDBObject()); - Map customDataTypesMap = new HashMap<>(); - for(CustomDataType customDataType : customDataTypes){ - customDataTypesMap.put(customDataType.getName(), customDataType); - } + String fileContent = fetchPIIFile(piiSource); + if (fileContent == null) { + loggerMaker.errorAndAddToDb("Failed to load file from github as well as resources: " + piiSource.getFileUrl(), LogDb.DASHBOARD); + continue; + } + BasicDBObject fileObj = BasicDBObject.parse(fileContent); + BasicDBList dataTypes = (BasicDBList) (fileObj.get("types")); + Bson findQ = Filters.eq("_id", id); + + List customDataTypes = CustomDataTypeDao.instance.findAll(new BasicDBObject()); + Map customDataTypesMap = new HashMap<>(); + for (CustomDataType customDataType : customDataTypes) { + customDataTypesMap.put(customDataType.getName(), customDataType); + } - List piiUpdates = new ArrayList<>(); + List piiUpdates = new ArrayList<>(); - for (Object dtObj : dataTypes) { - BasicDBObject dt = (BasicDBObject) dtObj; - String piiKey = dt.getString("name").toUpperCase(); - PIIType piiType = new PIIType( - piiKey, - dt.getBoolean("sensitive"), - dt.getString("regexPattern"), - dt.getBoolean("onKey") - ); + for (Object dtObj : dataTypes) { + BasicDBObject dt = (BasicDBObject) dtObj; + String piiKey = dt.getString("name").toUpperCase(); + PIIType piiType = new PIIType( + piiKey, + dt.getBoolean("sensitive"), + dt.getString("regexPattern"), + dt.getBoolean("onKey") + ); - CustomDataType existingCDT = customDataTypesMap.getOrDefault(piiKey, null); - CustomDataType newCDT = getCustomDataTypeFromPiiType(piiSource, piiType, false); + CustomDataType existingCDT = customDataTypesMap.getOrDefault(piiKey, null); + CustomDataType newCDT = getCustomDataTypeFromPiiType(piiSource, piiType, false); - if (currTypes.containsKey(piiKey) && - (currTypes.get(piiKey).equals(piiType) && - dt.getBoolean(PIISource.ACTIVE, true))) { - continue; + if (currTypes.containsKey(piiKey) && + (currTypes.get(piiKey).equals(piiType) && + dt.getBoolean(PIISource.ACTIVE, true))) { + continue; + } else { + if (!dt.getBoolean(PIISource.ACTIVE, true)) { + if (currTypes.getOrDefault(piiKey, null) != null || piiSource.getLastSynced() == 0) { + piiUpdates.add(Updates.unset(PIISource.MAP_NAME_TO_PII_TYPE + "." + piiKey)); + } } else { - if (!dt.getBoolean(PIISource.ACTIVE, true)) { - if (currTypes.getOrDefault(piiKey, null) != null || piiSource.getLastSynced() == 0) { - piiUpdates.add(Updates.unset(PIISource.MAP_NAME_TO_PII_TYPE + "." + piiKey)); - } - } else { - if (currTypes.getOrDefault(piiKey, null) != piiType || piiSource.getLastSynced() == 0) { - piiUpdates.add(Updates.set(PIISource.MAP_NAME_TO_PII_TYPE + "." + piiKey, piiType)); - } - newCDT.setActive(true); + if (currTypes.getOrDefault(piiKey, null) != piiType || piiSource.getLastSynced() == 0) { + piiUpdates.add(Updates.set(PIISource.MAP_NAME_TO_PII_TYPE + "." + piiKey, piiType)); } + newCDT.setActive(true); + } - if (existingCDT == null) { - CustomDataTypeDao.instance.insertOne(newCDT); - } else { - List updates = getCustomDataTypeUpdates(existingCDT, newCDT); - if (!updates.isEmpty()) { - CustomDataTypeDao.instance.updateOne( + if (existingCDT == null) { + CustomDataTypeDao.instance.insertOne(newCDT); + } else { + List updates = getCustomDataTypeUpdates(existingCDT, newCDT); + if (!updates.isEmpty()) { + CustomDataTypeDao.instance.updateOne( Filters.eq(CustomDataType.NAME, piiKey), Updates.combine(updates) - ); - } + ); } } - } - if(!piiUpdates.isEmpty()){ - piiUpdates.add(Updates.set(PIISource.LAST_SYNCED, Context.now())); - PIISourceDao.instance.updateOne(findQ, Updates.combine(piiUpdates)); - } + } - } catch (IOException e) { - loggerMaker.errorAndAddToDb(e, String.format("failed to read file %s", e.toString()), LogDb.DASHBOARD); + if (!piiUpdates.isEmpty()) { + piiUpdates.add(Updates.set(PIISource.LAST_SYNCED, Context.now())); + PIISourceDao.instance.updateOne(findQ, Updates.combine(piiUpdates)); } + } } @@ -1609,27 +1642,34 @@ private void updateGlobalAktoVersion() throws Exception{ } public static void insertPiiSources(){ + Map map = new HashMap<>(); + String fileUrl = "https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json"; + map.put(fileUrl, "general.json"); if (PIISourceDao.instance.findOne("_id", "A") == null) { - String fileUrl = "https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json"; PIISource piiSource = new PIISource(fileUrl, 0, 1638571050, 0, new HashMap<>(), true); piiSource.setId("A"); - PIISourceDao.instance.insertOne(piiSource); } + fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json"; + map.put(fileUrl, "fintech.json"); if (PIISourceDao.instance.findOne("_id", "Fin") == null) { - String fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json"; PIISource piiSource = new PIISource(fileUrl, 0, 1638571050, 0, new HashMap<>(), true); piiSource.setId("Fin"); PIISourceDao.instance.insertOne(piiSource); } + fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json"; + map.put(fileUrl, "filetypes.json"); if (PIISourceDao.instance.findOne("_id", "File") == null) { - String fileUrl = "https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json"; PIISource piiSource = new PIISource(fileUrl, 0, 1638571050, 0, new HashMap<>(), true); piiSource.setId("File"); PIISourceDao.instance.insertOne(piiSource); } + + if(piiFileMap == null){ + piiFileMap = Collections.unmodifiableMap(map); + } } static boolean executedOnce = false; From f4e4acfd77f0a51d990a8bbc9cf0cde63acda546 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 16:32:02 +0530 Subject: [PATCH 04/11] Removed unused attributes from Backward compatibility --- .../java/com/akto/dto/BackwardCompatibility.java | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java index 442852148f..b45f6412ba 100644 --- a/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java +++ b/libs/dao/src/main/java/com/akto/dto/BackwardCompatibility.java @@ -39,7 +39,6 @@ public class BackwardCompatibility { private int enableNewMerging; public static final String LOAD_TEMPLATES_FILES_FROM_DIRECTORY = "loadTemplateFilesFromDirectory"; - private int loadTemplateFilesFromDirectory; public static final String DEFAULT_NEW_UI = "aktoDefaultNewUI"; private int aktoDefaultNewUI; @@ -58,7 +57,7 @@ public class BackwardCompatibility { public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTypeInfoCount, int dropWorkflowTestResult, int readyForNewTestingFramework,int addAktoDataTypes, boolean deploymentStatusUpdated, int authMechanismData, boolean mirroringLambdaTriggered, int deleteAccessListFromApiToken, - int deleteNullSubCategoryIssues, int enableNewMerging, int loadTemplateFilesFromDirectory, + int deleteNullSubCategoryIssues, int enableNewMerging, int aktoDefaultNewUI, int initializeOrganizationAccountBelongsTo, int orgsInBilling, int computeIntegratedConnections, int deleteLastCronRunInfo) { this.id = id; @@ -73,7 +72,6 @@ public BackwardCompatibility(int id, int dropFilterSampleData, int resetSingleTy this.deleteAccessListFromApiToken = deleteAccessListFromApiToken; this.deleteNullSubCategoryIssues = deleteNullSubCategoryIssues; this.enableNewMerging = enableNewMerging; - this.loadTemplateFilesFromDirectory = loadTemplateFilesFromDirectory; this.aktoDefaultNewUI = aktoDefaultNewUI; this.computeIntegratedConnections = computeIntegratedConnections; this.initializeOrganizationAccountBelongsTo = initializeOrganizationAccountBelongsTo; @@ -188,14 +186,6 @@ public void setEnableNewMerging(int enableNewMerging) { this.enableNewMerging = enableNewMerging; } - public int getLoadTemplateFilesFromDirectory() { - return loadTemplateFilesFromDirectory; - } - - public int setLoadTemplateFilesFromDirectory(int loadTemplateFilesFromDirectory) { - return this.loadTemplateFilesFromDirectory = loadTemplateFilesFromDirectory; - } - public int getAktoDefaultNewUI() { return aktoDefaultNewUI; } From 7d88e28b6d4bcac5899c374fe273ebc4c640e957 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 16:40:44 +0530 Subject: [PATCH 05/11] Removed GithubAccountTask --- .../java/com/akto/action/AccountAction.java | 7 +- .../akto/listener/InitializerListener.java | 15 ++-- .../com/akto/utils/GithubAccountTask.java | 68 ------------------- .../com/akto/utils/TestTemplateUtils.java | 30 ++++++++ 4 files changed, 40 insertions(+), 80 deletions(-) delete mode 100644 apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java create mode 100644 apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java diff --git a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java index 4afe41b9ed..6b2d7f88aa 100644 --- a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java @@ -12,8 +12,7 @@ import com.akto.runtime.Main; import com.akto.util.enums.GlobalEnums.YamlTemplateSource; import com.akto.util.DashboardMode; -import com.akto.utils.GithubAccountTask; -import com.akto.utils.GithubSync; +import com.akto.utils.TestTemplateUtils; import com.akto.utils.billing.OrganizationUtils; import com.akto.utils.cloud.Utils; import com.akto.utils.cloud.serverless.aws.Lambda; @@ -21,8 +20,6 @@ import com.akto.utils.cloud.stack.dto.StackState; import com.akto.utils.platform.DashboardStackDetails; import com.akto.utils.platform.MirroringStackDetails; -import com.amazonaws.services.autoscaling.AmazonAutoScaling; -import com.amazonaws.services.autoscaling.AmazonAutoScalingClientBuilder; import com.amazonaws.services.autoscaling.model.RefreshPreferences; import com.amazonaws.services.autoscaling.model.StartInstanceRefreshRequest; import com.amazonaws.services.autoscaling.model.StartInstanceRefreshResult; @@ -314,7 +311,7 @@ public void run() { } try { - ByteArrayWrapper testingTemplates = GithubAccountTask.getTestingTemplates(); + ByteArrayWrapper testingTemplates = TestTemplateUtils.getTestingTemplates(); if(testingTemplates == null){ loggerMaker.errorAndAddToDb("Failed to load test templates", LogDb.DASHBOARD); return; diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 10a2cfb323..8d17dad935 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -7,7 +7,6 @@ import com.akto.action.observe.InventoryAction; import com.akto.action.testing.StartTestAction; import com.akto.dao.*; -import com.akto.dao.billing.OrganizationUsageDao; import com.akto.dao.billing.OrganizationsDao; import com.akto.dao.context.Context; import com.akto.dao.loaders.LoadersDao; @@ -43,7 +42,6 @@ import com.akto.dto.test_editor.TestConfig; import com.akto.dto.test_editor.YamlTemplate; import com.akto.dto.traffic.Key; -import com.akto.dto.testing.DeleteTestRuns; import com.akto.dto.traffic.SampleData; import com.akto.dto.type.SingleTypeInfo; import com.akto.dto.usage.MetricTypes; @@ -91,12 +89,10 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.bson.conversions.Bson; -import org.bson.types.ObjectId; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.print.attribute.standard.Severity; import javax.servlet.ServletContextListener; import java.io.*; import java.net.URI; @@ -1898,10 +1894,15 @@ private static String getUpdateDeploymentStatusUrl() { public void setUpTestEditorTemplatesScheduler() { scheduler.scheduleAtFixedRate(new Runnable() { public void run() { - GithubAccountTask.instance.executeTask((data) -> { + ByteArrayWrapper testingTemplates = TestTemplateUtils.getTestingTemplates(); + if(testingTemplates == null){ + loggerMaker.errorAndAddToDb("Error while fetching Test Editor Templates from Github and local", LogDb.DASHBOARD); + return; + } + AccountTask.instance.executeTask((account) -> { try { - loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + data.getFirst(), LogDb.DASHBOARD); - processTemplateFilesZip(data.getSecond().getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + account.getId(), LogDb.DASHBOARD); + processTemplateFilesZip(testingTemplates.getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); } catch (Exception e) { cacheLoggerMaker.errorAndAddToDb(e, String.format("Error while updating Test Editor Files %s", e.toString()), diff --git a/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java b/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java deleted file mode 100644 index 47e2a7dc78..0000000000 --- a/apps/dashboard/src/main/java/com/akto/utils/GithubAccountTask.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.akto.utils; - -import com.akto.dao.AccountsDao; -import com.akto.dao.context.Context; -import com.akto.dto.Account; -import com.akto.dto.ByteArrayWrapper; -import com.akto.log.LoggerMaker; -import com.akto.util.AccountTask; -import com.akto.util.Pair; -import com.mongodb.client.model.Filters; -import org.bson.conversions.Bson; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.List; -import java.util.function.Consumer; - -import static com.akto.listener.InitializerListener.loadTemplateFilesFromDirectory; - -public class GithubAccountTask { - - private static final LoggerMaker loggerMaker = new LoggerMaker(GithubAccountTask.class, LoggerMaker.LogDb.DASHBOARD); - public static final GithubAccountTask instance = new GithubAccountTask(); - - public void executeTask(Consumer> consumeAccount, String taskName) { - - Bson activeFilter = Filters.or( - Filters.exists(Account.INACTIVE_STR, false), - Filters.eq(Account.INACTIVE_STR, false) - ); - - ByteArrayWrapper baw = getTestingTemplates(); - if (baw == null) { - loggerMaker.errorAndAddToDb("Failed to load test templates"); - return; - } - - List activeAccounts = AccountsDao.instance.findAll(activeFilter); - for(Account account: activeAccounts) { - try { - Context.accountId.set(account.getId()); - consumeAccount.accept(new Pair<>(account, baw)); - } catch (Exception e) { - String msgString = String.format("Error in executing task %s for account %d", taskName, account.getId()); - loggerMaker.errorAndAddToDb(e, msgString); - } - } - } - - public static ByteArrayWrapper getTestingTemplates() { - GithubSync githubSync = new GithubSync(); - byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); - if(repoZip == null) { - loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory"); - repoZip = loadTemplateFilesFromDirectory(); - if(repoZip == null) { - loggerMaker.errorAndAddToDb("Failed to load test templates from github or local directory"); - return null; - } else { - loggerMaker.infoAndAddToDb("Loaded test templates from local directory"); - } - } else { - loggerMaker.infoAndAddToDb("Loaded test templates from github"); - } - return new ByteArrayWrapper(repoZip); - } - -} diff --git a/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java b/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java new file mode 100644 index 0000000000..f6b31180e7 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java @@ -0,0 +1,30 @@ +package com.akto.utils; + +import com.akto.dto.ByteArrayWrapper; +import com.akto.log.LoggerMaker; + +import static com.akto.listener.InitializerListener.loadTemplateFilesFromDirectory; + +public class TestTemplateUtils { + + private static final LoggerMaker loggerMaker = new LoggerMaker(TestTemplateUtils.class, LoggerMaker.LogDb.DASHBOARD); + + public static ByteArrayWrapper getTestingTemplates() { + GithubSync githubSync = new GithubSync(); + byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); + if(repoZip == null) { + loggerMaker.infoAndAddToDb("Failed to load test templates from github, trying to load from local directory"); + repoZip = loadTemplateFilesFromDirectory(); + if(repoZip == null) { + loggerMaker.errorAndAddToDb("Failed to load test templates from github or local directory"); + return null; + } else { + loggerMaker.infoAndAddToDb("Loaded test templates from local directory"); + } + } else { + loggerMaker.infoAndAddToDb("Loaded test templates from github"); + } + return new ByteArrayWrapper(repoZip); + } + +} From 532b84bae03cc10c507bb6b684de531c13db4990 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 16:50:49 +0530 Subject: [PATCH 06/11] Removed ByteArrayWrapper --- .../main/java/com/akto/action/AccountAction.java | 4 ++-- .../com/akto/listener/InitializerListener.java | 4 ++-- .../java/com/akto/utils/TestTemplateUtils.java | 5 ++--- .../main/java/com/akto/dto/ByteArrayWrapper.java | 14 -------------- 4 files changed, 6 insertions(+), 21 deletions(-) delete mode 100644 libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java diff --git a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java index 6b2d7f88aa..e5705d50fe 100644 --- a/apps/dashboard/src/main/java/com/akto/action/AccountAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/AccountAction.java @@ -311,13 +311,13 @@ public void run() { } try { - ByteArrayWrapper testingTemplates = TestTemplateUtils.getTestingTemplates(); + byte[] testingTemplates = TestTemplateUtils.getTestingTemplates(); if(testingTemplates == null){ loggerMaker.errorAndAddToDb("Failed to load test templates", LogDb.DASHBOARD); return; } loggerMaker.infoAndAddToDb(String.format("Updating akto test templates for new account: %d", newAccountId), LogDb.DASHBOARD); - InitializerListener.processTemplateFilesZip(testingTemplates.getData(), InitializerListener._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + InitializerListener.processTemplateFilesZip(testingTemplates, InitializerListener._AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); } catch (Exception e) { loggerMaker.errorAndAddToDb(e,String.format("Error while adding test editor templates for new account %d, Error: %s", newAccountId, e.getMessage()), LogDb.DASHBOARD); } diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index 8d17dad935..b3d540145e 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -1894,7 +1894,7 @@ private static String getUpdateDeploymentStatusUrl() { public void setUpTestEditorTemplatesScheduler() { scheduler.scheduleAtFixedRate(new Runnable() { public void run() { - ByteArrayWrapper testingTemplates = TestTemplateUtils.getTestingTemplates(); + byte[] testingTemplates = TestTemplateUtils.getTestingTemplates(); if(testingTemplates == null){ loggerMaker.errorAndAddToDb("Error while fetching Test Editor Templates from Github and local", LogDb.DASHBOARD); return; @@ -1902,7 +1902,7 @@ public void run() { AccountTask.instance.executeTask((account) -> { try { loggerMaker.infoAndAddToDb("Updating Test Editor Templates for accountId: " + account.getId(), LogDb.DASHBOARD); - processTemplateFilesZip(testingTemplates.getData(), _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); + processTemplateFilesZip(testingTemplates, _AKTO, YamlTemplateSource.AKTO_TEMPLATES.toString(), ""); } catch (Exception e) { cacheLoggerMaker.errorAndAddToDb(e, String.format("Error while updating Test Editor Files %s", e.toString()), diff --git a/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java b/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java index f6b31180e7..a4ec25fe35 100644 --- a/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java +++ b/apps/dashboard/src/main/java/com/akto/utils/TestTemplateUtils.java @@ -1,6 +1,5 @@ package com.akto.utils; -import com.akto.dto.ByteArrayWrapper; import com.akto.log.LoggerMaker; import static com.akto.listener.InitializerListener.loadTemplateFilesFromDirectory; @@ -9,7 +8,7 @@ public class TestTemplateUtils { private static final LoggerMaker loggerMaker = new LoggerMaker(TestTemplateUtils.class, LoggerMaker.LogDb.DASHBOARD); - public static ByteArrayWrapper getTestingTemplates() { + public static byte[] getTestingTemplates() { GithubSync githubSync = new GithubSync(); byte[] repoZip = githubSync.syncRepo("akto-api-security/tests-library", "master"); if(repoZip == null) { @@ -24,7 +23,7 @@ public static ByteArrayWrapper getTestingTemplates() { } else { loggerMaker.infoAndAddToDb("Loaded test templates from github"); } - return new ByteArrayWrapper(repoZip); + return repoZip; } } diff --git a/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java b/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java deleted file mode 100644 index 3209c13ad4..0000000000 --- a/libs/dao/src/main/java/com/akto/dto/ByteArrayWrapper.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.akto.dto; - -public class ByteArrayWrapper { - - private byte[] data; - - public ByteArrayWrapper(byte[] data) { - this.data = data; - } - - public byte[] getData() { - return data; - } -} From e1ed846e023d1191892bc634c013adf3a533ea8d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 16:54:42 +0530 Subject: [PATCH 07/11] Refactored fetchPIIFile method --- .../akto/listener/InitializerListener.java | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index b3d540145e..a428144969 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -462,24 +462,15 @@ private static String loadPIIFileFromResources(String fileUrl){ private static String fetchPIIFile(PIISource piiSource){ String fileUrl = piiSource.getFileUrl(); String id = piiSource.getId(); - if (fileUrl.startsWith("http")) { - String tempFileUrl = "temp_" + id; - if(downloadFileCheck(tempFileUrl)){ - try { - FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); - } catch (IOException e) { - loggerMaker.errorAndAddToDb(e, String.format("failed to read file %s", piiSource.getFileUrl()), LogDb.DASHBOARD); - return loadPIIFileFromResources(piiSource.getFileUrl()); - } + String tempFileUrl = "temp_" + id; + try { + if (downloadFileCheck(tempFileUrl)) { + FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); } fileUrl = tempFileUrl; - try { - return FileUtils.readFileToString(new File(fileUrl), StandardCharsets.UTF_8); - } catch (IOException e){ - loggerMaker.errorAndAddToDb(e, String.format("failed to read file %s", piiSource.getFileUrl()), LogDb.DASHBOARD); - return loadPIIFileFromResources(piiSource.getFileUrl()); - } - } else { + return FileUtils.readFileToString(new File(fileUrl), StandardCharsets.UTF_8); + } catch (Exception e){ + loggerMaker.errorAndAddToDb(e, String.format("failed to fetch PII file %s from github, trying locally", piiSource.getFileUrl()), LogDb.DASHBOARD); return loadPIIFileFromResources(piiSource.getFileUrl()); } } From 77f2c726696ae44461c5f12d234d1e752cc2788d Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 17:23:18 +0530 Subject: [PATCH 08/11] Fixed file name is prod.yml --- .github/workflows/prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index 7109a8be90..c2d52800eb 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -31,7 +31,7 @@ jobs: - name: Download Akto templates zip and PII files working-directory: ./apps/dashboard/src/main/resources run: | - wget -O test-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip + wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json From 3b66af235c306c4551809e6fcbbb699fda13779b Mon Sep 17 00:00:00 2001 From: aktoboy Date: Wed, 21 Feb 2024 17:23:29 +0530 Subject: [PATCH 09/11] Updated staging.yml --- .github/workflows/staging.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 0116ed8b2f..97f97aad82 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -25,9 +25,13 @@ jobs: - name: Convert github branch name to be compatible with docker tag name convention and generate tag name id: docker_tag run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT - - name: Download Akto templates zip + - name: Download Akto templates zip and PII files working-directory: ./apps/dashboard/src/main/resources - run: wget -O test-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip + run: | + wget -O tests-library-master.zip https://github.com/akto-api-security/tests-library/archive/refs/heads/master.zip + wget -O general.json https://raw.githubusercontent.com/akto-api-security/pii-types/master/general.json + wget -O fintech.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/fintech.json + wget -O filetypes.json https://raw.githubusercontent.com/akto-api-security/akto/master/pii-types/filetypes.json - name: Prepare Dashboard UI working-directory: ./apps/dashboard/ run: npm install && export RELEASE_VERSION=${{steps.docker_tag.outputs.IMAGE_TAG}} && npm run build From 86a7a166026612b5fb1ed7082cf1c6e6df86b9d7 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 22 Feb 2024 11:16:11 +0530 Subject: [PATCH 10/11] Added if check for failing test --- .../main/java/com/akto/listener/InitializerListener.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java index a428144969..1782b9839a 100644 --- a/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java +++ b/apps/dashboard/src/main/java/com/akto/listener/InitializerListener.java @@ -464,10 +464,12 @@ private static String fetchPIIFile(PIISource piiSource){ String id = piiSource.getId(); String tempFileUrl = "temp_" + id; try { - if (downloadFileCheck(tempFileUrl)) { - FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); + if (fileUrl.startsWith("http")) { + if (downloadFileCheck(tempFileUrl)) { + FileUtils.copyURLToFile(new URL(fileUrl), new File(tempFileUrl), CONNECTION_TIMEOUT, CONNECTION_TIMEOUT); + } + fileUrl = tempFileUrl; } - fileUrl = tempFileUrl; return FileUtils.readFileToString(new File(fileUrl), StandardCharsets.UTF_8); } catch (Exception e){ loggerMaker.errorAndAddToDb(e, String.format("failed to fetch PII file %s from github, trying locally", piiSource.getFileUrl()), LogDb.DASHBOARD); From 6f196b107ec1dc046422fe10933cd6467c16b1e0 Mon Sep 17 00:00:00 2001 From: aktoboy Date: Thu, 22 Feb 2024 14:18:15 +0530 Subject: [PATCH 11/11] Updates to traffic connectors --- .../web/src/apps/dashboard/pages/quick_start/transform.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/transform.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/transform.js index 14817b97dc..1db7669530 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/transform.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/quick_start/transform.js @@ -12,7 +12,6 @@ const mirroringObj = { icon: '/public/aws.svg', label: "AWS Mirroring", text: "You can deploy Akto in AWS and collect traffic through traffic mirroring.", - badge: "Recommended", docsUrl: 'https://docs.akto.io/traffic-connections/amazon-aws', key: "AWS", component: @@ -113,6 +112,7 @@ const kongObj = { const kubernetesObj = { icon: '/public/kubernetes.svg', label: 'Kubernetes Daemonset', + badge: "Recommended", text: 'You can deploy Akto in Kubernetes and collect traffic through a daemonset on your Kubernetes configuration.', docsUrl: 'https://docs.akto.io/traffic-connections/kubernetes', key: "KUBERNETES", @@ -686,9 +686,9 @@ const yaml_kubernetes = [ const quickStartFunc = { getConnectorsList: function (){ - const connectorsList = [mirroringObj, gcpObj, kubernetesObj, fargateObj, nginxObj, burpObj, postmanObj, + const connectorsList = [gcpObj, kubernetesObj, fargateObj, nginxObj, burpObj, postmanObj, openApiObj, beanStalkObj, eksObj, dockerObj, envoyObj, ebpfObj, - harFileUploadObj, kongObj, tcpObj + harFileUploadObj, kongObj, tcpObj, mirroringObj ] return connectorsList },