diff --git a/.github/workflows/prod.yml b/.github/workflows/prod.yml index c0aa1ecfe2..e339e91bd9 100644 --- a/.github/workflows/prod.yml +++ b/.github/workflows/prod.yml @@ -47,6 +47,11 @@ jobs: - uses: actions/setup-node@v2 with: node-version: '17' + - uses: bufbuild/buf-action@v1 + with: + setup_only: true + - name: Generate Proto files + run: make proto-gen - name: Download Akto templates zip and PII files working-directory: ./apps/dashboard/src/main/resources run: | diff --git a/.github/workflows/staging.yml b/.github/workflows/staging.yml index 02920d5ba9..4f3fbe6e2c 100644 --- a/.github/workflows/staging.yml +++ b/.github/workflows/staging.yml @@ -26,6 +26,11 @@ jobs: - uses: actions/setup-node@v2 with: node-version: "17" + - uses: bufbuild/buf-action@v1 + with: + setup_only: true + - name: Generate Proto files + run: make proto-gen - name: Convert github branch name to be compatible with docker tag name convention and generate tag name id: docker_tag run: echo "IMAGE_TAG=a-$(echo ${{ github.ref_name }} | sed 's/[^a-zA-Z0-9]/-/g')" >> $GITHUB_OUTPUT diff --git a/.gitignore b/.gitignore index 685da2accc..d90e415a62 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,4 @@ https: **/data-zoo-data **/data-zoo-logs **/bin +.factorypath diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..b79b0909a1 --- /dev/null +++ b/Makefile @@ -0,0 +1,8 @@ +proto-gen: + sh ./scripts/proto-gen.sh + +build: proto-gen + mvn install -DskipTests + +build-clean: proto-gen + mvn clean install -DskipTests \ No newline at end of file diff --git a/apps/dashboard/pom.xml b/apps/dashboard/pom.xml index c7966302b5..6cbf9bd3ee 100644 --- a/apps/dashboard/pom.xml +++ b/apps/dashboard/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 @@ -25,7 +26,7 @@ - + com.amazonaws aws-java-sdk-lambda 1.12.405 @@ -109,6 +110,11 @@ dao ${project.version} + + com.akto.libs.protobuf + protobuf + ${project.version} + com.akto.libs.utils utils @@ -294,4 +300,4 @@ ${project.artifactId} - + \ No newline at end of file diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/AbstractThreatDetectionAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/AbstractThreatDetectionAction.java new file mode 100644 index 0000000000..17c5e6c025 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/AbstractThreatDetectionAction.java @@ -0,0 +1,42 @@ +package com.akto.action.threat_detection; + +import com.akto.action.UserAction; +import com.akto.dao.context.Context; +import com.akto.database_abstractor_authenticator.JwtAuthenticator; +import java.util.Calendar; +import java.util.HashMap; +import java.util.Map; + +public class AbstractThreatDetectionAction extends UserAction { + + private Map tokens = new HashMap<>(); + private String backendUrl; + + public AbstractThreatDetectionAction() { + super(); + this.backendUrl = System.getenv().getOrDefault("THREAT_DETECTION_BACKEND_URL", "https://tbs.akto.io"); + } + + public String getApiToken() { + try { + int accountId = Context.accountId.get(); + if (tokens.containsKey(accountId)) { + return tokens.get(accountId); + } + + Map claims = new HashMap<>(); + claims.put("accountId", accountId); + String token = JwtAuthenticator.createJWT(claims, "Akto", "access_tbs", Calendar.MINUTE, 1); + tokens.put(accountId, token); + + return token; + } catch (Exception e) { + System.out.println(e); + return ""; + } + } + + public String getBackendUrl() { + return backendUrl; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardMaliciousEvent.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardMaliciousEvent.java new file mode 100644 index 0000000000..d63268b448 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardMaliciousEvent.java @@ -0,0 +1,111 @@ +package com.akto.action.threat_detection; + +import com.akto.dto.type.URLMethods; +import com.akto.dto.type.URLMethods.Method; + +public class DashboardMaliciousEvent { + private String id; + private String actor; + private String filter_id; + private String url; + private URLMethods.Method method; + private int apiCollectionId; + private String ip; + private String country; + private long timestamp; + + public DashboardMaliciousEvent() {} + + public DashboardMaliciousEvent( + String id, + String actor, + String filter, + String url, + Method method, + int apiCollectionId, + String ip, + String country, + long timestamp) { + this.id = id; + this.actor = actor; + this.filter_id = filter; + this.url = url; + this.method = method; + this.apiCollectionId = apiCollectionId; + this.ip = ip; + this.country = country; + this.timestamp = timestamp; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getActor() { + return actor; + } + + public void setActor(String actor) { + this.actor = actor; + } + + public String getFilterId() { + return filter_id; + } + + public void setFilterId(String filter) { + this.filter_id = filter; + } + + public String getUrl() { + return url; + } + + public void setUrl(String url) { + this.url = url; + } + + public URLMethods.Method getMethod() { + return method; + } + + public void setMethod(URLMethods.Method method) { + this.method = method; + } + + public String getIp() { + return ip; + } + + public void setIp(String ip) { + this.ip = ip; + } + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country = country; + } + + public long getTimestamp() { + return timestamp; + } + + public void setTimestamp(long timestamp) { + this.timestamp = timestamp; + } + + public int getApiCollectionId() { + return apiCollectionId; + } + + public void setApiCollectionId(int apiCollectionId) { + this.apiCollectionId = apiCollectionId; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatActor.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatActor.java new file mode 100644 index 0000000000..5b7c3a7d04 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatActor.java @@ -0,0 +1,77 @@ +package com.akto.action.threat_detection; + +import com.akto.dto.type.URLMethods.Method; + +public class DashboardThreatActor { + + private String id; + private String latestApiEndpoint; + private String latestApiIp; + private Method latestApiMethod; + private long discoveredAt; + private String country; + + public DashboardThreatActor( + String id, + String latestApiEndpoint, + String latestApiIp, + Method latestApiMethod, + long discoveredAt, + String country) { + + this.id = id; + this.latestApiEndpoint = latestApiEndpoint; + this.latestApiIp = latestApiIp; + this.latestApiMethod = latestApiMethod; + this.discoveredAt = discoveredAt; + this.country = country; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getLatestApiEndpoint() { + return latestApiEndpoint; + } + + public void setLatestApiEndpoint(String latestApiEndpoint) { + this.latestApiEndpoint = latestApiEndpoint; + } + + public String getLatestApiIp() { + return latestApiIp; + } + + public void setLatestApiIp(String latestApiIp) { + this.latestApiIp = latestApiIp; + } + + public Method getLatestApiMethod() { + return latestApiMethod; + } + + public void setLatestApiMethod(Method latestApiMethod) { + this.latestApiMethod = latestApiMethod; + } + + public long getDiscoveredAt() { + return discoveredAt; + } + + public void setDiscoveredAt(long discoveredAt) { + this.discoveredAt = discoveredAt; + } + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country = country; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatApi.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatApi.java new file mode 100644 index 0000000000..9ebe40965b --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/DashboardThreatApi.java @@ -0,0 +1,61 @@ +package com.akto.action.threat_detection; + +import com.akto.dto.type.URLMethods; + +public class DashboardThreatApi { + + private String api; + private URLMethods.Method method; + private int actorsCount; + private int requestsCount; + private long discoveredAt; + + public DashboardThreatApi( + String api, URLMethods.Method method, int actorsCount, int requestsCount, long discoveredAt) { + this.api = api; + this.method = method; + this.actorsCount = actorsCount; + this.requestsCount = requestsCount; + this.discoveredAt = discoveredAt; + } + + public String getApi() { + return api; + } + + public void setApi(String api) { + this.api = api; + } + + public URLMethods.Method getMethod() { + return method; + } + + public void setMethod(URLMethods.Method method) { + this.method = method; + } + + public int getActorsCount() { + return actorsCount; + } + + public void setActorsCount(int actorsCount) { + this.actorsCount = actorsCount; + } + + public int getRequestsCount() { + return requestsCount; + } + + public void setRequestsCount(int requestsCount) { + this.requestsCount = requestsCount; + } + + public long getDiscoveredAt() { + return discoveredAt; + } + + public void setDiscoveredAt(long discoveredAt) { + this.discoveredAt = discoveredAt; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java index d271463da8..d7542eacad 100644 --- a/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/SuspectSampleDataAction.java @@ -1,184 +1,203 @@ package com.akto.action.threat_detection; -import java.util.ArrayList; -import java.util.HashSet; +import com.akto.dto.traffic.SuspectSampleData; +import com.akto.dto.type.URLMethods; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.FetchAlertFiltersResponse; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListMaliciousRequestsResponse; +import com.akto.proto.utils.ProtoMessageUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Set; - -import org.bson.conversions.Bson; - -import com.akto.action.UserAction; -import com.akto.dao.SuspectSampleDataDao; -import com.akto.dao.context.Context; -import com.akto.dto.traffic.SuspectSampleData; -import com.akto.util.Constants; -import com.mongodb.client.model.Filters; -import com.mongodb.client.model.Sorts; - -public class SuspectSampleDataAction extends UserAction { - - List sampleData; - int skip; - static final int LIMIT = 50; - List ips; - List urls; - List apiCollectionIds; - long total; - Map sort; - int startTimestamp, endTimestamp; - - public String fetchSuspectSampleData() { - - List filterList = new ArrayList<>(); - - /* - * In case time filters are empty, - * using default filter as 2 months. - */ - - if (startTimestamp <= 0) { - startTimestamp = Context.now() - 2 * 30 * 24 * 60 * 60; - } - if (endTimestamp <= 0) { - endTimestamp = Context.now() + 10 * 60; - } - - filterList.add(Filters.gte(SuspectSampleData._DISCOVERED, startTimestamp)); - filterList.add(Filters.lte(SuspectSampleData._DISCOVERED, endTimestamp)); - - if (ips != null && !ips.isEmpty()) { - filterList.add(Filters.in(SuspectSampleData.SOURCE_IPS, ips)); - } - if (urls != null && !urls.isEmpty()) { - filterList.add(Filters.in(SuspectSampleData.MATCHING_URL, urls)); - } - if (apiCollectionIds != null && !apiCollectionIds.isEmpty()) { - filterList.add(Filters.in(SuspectSampleData.API_COLLECTION_ID, apiCollectionIds)); - } - - Bson finalFilter = Filters.empty(); - - if (!filterList.isEmpty()) { - finalFilter = Filters.and(filterList); - } - - String sortKey = SuspectSampleData._DISCOVERED; - int sortDirection = -1; - /* - * add any new sort key here, - * for validation and sanity. - */ - Set sortKeys = new HashSet<>(); - sortKeys.add(SuspectSampleData._DISCOVERED); - - if (sort != null && !sort.isEmpty()) { - Entry sortEntry = sort.entrySet().iterator().next(); - sortKey = sortEntry.getKey(); - if (!sortKeys.contains(sortKey)) { - sortKey = SuspectSampleData._DISCOVERED; - } - sortDirection = sortEntry.getValue(); - if (!(sortDirection == -1 || sortDirection == 1)) { - sortDirection = -1; - } - } - - /* - * In case timestamp is same, then id acts as tie-breaker, - * to avoid repeating the same documents again. - */ - Bson sort = sortDirection == -1 ? Sorts.descending(sortKey, Constants.ID) - : Sorts.ascending(sortKey, Constants.ID); - sampleData = SuspectSampleDataDao.instance.findAll(finalFilter, skip, LIMIT, sort); - total = SuspectSampleDataDao.instance.count(finalFilter); - - return SUCCESS.toUpperCase(); - } - - public String fetchFilters() { - ips = new ArrayList<>( - SuspectSampleDataDao.instance.findDistinctFields(SuspectSampleData.SOURCE_IPS, String.class, Filters.empty())); - urls = new ArrayList<>(SuspectSampleDataDao.instance.findDistinctFields(SuspectSampleData.MATCHING_URL, String.class, - Filters.empty())); - return SUCCESS.toUpperCase(); - } - - public List getSampleData() { - return sampleData; - } - - public void setSampleData(List sampleData) { - this.sampleData = sampleData; - } +import java.util.stream.Collectors; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +public class SuspectSampleDataAction extends AbstractThreatDetectionAction { + + List sampleData; + List maliciousEvents; + int skip; + static final int LIMIT = 50; + List ips; + List urls; + List apiCollectionIds; + long total; + Map sort; + int startTimestamp, endTimestamp; + + private final CloseableHttpClient httpClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + public SuspectSampleDataAction() { + super(); + this.httpClient = HttpClients.createDefault(); + } + + public String fetchSampleData() { + HttpPost post = new HttpPost(String.format("%s/api/dashboard/list_malicious_requests", this.getBackendUrl())); + post.addHeader("Authorization", "Bearer " + this.getApiToken()); + post.addHeader("Content-Type", "application/json"); + + System.out.print("API Token: " + this.getApiToken()); + + Map body = new HashMap() { + { + put("skip", skip); + put("limit", LIMIT); + } + }; + String msg = objectMapper.valueToTree(body).toString(); + + System.out.println("Request body for list malicious requests" + msg); + + StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); + post.setEntity(requestEntity); + + try (CloseableHttpResponse resp = this.httpClient.execute(post)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + ListMaliciousRequestsResponse.class, responseBody) + .ifPresent( + m -> { + this.maliciousEvents = m.getMaliciousEventsList().stream() + .map( + smr -> new DashboardMaliciousEvent( + smr.getId(), + smr.getActor(), + smr.getFilterId(), + smr.getEndpoint(), + URLMethods.Method.fromString(smr.getMethod()), + smr.getApiCollectionId(), + smr.getIp(), + smr.getCountry(), + smr.getDetectedAt())) + .collect(Collectors.toList()); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public String fetchFilters() { + HttpGet get = new HttpGet(String.format("%s/api/dashboard/fetch_filters", this.getBackendUrl())); + get.addHeader("Authorization", "Bearer " + this.getApiToken()); + get.addHeader("Content-Type", "application/json"); + + try (CloseableHttpResponse resp = this.httpClient.execute(get)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + FetchAlertFiltersResponse.class, responseBody) + .ifPresent( + msg -> { + this.ips = msg.getActorsList(); + this.urls = msg.getUrlsList(); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public List getSampleData() { + return sampleData; + } + + public void setSampleData(List sampleData) { + this.sampleData = sampleData; + } + + public int getSkip() { + return skip; + } + + public void setSkip(int skip) { + this.skip = skip; + } + + public static int getLimit() { + return LIMIT; + } + + public List getIps() { + return ips; + } + + public void setIps(List ips) { + this.ips = ips; + } + + public List getUrls() { + return urls; + } + + public void setUrls(List urls) { + this.urls = urls; + } + + public List getApiCollectionIds() { + return apiCollectionIds; + } + + public void setApiCollectionIds(List apiCollectionIds) { + this.apiCollectionIds = apiCollectionIds; + } + + public long getTotal() { + return total; + } + + public void setTotal(long total) { + this.total = total; + } + + public Map getSort() { + return sort; + } + + public void setSort(Map sort) { + this.sort = sort; + } + + public int getStartTimestamp() { + return startTimestamp; + } + + public void setStartTimestamp(int startTimestamp) { + this.startTimestamp = startTimestamp; + } - public int getSkip() { - return skip; - } - - public void setSkip(int skip) { - this.skip = skip; - } - - public static int getLimit() { - return LIMIT; - } - - public List getIps() { - return ips; - } - - public void setIps(List ips) { - this.ips = ips; - } - - public List getUrls() { - return urls; - } + public int getEndTimestamp() { + return endTimestamp; + } - public void setUrls(List urls) { - this.urls = urls; - } - - public List getApiCollectionIds() { - return apiCollectionIds; - } - - public void setApiCollectionIds(List apiCollectionIds) { - this.apiCollectionIds = apiCollectionIds; - } - - public long getTotal() { - return total; - } - - public void setTotal(long total) { - this.total = total; - } - - public Map getSort() { - return sort; - } - - public void setSort(Map sort) { - this.sort = sort; - } - - public int getStartTimestamp() { - return startTimestamp; - } - - public void setStartTimestamp(int startTimestamp) { - this.startTimestamp = startTimestamp; - } - - public int getEndTimestamp() { - return endTimestamp; - } - - public void setEndTimestamp(int endTimestamp) { - this.endTimestamp = endTimestamp; - } + public void setEndTimestamp(int endTimestamp) { + this.endTimestamp = endTimestamp; + } + + public List getMaliciousEvents() { + return maliciousEvents; + } + public void setMaliciousEvents(List maliciousRequests) { + this.maliciousEvents = maliciousRequests; + } } diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorAction.java new file mode 100644 index 0000000000..fbfe0a34ca --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorAction.java @@ -0,0 +1,147 @@ +package com.akto.action.threat_detection; + +import com.akto.dto.type.URLMethods; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListThreatActorResponse; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ThreatActorByCountryResponse; +import com.akto.proto.utils.ProtoMessageUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +public class ThreatActorAction extends AbstractThreatDetectionAction { + + List actors; + List actorsCountPerCountry; + int skip; + static final int LIMIT = 50; + long total; + Map sort; + int startTimestamp, endTimestamp; + + private final CloseableHttpClient httpClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + public ThreatActorAction() { + super(); + this.httpClient = HttpClients.createDefault(); + } + + public String getActorsCountPerCounty() { + HttpGet get = new HttpGet(String.format("%s/api/dashboard/get_actors_count_per_country", this.getBackendUrl())); + get.addHeader("Authorization", "Bearer " + this.getApiToken()); + get.addHeader("Content-Type", "application/json"); + + try (CloseableHttpResponse resp = this.httpClient.execute(get)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + ThreatActorByCountryResponse.class, responseBody) + .ifPresent( + m -> { + this.actorsCountPerCountry = m.getCountriesList().stream() + .map(smr -> new ThreatActorPerCountry(smr.getCode(), smr.getCount())) + .collect(Collectors.toList()); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public String fetchThreatActors() { + HttpPost post = new HttpPost(String.format("%s/api/dashboard/list_threat_actors", this.getBackendUrl())); + post.addHeader("Authorization", "Bearer " + this.getApiToken()); + post.addHeader("Content-Type", "application/json"); + + Map body = new HashMap() { + { + put("skip", skip); + put("limit", LIMIT); + } + }; + String msg = objectMapper.valueToTree(body).toString(); + + System.out.println("Request body for list threat actors" + msg); + + StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); + post.setEntity(requestEntity); + + try (CloseableHttpResponse resp = this.httpClient.execute(post)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + ListThreatActorResponse.class, responseBody) + .ifPresent( + m -> { + this.actors = m.getActorsList().stream() + .map( + smr -> new DashboardThreatActor( + smr.getId(), + smr.getLatestApiEndpoint(), + smr.getLatestApiIp(), + URLMethods.Method.fromString(smr.getLatestApiMethod()), + smr.getDiscoveredAt(), + smr.getCountry())) + .collect(Collectors.toList()); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public int getSkip() { + return skip; + } + + public void setSkip(int skip) { + this.skip = skip; + } + + public static int getLimit() { + return LIMIT; + } + + public long getTotal() { + return total; + } + + public void setTotal(long total) { + this.total = total; + } + + public List getActors() { + return actors; + } + + public void setActors(List actor) { + this.actors = actor; + } + + public List getActorsCountPerCountry() { + return actorsCountPerCountry; + } + + public void setActorsCountPerCountry(List actorsCountPerCountry) { + this.actorsCountPerCountry = actorsCountPerCountry; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorPerCountry.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorPerCountry.java new file mode 100644 index 0000000000..33765fe5dc --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatActorPerCountry.java @@ -0,0 +1,27 @@ +package com.akto.action.threat_detection; + +public class ThreatActorPerCountry { + private String country; + private int count; + + public ThreatActorPerCountry(String country, int count) { + this.country = country; + this.count = count; + } + + public String getCountry() { + return country; + } + + public void setCountry(String country) { + this.country = country; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatApiAction.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatApiAction.java new file mode 100644 index 0000000000..82f5695ad3 --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatApiAction.java @@ -0,0 +1,156 @@ +package com.akto.action.threat_detection; + +import com.akto.dto.type.URLMethods; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ListThreatApiResponse; +import com.akto.proto.generated.threat_detection.service.dashboard_service.v1.ThreatCategoryWiseCountResponse; +import com.akto.proto.utils.ProtoMessageUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ContentType; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.util.EntityUtils; + +public class ThreatApiAction extends AbstractThreatDetectionAction { + + List apis; + List categoryCounts; + int skip; + static final int LIMIT = 50; + long total; + Map sort; + int startTimestamp, endTimestamp; + + private final CloseableHttpClient httpClient; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + public ThreatApiAction() { + super(); + this.httpClient = HttpClients.createDefault(); + } + + public String fetchThreatCategoryCount() { + HttpGet get = new HttpGet(String.format("%s/api/dashboard/get_subcategory_wise_count", this.getBackendUrl())); + get.addHeader("Authorization", "Bearer " + this.getApiToken()); + get.addHeader("Content-Type", "application/json"); + + try (CloseableHttpResponse resp = this.httpClient.execute(get)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + ThreatCategoryWiseCountResponse.class, responseBody) + .ifPresent( + m -> { + this.categoryCounts = m.getCategoryWiseCountsList().stream() + .map( + smr -> new ThreatCategoryCount( + smr.getCategory(), smr.getSubCategory(), smr.getCount())) + .collect(Collectors.toList()); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public String fetchThreatApis() { + HttpPost post = new HttpPost(String.format("%s/api/dashboard/list_threat_apis", this.getBackendUrl())); + post.addHeader("Authorization", "Bearer " + this.getApiToken()); + post.addHeader("Content-Type", "application/json"); + + Map body = new HashMap() { + { + put("skip", skip); + put("limit", LIMIT); + } + }; + String msg = objectMapper.valueToTree(body).toString(); + + System.out.println("Request body for list threat actors" + msg); + + StringEntity requestEntity = new StringEntity(msg, ContentType.APPLICATION_JSON); + post.setEntity(requestEntity); + + try (CloseableHttpResponse resp = this.httpClient.execute(post)) { + String responseBody = EntityUtils.toString(resp.getEntity()); + + System.out.println(responseBody); + + ProtoMessageUtils.toProtoMessage( + ListThreatApiResponse.class, responseBody) + .ifPresent( + m -> { + this.apis = m.getApisList().stream() + .map( + smr -> new DashboardThreatApi( + smr.getEndpoint(), + URLMethods.Method.fromString(smr.getMethod()), + smr.getActorsCount(), + smr.getRequestsCount(), + smr.getDiscoveredAt())) + .collect(Collectors.toList()); + }); + } catch (Exception e) { + e.printStackTrace(); + return ERROR.toUpperCase(); + } + + return SUCCESS.toUpperCase(); + } + + public int getSkip() { + return skip; + } + + public void setSkip(int skip) { + this.skip = skip; + } + + public static int getLimit() { + return LIMIT; + } + + public long getTotal() { + return total; + } + + public void setTotal(long total) { + this.total = total; + } + + public List getApis() { + return apis; + } + + public void setApis(List apis) { + this.apis = apis; + } + + public Map getSort() { + return sort; + } + + public void setSort(Map sort) { + this.sort = sort; + } + + public List getCategoryCounts() { + return categoryCounts; + } + + public void setCategoryCounts(List categoryCounts) { + this.categoryCounts = categoryCounts; + } +} diff --git a/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatCategoryCount.java b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatCategoryCount.java new file mode 100644 index 0000000000..e2ac0b4f5c --- /dev/null +++ b/apps/dashboard/src/main/java/com/akto/action/threat_detection/ThreatCategoryCount.java @@ -0,0 +1,37 @@ +package com.akto.action.threat_detection; + +public class ThreatCategoryCount { + private String category; + private String subCategory; + private int count; + + public ThreatCategoryCount(String category, String subCategory, int count) { + this.category = category; + this.subCategory = subCategory; + this.count = count; + } + + public String getCategory() { + return category; + } + + public void setCategory(String category) { + this.category = category; + } + + public String getSubCategory() { + return subCategory; + } + + public void setSubCategory(String subCategory) { + this.subCategory = subCategory; + } + + public int getCount() { + return count; + } + + public void setCount(int count) { + this.count = count; + } +} diff --git a/apps/dashboard/src/main/resources/struts.xml b/apps/dashboard/src/main/resources/struts.xml index a5d6881002..f6b07cae05 100644 --- a/apps/dashboard/src/main/resources/struts.xml +++ b/apps/dashboard/src/main/resources/struts.xml @@ -7279,40 +7279,122 @@ - - + + 403 false - ^actionErrors.* + ^actionErrors.* - + + + 422 + false + ^actionErrors.* + + + + + + + + 403 + false + ^actionErrors.* + 422 false - ^actionErrors.* + ^actionErrors.* - - + + + 403 false - ^actionErrors.* + ^actionErrors.* - + + + 422 + false + ^actionErrors.* + + + + + + 403 + false + ^actionErrors.* + + 422 false - ^actionErrors.* + ^actionErrors.* + + + + + 403 + false + ^actionErrors.* + + + + 422 + false + ^actionErrors.* + + + + + + + + 403 + false + ^actionErrors.* + + + + 422 + false + ^actionErrors.* + + diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatActorPage.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatActorPage.jsx new file mode 100644 index 0000000000..a8c3f58e34 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatActorPage.jsx @@ -0,0 +1,147 @@ +import { useReducer, useState, useEffect } from "react"; +import DateRangeFilter from "../../components/layouts/DateRangeFilter"; +import PageWithMultipleCards from "../../components/layouts/PageWithMultipleCards"; +import TitleWithInfo from "../../components/shared/TitleWithInfo"; +import values from "@/util/values"; +import { produce } from "immer"; +import func from "@/util/func"; +import ThreatActorTable from "./components/ThreatActorsTable"; +import ThreatWorldMap from "./components/ThreatWorldMap"; +import ThreatApiSubcategoryCount from "./components/ThreatApiSubcategoryCount"; + +import api from "./api"; +import { HorizontalGrid, VerticalStack } from "@shopify/polaris"; +import TopThreatTypeChart from "./components/TopThreatTypeChart"; +function ThreatActorPage() { + const [mapData, setMapData] = useState([]); + const [loading, setLoading] = useState(false); + const [subCategoryCount, setSubCategoryCount] = useState([]); + const [categoryCount, setCategoryCount] = useState([]); + const initialVal = values.ranges[3]; + const [currDateRange, dispatchCurrDateRange] = useReducer( + produce((draft, action) => func.dateRangeReducer(draft, action)), + initialVal + ); + + useEffect(() => { + const fetchActorsPerCountry = async () => { + setLoading(true); + const res = await api.getActorsCountPerCounty(); + if (res?.actorsCountPerCountry) { + setMapData( + res.actorsCountPerCountry.map((x) => { + return { + code: x.country, + z: 100, + count: x.count, + }; + }) + ); + } + setLoading(false); + }; + const fetchThreatCategoryCount = async () => { + setLoading(true); + const res = await api.fetchThreatCategoryCount(); + if (res?.categoryCounts) { + const categoryRes = {}; + const subCategoryRes = {}; + for (const cc of res.categoryCounts) { + if (categoryRes[cc.category]) { + categoryRes[cc.category] += cc.count; + } else { + categoryRes[cc.category] = cc.count; + } + + if (subCategoryRes[cc.subCategory]) { + subCategoryRes[cc.subCategory] += cc.count; + } else { + subCategoryRes[cc.subCategory] = cc.count; + } + } + + setSubCategoryCount( + Object.keys(subCategoryRes).map((x) => { + return { + text: x.replaceAll("_", " "), + value: subCategoryRes[x], + color: "#A5B4FC", + }; + }) + ); + + setCategoryCount( + Object.keys(categoryRes).map((x) => { + return { + text: x.replaceAll("_", " "), + value: categoryRes[x], + color: "#A5B4FC", + }; + }) + ); + } + setLoading(false); + }; + fetchActorsPerCountry(); + fetchThreatCategoryCount(); + }, []); + + const ChartComponent = () => { + return ( + + + + + + + + ); + }; + + const components = [ + , + , + ]; + + return ( + } + isFirstPage={true} + primaryAction={ + + dispatchCurrDateRange({ + type: "update", + period: dateObj.period, + title: dateObj.title, + alias: dateObj.alias, + }) + } + /> + } + components={components} + /> + ); +} + +export default ThreatActorPage; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatApiPage.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatApiPage.jsx new file mode 100644 index 0000000000..12af7273f6 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/ThreatApiPage.jsx @@ -0,0 +1,111 @@ +import { useEffect, useReducer, useState } from "react"; +import DateRangeFilter from "../../components/layouts/DateRangeFilter"; +import PageWithMultipleCards from "../../components/layouts/PageWithMultipleCards"; +import TitleWithInfo from "../../components/shared/TitleWithInfo"; +import values from "@/util/values"; +import { produce } from "immer"; +import func from "@/util/func"; +import ThreatApisTable from "./components/ThreatApisTable"; +import TopThreatTypeChart from "./components/TopThreatTypeChart"; +import ThreatApiSubcategoryCount from "./components/ThreatApiSubcategoryCount"; + +import api from "./api"; +import { HorizontalGrid } from "@shopify/polaris"; +function ThreatApiPage() { + const [loading, setLoading] = useState(false); + const [categoryCount, setCategoryCount] = useState([]); + const [subCategoryCount, setSubCategoryCount] = useState([]); + const initialVal = values.ranges[3]; + const [currDateRange, dispatchCurrDateRange] = useReducer( + produce((draft, action) => func.dateRangeReducer(draft, action)), + initialVal + ); + + const ChartComponent = () => { + return ( + + + + + ); + }; + const components = [ + , + , + ]; + + useEffect(() => { + const fetchThreatCategoryCount = async () => { + setLoading(true); + const res = await api.fetchThreatCategoryCount(); + if (res?.categoryCounts) { + const categoryRes = {}; + const subCategoryRes = {}; + for (const cc of res.categoryCounts) { + if (categoryRes[cc.category]) { + categoryRes[cc.category] += cc.count; + } else { + categoryRes[cc.category] = cc.count; + } + + if (subCategoryRes[cc.subCategory]) { + subCategoryRes[cc.subCategory] += cc.count; + } else { + subCategoryRes[cc.subCategory] = cc.count; + } + } + + setSubCategoryCount( + Object.keys(subCategoryRes).map((x) => { + return { + text: x.replaceAll("_", " "), + value: subCategoryRes[x], + }; + }) + ); + + setCategoryCount( + Object.keys(categoryRes).map((x) => { + return { + text: x.replaceAll("_", " "), + value: categoryRes[x], + color: "#A5B4FC", + }; + }) + ); + } + setLoading(false); + }; + + fetchThreatCategoryCount(); + }, []); + + return ( + } + isFirstPage={true} + primaryAction={ + + dispatchCurrDateRange({ + type: "update", + period: dateObj.period, + title: dateObj.title, + alias: dateObj.alias, + }) + } + /> + } + components={components} + /> + ); +} + +export default ThreatApiPage; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/api.js b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/api.js index d85a5597ab..61379cdc98 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/api.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/api.js @@ -38,6 +38,38 @@ const threatDetectionRequests = { data: {} }) }, + fetchThreatActors(skip) { + return request({ + url: '/api/fetchThreatActors', + method: 'post', + data: { + skip: skip + } + }) + }, + fetchThreatApis(skip) { + return request({ + url: '/api/fetchThreatApis', + method: 'post', + data: { + skip: skip + } + }) + }, + getActorsCountPerCounty() { + return request({ + url: '/api/getActorsCountPerCounty', + method: 'get', + data: {} + }) + }, + fetchThreatCategoryCount() { + return request({ + url: '/api/fetchThreatCategoryCount', + method: 'get', + data: {} + }) + } } export default threatDetectionRequests \ No newline at end of file diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx index 9e3ea68191..4af459eab5 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/SusDataTable.jsx @@ -1,184 +1,231 @@ import { useEffect, useState } from "react"; import GithubServerTable from "../../../components/tables/GithubServerTable"; -import api from "../api" +import api from "../api"; import { CellType } from "../../../components/tables/rows/GithubRow"; import GetPrettifyEndpoint from "../../observe/GetPrettifyEndpoint"; import PersistStore from "../../../../main/PersistStore"; import func from "../../../../../util/func"; -import { Text } from "@shopify/polaris"; const resourceName = { - singular: 'sample', - plural: 'samples', + singular: "sample", + plural: "samples", }; const headers = [ - { - text: "Endpoint", - value: "endpointComp", - title: "Api endpoints", - }, - { - text: "matchingUrl", - value: "matchingUrl", - title: "Affected url", - }, - { - text: "Filter", - value: "filterId", - title: "Threat filter", - }, - { - text: "Collection", - value: 'apiCollectionName', - title: "Collection", - maxWidth: '95px', - type: CellType.TEXT, - }, - { - text: 'Discovered', - title: 'Discovered', - value: 'discoveredTs', - type: CellType.TEXT, - sortActive: true - }, - { - text: 'Source IPs', - title: 'Source IPs', - value: 'sourceIPComponent', - }, - { - title: '', - type: CellType.ACTION, - } -] + { + text: "Endpoint", + value: "endpointComp", + title: "Endpoint", + }, + { + text: "Actor", + value: "actorComp", + title: "Actor", + }, + { + text: "Filter", + value: "filterId", + title: "Threat filter", + }, + { + text: "Collection", + value: "apiCollectionName", + title: "Collection", + maxWidth: "95px", + type: CellType.TEXT, + }, + { + text: "Discovered", + title: "Discovered", + value: "discoveredTs", + type: CellType.TEXT, + sortActive: true, + }, + { + text: "Source IP", + title: "Source IP", + value: "sourceIPComponent", + }, + { + title: "", + type: CellType.ACTION, + }, +]; const sortOptions = [ - { label: 'Discovered time', value: 'discovered asc', directionLabel: 'Newest', sortKey: 'discovered', columnIndex: 3 }, - { label: 'Discovered time', value: 'discovered desc', directionLabel: 'Oldest', sortKey: 'discovered', columnIndex: 3 }, + { + label: "Discovered time", + value: "discovered asc", + directionLabel: "Newest", + sortKey: "discovered", + columnIndex: 3, + }, + { + label: "Discovered time", + value: "discovered desc", + directionLabel: "Oldest", + sortKey: "discovered", + columnIndex: 3, + }, ]; -let filters = [] +let filters = []; function SusDataTable({ currDateRange, rowClicked }) { - const getTimeEpoch = (key) => { - return Math.floor(Date.parse(currDateRange.period[key]) / 1000) - } - const startTimestamp = getTimeEpoch("since") - const endTimestamp = getTimeEpoch("until") + const getTimeEpoch = (key) => { + return Math.floor(Date.parse(currDateRange.period[key]) / 1000); + }; + const startTimestamp = getTimeEpoch("since"); + const endTimestamp = getTimeEpoch("until"); - const [loading, setLoading] = useState(true); - const collectionsMap = PersistStore(state => state.collectionsMap) - const allCollections = PersistStore(state => state.allCollections) + const [loading, setLoading] = useState(true); + const collectionsMap = PersistStore((state) => state.collectionsMap); + const allCollections = PersistStore((state) => state.allCollections); - async function fetchData(sortKey, sortOrder, skip, limit, filters, filterOperators, queryValue) { - setLoading(true); - let sourceIpsFilter = [], apiCollectionIdsFilter = [], matchingUrlFilter = [] - if (filters?.sourceIps) { - sourceIpsFilter = filters?.sourceIps - } - if (filters?.apiCollectionId) { - apiCollectionIdsFilter = filters?.apiCollectionId - } - if (filters?.url) { - matchingUrlFilter = filters?.url - } - const sort = { [sortKey]: sortOrder } - const res = await api.fetchSuspectSampleData(skip, sourceIpsFilter, apiCollectionIdsFilter, matchingUrlFilter, sort, startTimestamp, endTimestamp) - let total = res.total; - let ret = res?.sampleData.map(x => { - return { - ...x, - endpointComp: , - apiCollectionName: collectionsMap[x.apiCollectionId] || '-', - discoveredTs: func.prettifyEpoch(x.discovered), - sourceIPComponent: {x?.sourceIPs ? x.sourceIPs.reduce((a, b) => a += ((a.length > 0 ? ", " : "") + b), "") : "-"} - } - }) - setLoading(false); - return { value: ret, total: total }; + async function fetchData( + sortKey, + sortOrder, + skip, + limit, + filters, + filterOperators, + queryValue + ) { + setLoading(true); + let sourceIpsFilter = [], + apiCollectionIdsFilter = [], + matchingUrlFilter = []; + if (filters?.sourceIps) { + sourceIpsFilter = filters?.sourceIps; + } + if (filters?.apiCollectionId) { + apiCollectionIdsFilter = filters?.apiCollectionId; + } + if (filters?.url) { + matchingUrlFilter = filters?.url; } + const sort = { [sortKey]: sortOrder }; + const res = await api.fetchSuspectSampleData( + skip, + sourceIpsFilter, + apiCollectionIdsFilter, + matchingUrlFilter, + sort, + startTimestamp, + endTimestamp + ); + let total = res.total; + let ret = res?.maliciousEvents.map((x) => { + return { + ...x, + actorComp: x?.actor, + endpointComp: ( + + ), + apiCollectionName: collectionsMap[x.apiCollectionId] || "-", + discoveredTs: func.prettifyEpoch(x.timestamp), + sourceIPComponent: x?.ip || "-", + }; + }); + setLoading(false); + return { value: ret, total: total }; + } - async function fillFilters() { - const res = await api.fetchFiltersThreatTable(); - let apiCollectionFilterChoices = allCollections.filter(x => { - return x.type !== 'API_GROUP' - }).map(x => { - return { label: x.displayName, value: x.id } - }) - let urlChoices = res?.urls.filter(x => { - return x.length > 0 - }).map(x => { - return { label: x, value: x } - }) - let ipChoices = res?.ips.map(x => { - return { label: x, value: x } - }) + async function fillFilters() { + const res = await api.fetchFiltersThreatTable(); + let apiCollectionFilterChoices = allCollections + .filter((x) => { + return x.type !== "API_GROUP"; + }) + .map((x) => { + return { label: x.displayName, value: x.id }; + }); + let urlChoices = res?.urls + .filter((x) => { + return x.length > 0; + }) + .map((x) => { + return { label: x, value: x }; + }); + let ipChoices = res?.ips.map((x) => { + return { label: x, value: x }; + }); - filters = [ - { - key: 'apiCollectionId', - label: 'Collection', - title: 'Collection', - choices: apiCollectionFilterChoices, - }, { - key: 'sourceIps', - label: 'Source IP', - title: 'Source IP', - choices: ipChoices, - }, { - key: 'url', - label: 'URL', - title: 'URL', - choices: urlChoices, - }, - ] - } + filters = [ + { + key: "apiCollectionId", + label: "Collection", + title: "Collection", + choices: apiCollectionFilterChoices, + }, + { + key: "sourceIps", + label: "Source IP", + title: "Source IP", + choices: ipChoices, + }, + { + key: "url", + label: "URL", + title: "URL", + choices: urlChoices, + }, + ]; + } - useEffect(() => { - fillFilters() - }, []) + useEffect(() => { + fillFilters(); + }, []); - function disambiguateLabel(key, value) { - switch (key) { - case "apiCollectionId": - return func.convertToDisambiguateLabelObj(value, collectionsMap, 2) - default: - return func.convertToDisambiguateLabelObj(value, null, 2); - } + function disambiguateLabel(key, value) { + switch (key) { + case "apiCollectionId": + return func.convertToDisambiguateLabelObj(value, collectionsMap, 2); + default: + return func.convertToDisambiguateLabelObj(value, null, 2); } + } - const getActions = (item) => { - return [{ - items: [{ - content: 'View in collection', - onAction: () => { - window.open(`/dashboard/observe/inventory/${item.apiCollectionId}`, "_blank") - }, - }] - }] - } + const getActions = (item) => { + return [ + { + items: [ + { + content: "View in collection", + onAction: () => { + window.open( + `/dashboard/observe/inventory/${item.apiCollectionId}`, + "_blank" + ); + }, + }, + ], + }, + ]; + }; - const key = startTimestamp + endTimestamp; - return ( rowClicked(data)} - fetchData={fetchData} - filters={filters} - selectable={false} - hasRowActions={true} - getActions={getActions} - hideQueryField={true} - headings={headers} - useNewRow={true} - condensedHeight={true} - />) + const key = startTimestamp + endTimestamp; + return ( + rowClicked(data)} [For now removing on row click functionality] + fetchData={fetchData} + filters={filters} + selectable={false} + hasRowActions={true} + getActions={getActions} + hideQueryField={true} + headings={headers} + useNewRow={true} + condensedHeight={true} + /> + ); } export default SusDataTable; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatActorsTable.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatActorsTable.jsx new file mode 100644 index 0000000000..a4ff602489 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatActorsTable.jsx @@ -0,0 +1,102 @@ +import { useState } from "react"; +import GithubServerTable from "../../../components/tables/GithubServerTable"; +import api from "../api"; +import { CellType } from "../../../components/tables/rows/GithubRow"; +import GetPrettifyEndpoint from "../../observe/GetPrettifyEndpoint"; +import func from "../../../../../util/func"; + +const resourceName = { + singular: "actor", + plural: "actors", +}; + +const headers = [ + { + text: "Actor", + value: "actor", + title: "Actor", + }, + { + text: "Latest IP", + title: "Latest IP", + value: "latestIp", + }, + { + text: "Latest API", + title: "Latest API", + value: "latestApi", + }, + { + text: "Discovered", + title: "Discovered", + value: "discoveredAt", + type: CellType.TEXT, + sortActive: true, + }, +]; + +const sortOptions = []; + +let filters = []; + +function ThreatActorTable({ data, currDateRange, rowClicked }) { + const [loading, setLoading] = useState(false); + + const getTimeEpoch = (key) => { + return Math.floor(Date.parse(currDateRange.period[key]) / 1000); + }; + const startTimestamp = getTimeEpoch("since"); + const endTimestamp = getTimeEpoch("until"); + + function disambiguateLabel(key, value) { + return func.convertToDisambiguateLabelObj(value, null, 2); + } + + async function fetchData(sortKey, sortOrder, skip) { + setLoading(true); + const sort = { [sortKey]: sortOrder }; + const res = await api.fetchThreatActors(skip); + let total = res.total; + let ret = res?.actors?.map((x) => { + return { + ...x, + actor: x.id, + latestIp: x.latestApiIp, + discoveredAt: func.prettifyEpoch(x.discoveredAt), + latestApi: ( + + ), + }; + }); + setLoading(false); + return { value: ret, total: total }; + } + + const key = startTimestamp + endTimestamp; + return ( + { }} + hideQueryField={true} + headings={headers} + useNewRow={true} + condensedHeight={true} + /> + ); +} + +export default ThreatActorTable; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApiSubcategoryCount.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApiSubcategoryCount.jsx new file mode 100644 index 0000000000..0fd745bc7e --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApiSubcategoryCount.jsx @@ -0,0 +1,25 @@ +import React, { useState, useEffect } from "react"; +import { Box, Card, DataTable } from "@shopify/polaris"; +import InfoCard from "../../dashboard/new_components/InfoCard"; + +const ThreatApiSubcategoryCount = ({ data }) => { + const Data = () => ( + [x.text, x.value]) + .sort((a, b) => b[0].localeCompare(a[0]))} + /> + ); + + return ( + } + /> + ); +}; + +export default ThreatApiSubcategoryCount; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApisTable.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApisTable.jsx new file mode 100644 index 0000000000..0e7ae16d47 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatApisTable.jsx @@ -0,0 +1,104 @@ +import { useEffect, useState } from "react"; +import GithubServerTable from "../../../components/tables/GithubServerTable"; +import api from "../api"; +import { CellType } from "../../../components/tables/rows/GithubRow"; +import GetPrettifyEndpoint from "../../observe/GetPrettifyEndpoint"; +import PersistStore from "../../../../main/PersistStore"; +import func from "../../../../../util/func"; + +const resourceName = { + singular: "api", + plural: "apis", +}; + +const headers = [ + { + text: "Endpoint", + value: "api", + title: "Endpoint", + }, + { + text: "Malicious Actors", + value: "actorsCount", + title: "Malicious Actors", + }, + { + text: "Malicious Requests", + value: "requestsCount", + title: "Malicious Requests", + }, + { + text: "Discovered", + title: "Discovered", + value: "discoveredAt", + type: CellType.TEXT, + sortActive: true, + }, +]; + +const sortOptions = []; + +let filters = []; + +function ThreatApiTable({ currDateRange, rowClicked }) { + const getTimeEpoch = (key) => { + return Math.floor(Date.parse(currDateRange.period[key]) / 1000); + }; + const startTimestamp = getTimeEpoch("since"); + const endTimestamp = getTimeEpoch("until"); + + const [loading, setLoading] = useState(true); + const collectionsMap = PersistStore((state) => state.collectionsMap); + const allCollections = PersistStore((state) => state.allCollections); + + useEffect(() => {}, []); + + function disambiguateLabel(key, value) { + return func.convertToDisambiguateLabelObj(value, null, 2); + } + + async function fetchData(sortKey, sortOrder, skip) { + setLoading(true); + const sort = { [sortKey]: sortOrder }; + const res = await api.fetchThreatApis(skip); + let total = res.total; + let ret = res?.apis?.map((x) => { + return { + ...x, + id: `${x.method}-${x.api}`, + actorsCount: x.actorsCount, + requestsCount: x.requestsCount, + discoveredAt: func.prettifyEpoch(x.discoveredAt), + api: ( + + ), + }; + }); + setLoading(false); + return { value: ret, total: total }; + } + + const key = startTimestamp + endTimestamp; + return ( + {}} + hideQueryField={true} + headings={headers} + useNewRow={true} + condensedHeight={true} + /> + ); +} + +export default ThreatApiTable; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatWorldMap.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatWorldMap.jsx new file mode 100644 index 0000000000..7b737a5228 --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/ThreatWorldMap.jsx @@ -0,0 +1,86 @@ +import { useEffect } from "react"; +import Highcharts from "highcharts/highmaps"; + +function ThreatWorldMap({ data, style, loading }) { + useEffect(() => { + const fetchMapData = async () => { + const topology = await fetch( + "https://code.highcharts.com/mapdata/custom/world.topo.json" + ).then((response) => response.json()); + + Highcharts.mapChart("threat-world-map-container", { + chart: { + map: topology, + backgroundColor: "#fff", + }, + + title: { + text: "Threat Actor Map", + }, + + credits: { + enabled: false, + }, + + subtitle: { + text: "", + }, + + legend: { + enabled: false, + }, + + mapNavigation: { + enabled: false, + }, + + mapView: { + fitToGeometry: { + type: "MultiPoint", + coordinates: [ + [-164, 54], // Alaska west + [-35, 84], // Greenland north + [179, -38], // New Zealand east + [-68, -55], // Chile south + ], + }, + }, + + series: [ + { + name: "Countries", + color: "#E0E0E0", + enableMouseTracking: false, + states: { + inactive: { + enabled: true, + opacity: 1, + }, + }, + }, + { + type: "mapbubble", + name: "", + data: data, + minSize: "4%", + maxSize: "4%", + joinBy: ["iso-a2", "code"], + marker: { + fillOpacity: 0.5, + lineWidth: 0, + }, + tooltip: { + pointFormat: "{point.name}
Actors: {point.count}", + }, + }, + ], + }); + }; + + fetchMapData(); + }, [data]); + + return
; +} + +export default ThreatWorldMap; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/TopThreatTypeChart.jsx b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/TopThreatTypeChart.jsx new file mode 100644 index 0000000000..43c1d9858c --- /dev/null +++ b/apps/dashboard/web/polaris_web/web/src/apps/dashboard/pages/threat_detection/components/TopThreatTypeChart.jsx @@ -0,0 +1,43 @@ +import React, { useEffect, useState } from "react"; +import InfoCard from "../../dashboard/new_components/InfoCard"; +import ChartypeComponent from "../../testing/TestRunsPage/ChartypeComponent"; +import BarGraph from "../../../components/charts/BarGraph"; + +const TopThreatTypeChart = ({ data }) => { + const [chartData, setChartData] = useState([]); + useEffect(() => { + const chartData = data + .sort((a, b) => a.text.localeCompare(b.text)) + .map((x) => ({ + text: x.text.replaceAll("_", " "), + value: x.value, + color: x.color, + })); + setChartData(chartData); + }, [data]); + return ( + + } + /> + ); +}; + +export default TopThreatTypeChart; diff --git a/apps/dashboard/web/polaris_web/web/src/apps/main/App.js b/apps/dashboard/web/polaris_web/web/src/apps/main/App.js index c6c5be0275..42bad35beb 100644 --- a/apps/dashboard/web/polaris_web/web/src/apps/main/App.js +++ b/apps/dashboard/web/polaris_web/web/src/apps/main/App.js @@ -78,6 +78,8 @@ import SignUpWithSSO from "../signup/components/SignUpWithSSO"; import TeamsWebhooks from "../dashboard/pages/settings/integrations/teamsWebhooks/TeamsWebhooks"; import TeamsWebhook from "../dashboard/pages/settings/integrations/teamsWebhooks/TeamsWebhook"; import AuditLogs from "../dashboard/pages/settings/audit_logs/AuditLogs"; +import ThreatApiPage from "../dashboard/pages/threat_detection/ThreatApiPage"; +import ThreatActorPage from "../dashboard/pages/threat_detection/ThreatActorPage"; // if you add a component in a new path, please verify the search implementation in function -> 'getSearchItemsArr' in func.js @@ -180,6 +182,14 @@ const router = createBrowserRouter([ path:"threat-detection", element: }, + { + path: "threat-api", + element: + }, + { + path: "threat-actor", + element: + } ] }, { diff --git a/apps/pom.xml b/apps/pom.xml index ca5ec582af..20dcc1cb26 100644 --- a/apps/pom.xml +++ b/apps/pom.xml @@ -49,15 +49,15 @@ - api-threat-detection + threat-detection - api-threat-detection/pom.xml + threat-detection/pom.xml - api-threat-detection - + threat-detection + source-code-analyser @@ -136,6 +136,16 @@ testing-cli + + threat-detection-backend + + + threat-detection-backend/pom.xml + + + + threat-detection-backend + + - diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 0000000000..f282e25075 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,18 @@ +version: v2 + +managed: + enabled: true + + override: + - file_option: java_multiple_files + value: true + + - file_option: java_package_prefix + value: com.akto.proto.generated + +plugins: + - remote: buf.build/grpc/java:v1.69.0 + out: libs/protobuf/src/main/java + + - remote: buf.build/protocolbuffers/java:v28.3 + out: libs/protobuf/src/main/java diff --git a/libs/pom.xml b/libs/pom.xml index 8bce86a189..2dfb6103a8 100644 --- a/libs/pom.xml +++ b/libs/pom.xml @@ -18,6 +18,7 @@ dao utils integrations + protobuf diff --git a/libs/protobuf/.gitignore b/libs/protobuf/.gitignore new file mode 100644 index 0000000000..9b21cfdc49 --- /dev/null +++ b/libs/protobuf/.gitignore @@ -0,0 +1,2 @@ +src/main/java/com/akto/proto/generated + diff --git a/libs/protobuf/pom.xml b/libs/protobuf/pom.xml new file mode 100644 index 0000000000..6943beca2a --- /dev/null +++ b/libs/protobuf/pom.xml @@ -0,0 +1,76 @@ + + + 4.0.0 + + com.akto.libs + libs + ${revision} + + + + 1.69.0 + + + + com.akto.libs.protobuf + protobuf + jar + + + + + io.grpc + grpc-netty-shaded + 1.68.1 + + + io.grpc + grpc-core + ${grpc.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-services + ${grpc.version} + + + com.google.protobuf + protobuf-java + 4.28.3 + + + com.google.protobuf + protobuf-java-util + 3.25.5 + compile + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 8 + 8 + + + + src/main/java + + + \ No newline at end of file diff --git a/libs/protobuf/src/main/java/com/akto/grpc/auth/AuthToken.java b/libs/protobuf/src/main/java/com/akto/grpc/auth/AuthToken.java new file mode 100644 index 0000000000..670e945419 --- /dev/null +++ b/libs/protobuf/src/main/java/com/akto/grpc/auth/AuthToken.java @@ -0,0 +1,51 @@ +package com.akto.grpc.auth; + +import static io.grpc.Metadata.ASCII_STRING_MARSHALLER; + +import io.grpc.CallCredentials; +import io.grpc.Metadata; +import io.grpc.Status; +import java.util.Optional; +import java.util.concurrent.Executor; + +public class AuthToken extends CallCredentials { + + private final String token; + public static final Metadata.Key AUTHORIZATION_METADATA_KEY = + Metadata.Key.of("Authorization", ASCII_STRING_MARSHALLER); + + public AuthToken(String token) { + if (token == null || token.trim().isEmpty()) { + throw new IllegalArgumentException("Token cannot be null or empty"); + } + this.token = String.format("Bearer %s", token.trim()); + } + + @Override + public void applyRequestMetadata( + RequestInfo requestInfo, Executor appExecutor, MetadataApplier applier) { + appExecutor.execute( + () -> { + try { + Metadata headers = new Metadata(); + headers.put(AUTHORIZATION_METADATA_KEY, token); + applier.apply(headers); + } catch (Throwable e) { + applier.fail(Status.UNAUTHENTICATED.withCause(e)); + } + }); + } + + public static Optional getBearerTokenFromMeta(Metadata metadata) { + String val = metadata.get(AUTHORIZATION_METADATA_KEY); + if (val == null || val.trim().isEmpty()) { + return Optional.empty(); + } + + if (val.startsWith("Bearer ")) { + return Optional.of(val.substring("Bearer ".length()).trim()); + } + + return Optional.empty(); + } +} diff --git a/libs/protobuf/src/main/java/com/akto/proto/utils/ProtoMessageUtils.java b/libs/protobuf/src/main/java/com/akto/proto/utils/ProtoMessageUtils.java new file mode 100644 index 0000000000..b6d4973e86 --- /dev/null +++ b/libs/protobuf/src/main/java/com/akto/proto/utils/ProtoMessageUtils.java @@ -0,0 +1,27 @@ +package com.akto.proto.utils; + +import com.google.protobuf.Message; +import com.google.protobuf.util.JsonFormat; +import java.util.Optional; + +public class ProtoMessageUtils { + public static Optional toString(Message msg) { + try { + return Optional.of(JsonFormat.printer().print(msg)); + } catch (Exception e) { + // Ignore + } + return Optional.empty(); + } + + public static Optional toProtoMessage(Class clz, String msg) { + try { + T.Builder builder = (T.Builder) clz.getMethod("newBuilder").invoke(null); + JsonFormat.parser().merge(msg, builder); + return Optional.of((T) builder.build()); + } catch (Exception e) { + // Ignore + } + return Optional.empty(); + } +} diff --git a/protobuf/buf.yaml b/protobuf/buf.yaml new file mode 100644 index 0000000000..534fcc9218 --- /dev/null +++ b/protobuf/buf.yaml @@ -0,0 +1,7 @@ +version: v2 +breaking: + use: + - FILE +lint: + use: + - STANDARD diff --git a/protobuf/threat_detection/message/malicious_event/event_type/v1/event_type.proto b/protobuf/threat_detection/message/malicious_event/event_type/v1/event_type.proto new file mode 100644 index 0000000000..5887f5c6e4 --- /dev/null +++ b/protobuf/threat_detection/message/malicious_event/event_type/v1/event_type.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package threat_detection.message.malicious_event.event_type.v1; + +option java_outer_classname = "MaliciousEventProto"; +option java_package = "threat_detection.message.malicious_event.event_type.v1"; + +enum EventType { + EVENT_TYPE_UNSPECIFIED = 0; + EVENT_TYPE_SINGLE = 1; + EVENT_TYPE_AGGREGATED = 2; +} diff --git a/protobuf/threat_detection/message/malicious_event/v1/message.proto b/protobuf/threat_detection/message/malicious_event/v1/message.proto new file mode 100644 index 0000000000..238836ebb5 --- /dev/null +++ b/protobuf/threat_detection/message/malicious_event/v1/message.proto @@ -0,0 +1,28 @@ +syntax = "proto3"; + +package threat_detection.message.malicious_event.v1; + +import "threat_detection/message/malicious_event/event_type/v1/event_type.proto"; + +option java_outer_classname = "MaliciousEventProto"; +option java_package = "threat_detection.message.malicious_event.v1"; + +message MaliciousEventMessage { + string actor = 1; + string filter_id = 2; + int64 detected_at = 3; + string latest_api_ip = 4; + string latest_api_endpoint = 5; + string latest_api_method = 6; + int32 latest_api_collection_id = 7; + string latest_api_payload = 8; + threat_detection.message.malicious_event.event_type.v1.EventType event_type = 9; + string category = 10; + string sub_category = 11; +} + +message MaliciousEventKafkaEnvelope { + string account_id = 1; + string actor = 2; + MaliciousEventMessage malicious_event = 3; +} diff --git a/protobuf/threat_detection/message/sample_request/v1/message.proto b/protobuf/threat_detection/message/sample_request/v1/message.proto new file mode 100644 index 0000000000..4192572509 --- /dev/null +++ b/protobuf/threat_detection/message/sample_request/v1/message.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package threat_detection.message.sample_request.v1; + +option java_outer_classname = "SampleRequestProto"; +option java_package = "threat_detection.message.sample_request.v1"; + +message SampleMaliciousRequest { + string ip = 1; + int64 timestamp = 2; + string url = 3; + string method = 4; + int32 api_collection_id = 5; + string payload = 6; + string filter_id = 7; +} + +message SampleRequestKafkaEnvelope { + string account_id = 1; + string actor = 2; + SampleMaliciousRequest malicious_request = 3; +} diff --git a/protobuf/threat_detection/service/dashboard_service/v1/service.proto b/protobuf/threat_detection/service/dashboard_service/v1/service.proto new file mode 100644 index 0000000000..983cf18be7 --- /dev/null +++ b/protobuf/threat_detection/service/dashboard_service/v1/service.proto @@ -0,0 +1,106 @@ +syntax = "proto3"; + +package threat_detection.service.dashboard_service.v1; + +import "threat_detection/message/malicious_event/event_type/v1/event_type.proto"; + +// Dashboard service which the dashboard actions will call instead of directly calling DB +option java_outer_classname = "DashboardServiceProto"; +option java_package = "threat_detection.service.dashboard_service.v1"; + +message ListMaliciousRequestsResponse { + message MaliciousEvent { + string id = 1; + string actor = 2; + string filter_id = 3; + int64 detected_at = 4; + string ip = 5; + string endpoint = 6; + string method = 7; + int32 api_collection_id = 8; + string payload = 9; + string country = 10; + threat_detection.message.malicious_event.event_type.v1.EventType event_type = 11; + string category = 12; + string sub_category = 13; + } + repeated MaliciousEvent malicious_events = 1; + int32 total = 2; +} + +message ListMaliciousRequestsRequest { + // The number of alerts to return + optional uint32 skip = 1; + uint32 limit = 2; + map sort = 3; +} + +message FetchAlertFiltersRequest {} + +message FetchAlertFiltersResponse { + repeated string actors = 1; + repeated string urls = 2; +} + +message ListThreatActorsRequest { + optional uint32 skip = 1; + uint32 limit = 2; + map sort = 3; +} + +message ListThreatActorResponse { + message ThreatActor { + string id = 1; + string latest_api_ip = 2; + string latest_api_endpoint = 3; + string latest_api_method = 4; + uint64 discovered_at = 5; + string country = 6; + } + repeated ThreatActor actors = 1; + int32 total = 2; +} + +message ListThreatApiRequest { + optional uint32 skip = 1; + uint32 limit = 2; + map sort = 3; +} + +message ListThreatApiResponse { + message ThreatApi { + string endpoint = 1; + string method = 2; + uint64 discovered_at = 3; + uint32 actors_count = 4; + uint32 requests_count = 5; + } + + repeated ThreatApi apis = 1; + int32 total = 2; +} + +message ThreatActorByCountryRequest { +} + +message ThreatActorByCountryResponse { + message CountryCount { + string code = 1; + uint32 count = 2; + } + + repeated CountryCount countries = 1; +} + +message ThreatCategoryWiseCountRequest { +} + +message ThreatCategoryWiseCountResponse { + message SubCategoryCount { + string category = 1; + string sub_category = 2; + uint32 count = 3; + } + + repeated SubCategoryCount category_wise_counts = 1; +} diff --git a/protobuf/threat_detection/service/malicious_alert_service/v1/service.proto b/protobuf/threat_detection/service/malicious_alert_service/v1/service.proto new file mode 100644 index 0000000000..bde487cfe1 --- /dev/null +++ b/protobuf/threat_detection/service/malicious_alert_service/v1/service.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package threat_detection.service.malicious_alert_service.v1; + +import "threat_detection/message/malicious_event/v1/message.proto"; +import "threat_detection/message/sample_request/v1/message.proto"; + +// This is a consumer service for recording malicious alerts +// For dashboard purposes we will have a separate service to retrieve these events. +option java_outer_classname = "MaliciousAlertServiceProto"; +option java_package = "threat_detection.service.malicious_alert_service.v1"; + +message RecordMaliciousEventResponse { +} + +message RecordMaliciousEventRequest { + threat_detection.message.malicious_event.v1.MaliciousEventMessage malicious_event = 1; + repeated threat_detection.message.sample_request.v1.SampleMaliciousRequest sample_requests = 2; +} diff --git a/scripts/proto-gen.sh b/scripts/proto-gen.sh new file mode 100644 index 0000000000..ff32a1f164 --- /dev/null +++ b/scripts/proto-gen.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Check if buf is installed or not +# Please install buf if not already installed by following the instructions at https://docs.buf.build/installation +if ! command -v buf >/dev/null 2>&1; then + echo "Please install buf if not already installed by following the instructions at https://docs.buf.build/installation" + exit +fi + +buf lint protobuf +rm -rf libs/protobuf/src/main/java/com/akto/proto/generated/ +buf generate protobuf \ No newline at end of file