Skip to content

Commit

Permalink
feat(ecs): update application caching agent to store application and …
Browse files Browse the repository at this point in the history
…relationships (#5234)
  • Loading branch information
piradeepk committed Apr 21, 2021
1 parent 8c377ef commit 10f61d5
Show file tree
Hide file tree
Showing 13 changed files with 639 additions and 150 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@

public class Keys implements KeyParser {
public enum Namespace {
APPLICATIONS,
ECS_APPLICATIONS,
IAM_ROLE,
SERVICES,
ECS_CLUSTERS,
Expand Down Expand Up @@ -98,7 +98,7 @@ public static Map<String, String> parse(String key) {
Namespace.valueOf(CaseFormat.LOWER_CAMEL.to(CaseFormat.UPPER_UNDERSCORE, parts[1]));

switch (namespace) {
case APPLICATIONS:
case ECS_APPLICATIONS:
result.put("application", parts[2]);
break;
case SERVICES:
Expand Down Expand Up @@ -176,7 +176,7 @@ public static String getClusterKey(String account, String region, String cluster
}

public static String getApplicationKey(String name) {
return ID + SEPARATOR + Namespace.APPLICATIONS + SEPARATOR + name.toLowerCase();
return ID + SEPARATOR + Namespace.ECS_APPLICATIONS + SEPARATOR + name.toLowerCase();
}

public static String getTaskKey(String account, String region, String taskId) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
/*
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License
* for the specific language governing permissions and limitations under
* the License.
*/

package com.netflix.spinnaker.clouddriver.ecs.cache.client;

import static com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider.ID;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_APPLICATIONS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
import static java.util.stream.Collectors.toSet;

import com.netflix.spinnaker.cats.cache.Cache;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.cache.RelationshipCacheFilter;
import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
import com.netflix.spinnaker.clouddriver.ecs.model.EcsApplication;
import com.netflix.spinnaker.clouddriver.model.Application;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

@Slf4j
@Component
public class ApplicationCacheClient extends AbstractCacheClient<Application> {

@Autowired
public ApplicationCacheClient(Cache cacheView) {
super(cacheView, ECS_APPLICATIONS.toString());
}

@Override
protected Application convert(CacheData cacheData) {
log.debug("Translating CacheData to EcsApplication");
if (cacheData == null) {
return null;
}

String appName = (String) cacheData.getAttributes().get("name");

Map<String, String> attributes = new HashMap<>();
attributes.put("name", appName);

Map<String, Set<String>> clusterNames = new HashMap<>();

EcsApplication application = new EcsApplication(appName, attributes, clusterNames);

Set<String> services = getServiceRelationships(cacheData);
log.info("Found {} services for app {}", services.size(), appName);
services.forEach(
key -> {
Map<String, String> parsedKey = Keys.parse(key);
if (application.getClusterNames().get(parsedKey.get("account")) != null) {
application
.getClusterNames()
.get(parsedKey.get("account"))
.add(parsedKey.get("serviceName"));
} else {
application
.getClusterNames()
.put(
parsedKey.get("account"),
new HashSet<>(Arrays.asList(parsedKey.get("serviceName"))));
}
});

log.info(
"Found {} clusterNames for application {}",
application.getClusterNames(),
application.getName());
return application;
}

public Application getApplication(String name) {
return convert(
cacheView.get(
ECS_APPLICATIONS.ns,
Keys.getApplicationKey(name),
RelationshipCacheFilter.include(SERVICES.ns)));
}

public Set<Application> getApplications(boolean expand) {
RelationshipCacheFilter relationshipFilter =
expand ? RelationshipCacheFilter.include(SERVICES.ns) : RelationshipCacheFilter.none();
Collection<CacheData> applications =
cacheView.getAll(
ECS_APPLICATIONS.ns,
cacheView.filterIdentifiers(ECS_APPLICATIONS.ns, ID + ";*"),
relationshipFilter);
log.info("getApplications found {} Spinnaker applications in the cache.", applications.size());
return applications.stream().map(this::convert).collect(toSet());
}

private Set<String> getServiceRelationships(CacheData cacheData) {
Collection<String> serviceRelationships = cacheData.getRelationships().get(SERVICES.ns);
return serviceRelationships == null
? Collections.emptySet()
: new HashSet<>(serviceRelationships);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
package com.netflix.spinnaker.clouddriver.ecs.provider.agent;

import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.APPLICATIONS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_APPLICATIONS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_CLUSTERS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.IAM_ROLE;

Expand Down Expand Up @@ -221,7 +221,7 @@ protected boolean keyAccountRegionFilter(String authoritativeKeyName, String key
// Application keys are not account or region specific so this will be true. The region
// and
// account will be checked for other keys.
|| (authoritativeKeyName.equals(APPLICATIONS.ns)));
|| (authoritativeKeyName.equals(ECS_APPLICATIONS.ns)));
}

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -1,43 +1,47 @@
/*
* Copyright 2021 Amazon.com, Inc.
* Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
* http://www.apache.org/licenses/LICENSE-2.0
*
* This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
* CONDITIONS OF ANY KIND, either express or implied. See the License
* for the specific language governing permissions and limitations under
* the License.
*/

package com.netflix.spinnaker.clouddriver.ecs.provider.agent;

import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.APPLICATIONS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.ECS_APPLICATIONS;
import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;

import com.amazonaws.auth.AWSCredentialsProvider;
import com.amazonaws.services.ecs.AmazonECS;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Sets;
import com.netflix.spectator.api.Registry;
import com.netflix.spinnaker.cats.agent.AgentDataType;
import com.netflix.spinnaker.cats.cache.CacheData;
import com.netflix.spinnaker.cats.cache.DefaultCacheData;
import com.netflix.spinnaker.cats.provider.ProviderCache;
import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient;
import com.netflix.spinnaker.clouddriver.ecs.cache.model.Application;
import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

public class ApplicationCachingAgent extends AbstractEcsOnDemandAgent<Application> {
private static final Collection<AgentDataType> types =
Collections.unmodifiableCollection(
Arrays.asList(AUTHORITATIVE.forType(APPLICATIONS.toString())));
Arrays.asList(AUTHORITATIVE.forType(ECS_APPLICATIONS.toString())));
private final Logger log = LoggerFactory.getLogger(getClass());

private ObjectMapper objectMapper;
Expand Down Expand Up @@ -71,7 +75,42 @@ public String getAgentType() {

@Override
protected List<Application> getItems(AmazonECS ecs, ProviderCache providerCache) {
// get all ECS services
ServiceCacheClient serviceCacheClient = new ServiceCacheClient(providerCache, objectMapper);
Collection<Service> services = serviceCacheClient.getAll();
log.info("Found {} ECS services for which to cache applications", services.size());

Map<String, Map<String, Collection<String>>> appRelationships = new HashMap<>();

for (Service service : services) {
String applicationKey = service.getApplicationName();
String serviceKey =
Keys.getServiceKey(service.getAccount(), service.getRegion(), service.getServiceName());

appRelationships.put(
applicationKey,
updateApplicationRelationships(appRelationships, applicationKey, serviceKey)
.get(applicationKey));

log.debug(
"ECS application "
+ applicationKey
+ " with "
+ appRelationships.get(applicationKey).size()
+ " relationships");
}

List<Application> applications = new ArrayList<>();

for (Map.Entry<String, Map<String, Collection<String>>> appInfo : appRelationships.entrySet()) {
Application application = new Application();
application.setName(appInfo.getKey());
application.setRelationships(appInfo.getValue());

applications.add(application);
}

log.info("Cached {} applications for {} services", applications.size(), services.size());
return applications;
}

Expand All @@ -80,10 +119,44 @@ protected Map<String, Collection<CacheData>> generateFreshData(
Collection<Application> applications) {
Collection<CacheData> applicationData = new LinkedList<>();

for (Application application : applications) {
Map<String, Object> attributes = convertApplicationToAttributes(application);
String applicationKey = Keys.getApplicationKey(application.getName());

applicationData.add(
new DefaultCacheData(applicationKey, attributes, application.getRelationships()));
}

Map<String, Collection<CacheData>> cacheDataMap = new HashMap<>();
log.info("Amazon ECS ApplicationCachingAgent will cache applications in a future update");
cacheDataMap.put(APPLICATIONS.toString(), applicationData);
log.info("Caching " + applicationData.size() + " ECS applications in " + getAgentType());
cacheDataMap.put(ECS_APPLICATIONS.toString(), applicationData);

return cacheDataMap;
}

private Map<String, Map<String, Collection<String>>> updateApplicationRelationships(
Map<String, Map<String, Collection<String>>> appRelationships,
String applicationKey,
String serviceKey) {
Map<String, Collection<String>> existingAppRelation = appRelationships.get(applicationKey);
if (existingAppRelation != null && existingAppRelation.size() > 0) {
log.debug("Updating existing application relation for " + applicationKey);

Collection<String> serviceRelationship = existingAppRelation.get(SERVICES.ns);
if (serviceRelationship != null && !serviceRelationship.isEmpty()) {
serviceRelationship.add(serviceKey);
} else {
serviceRelationship = Sets.newHashSet(serviceKey);
}
existingAppRelation.put(SERVICES.ns, serviceRelationship);
} else {
log.debug("Creating new application relation for " + applicationKey);

existingAppRelation = new HashMap<String, Collection<String>>();
existingAppRelation.put(SERVICES.ns, Sets.newHashSet(serviceKey));
}

appRelationships.put(applicationKey, existingAppRelation);
return appRelationships;
}
}
Loading

0 comments on commit 10f61d5

Please sign in to comment.