feat(ecs): update application caching agent to store application and …

…relationships (spinnaker#5234)

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/cache/client/ApplicationCacheClient.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.netflix.spinnaker.clouddriver.ecs.cache.client;
+
+import static com.netflix.spinnaker.clouddriver.ecs.EcsCloudProvider.ID;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.APPLICATIONS;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
+import static java.util.stream.Collectors.toSet;
+
+import com.netflix.spinnaker.cats.cache.Cache;
+import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.RelationshipCacheFilter;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import com.netflix.spinnaker.clouddriver.ecs.model.EcsApplication;
+import com.netflix.spinnaker.clouddriver.model.Application;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+@Slf4j
+@Component
+public class ApplicationCacheClient extends AbstractCacheClient<Application> {
+
+  @Autowired
+  public ApplicationCacheClient(Cache cacheView) {
+    super(cacheView, APPLICATIONS.toString());
+  }
+
+  @Override
+  protected Application convert(CacheData cacheData) {
+    log.debug("Translating CacheData to EcsApplication");
+    if (cacheData == null) {
+      return null;
+    }
+
+    String appName = (String) cacheData.getAttributes().get("name");
+
+    Map<String, String> attributes = new HashMap<>();
+    attributes.put("name", appName);
+
+    Map<String, Set<String>> clusterNames = new HashMap<>();
+
+    EcsApplication application = new EcsApplication(appName, attributes, clusterNames);
+
+    Set<String> services = getServiceRelationships(cacheData);
+    log.info("Found {} services for app {}", services.size(), appName);
+    services.forEach(
+        key -> {
+          Map<String, String> parsedKey = Keys.parse(key);
+          if (application.getClusterNames().get(parsedKey.get("account")) != null) {
+            application
+                .getClusterNames()
+                .get(parsedKey.get("account"))
+                .add(parsedKey.get("serviceName"));
+          } else {
+            application
+                .getClusterNames()
+                .put(
+                    parsedKey.get("account"),
+                    new HashSet<>(Arrays.asList(parsedKey.get("serviceName"))));
+          }
+        });
+
+    log.info(
+        "Found {} clusterNames for application {}",
+        application.getClusterNames(),
+        application.getName());
+    return application;
+  }
+
+  public Application getApplication(String name) {
+    return convert(
+        cacheView.get(
+            APPLICATIONS.ns,
+            Keys.getApplicationKey(name),
+            RelationshipCacheFilter.include(SERVICES.ns)));
+  }
+
+  public Set<Application> getApplications(boolean expand) {
+    RelationshipCacheFilter relationshipFilter =
+        expand ? RelationshipCacheFilter.include(SERVICES.ns) : RelationshipCacheFilter.none();
+    Collection<CacheData> applications =
+        cacheView.getAll(
+            APPLICATIONS.ns,
+            cacheView.filterIdentifiers(APPLICATIONS.ns, ID + ";*"),
+            relationshipFilter);
+    log.info("getApplications found {} Spinnaker applications in the cache.", applications.size());
+    return applications.stream().map(this::convert).collect(toSet());
+  }
+
+  private Set<String> getServiceRelationships(CacheData cacheData) {
+    Collection<String> serviceRelationships = cacheData.getRelationships().get(SERVICES.ns);
+    return serviceRelationships == null
+        ? Collections.emptySet()
+        : new HashSet<>(serviceRelationships);
+  }
+}

clouddriver-ecs/src/main/java/com/netflix/spinnaker/clouddriver/ecs/provider/agent/ApplicationCachingAgent.java

@@ -1,35 +1,39 @@
 /*
- * Copyright 2021 Amazon.com, Inc.
+ * Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
  *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ *    http://www.apache.org/licenses/LICENSE-2.0
  *
+ * This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+ * CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under
+ * the License.
  */
 
 package com.netflix.spinnaker.clouddriver.ecs.provider.agent;
 
 import static com.netflix.spinnaker.cats.agent.AgentDataType.Authority.AUTHORITATIVE;
 import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.APPLICATIONS;
+import static com.netflix.spinnaker.clouddriver.ecs.cache.Keys.Namespace.SERVICES;
 
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.ecs.AmazonECS;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Sets;
 import com.netflix.spectator.api.Registry;
 import com.netflix.spinnaker.cats.agent.AgentDataType;
 import com.netflix.spinnaker.cats.cache.CacheData;
+import com.netflix.spinnaker.cats.cache.DefaultCacheData;
 import com.netflix.spinnaker.cats.provider.ProviderCache;
 import com.netflix.spinnaker.clouddriver.aws.security.AmazonClientProvider;
 import com.netflix.spinnaker.clouddriver.aws.security.NetflixAmazonCredentials;
+import com.netflix.spinnaker.clouddriver.ecs.cache.Keys;
+import com.netflix.spinnaker.clouddriver.ecs.cache.client.ServiceCacheClient;
 import com.netflix.spinnaker.clouddriver.ecs.cache.model.Application;
+import com.netflix.spinnaker.clouddriver.ecs.cache.model.Service;
 import java.util.*;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -71,7 +75,42 @@ public String getAgentType() {
 
   @Override
   protected List<Application> getItems(AmazonECS ecs, ProviderCache providerCache) {
+    // get all ECS services
+    ServiceCacheClient serviceCacheClient = new ServiceCacheClient(providerCache, objectMapper);
+    Collection<Service> services = serviceCacheClient.getAll();
+    log.info("Found {} ECS services for which to cache applications", services.size());
+
+    Map<String, Map<String, Collection<String>>> appRelationships = new HashMap<>();
+
+    for (Service service : services) {
+      String applicationKey = service.getApplicationName();
+      String serviceKey =
+          Keys.getServiceKey(service.getAccount(), service.getRegion(), service.getServiceName());
+
+      appRelationships.put(
+          applicationKey,
+          updateApplicationRelationships(appRelationships, applicationKey, serviceKey)
+              .get(applicationKey));
+
+      log.debug(
+          "ECS application "
+              + applicationKey
+              + " with "
+              + appRelationships.get(applicationKey).size()
+              + " relationships");
+    }
+
     List<Application> applications = new ArrayList<>();
+
+    for (Map.Entry<String, Map<String, Collection<String>>> appInfo : appRelationships.entrySet()) {
+      Application application = new Application();
+      application.setName(appInfo.getKey());
+      application.setRelationships(appInfo.getValue());
+
+      applications.add(application);
+    }
+
+    log.info("Cached {} applications for {} services", applications.size(), services.size());
     return applications;
   }
 
@@ -80,10 +119,44 @@ protected Map<String, Collection<CacheData>> generateFreshData(
       Collection<Application> applications) {
     Collection<CacheData> applicationData = new LinkedList<>();
 
+    for (Application application : applications) {
+      Map<String, Object> attributes = convertApplicationToAttributes(application);
+      String applicationKey = Keys.getApplicationKey(application.getName());
+
+      applicationData.add(
+          new DefaultCacheData(applicationKey, attributes, application.getRelationships()));
+    }
+
     Map<String, Collection<CacheData>> cacheDataMap = new HashMap<>();
-    log.info("Amazon ECS ApplicationCachingAgent will cache applications in a future update");
+    log.info("Caching " + applicationData.size() + " ECS applications in " + getAgentType());
     cacheDataMap.put(APPLICATIONS.toString(), applicationData);
 
     return cacheDataMap;
   }
+
+  private Map<String, Map<String, Collection<String>>> updateApplicationRelationships(
+      Map<String, Map<String, Collection<String>>> appRelationships,
+      String applicationKey,
+      String serviceKey) {
+    Map<String, Collection<String>> existingAppRelation = appRelationships.get(applicationKey);
+    if (existingAppRelation != null && existingAppRelation.size() > 0) {
+      log.debug("Updating existing application relation for " + applicationKey);
+
+      Collection<String> serviceRelationship = existingAppRelation.get(SERVICES.ns);
+      if (serviceRelationship != null && !serviceRelationship.isEmpty()) {
+        serviceRelationship.add(serviceKey);
+      } else {
+        serviceRelationship = Sets.newHashSet(serviceKey);
+      }
+      existingAppRelation.put(SERVICES.ns, serviceRelationship);
+    } else {
+      log.debug("Creating new application relation for " + applicationKey);
+
+      existingAppRelation = new HashMap<String, Collection<String>>();
+      existingAppRelation.put(SERVICES.ns, Sets.newHashSet(serviceKey));
+    }
+
+    appRelationships.put(applicationKey, existingAppRelation);
+    return appRelationships;
+  }
 }