Skip to content
This repository has been archived by the owner on Mar 12, 2024. It is now read-only.

Commit

Permalink
Merge pull request #12 from ThalesGroup/feature/k8s-extension
Browse files Browse the repository at this point in the history 
k8s extension - experiments on multiple namespaces
  • Loading branch information
a-mantha authored Nov 27, 2020
2 parents b3a7dc1 + 4cacadc commit b87d548
Show file tree
Hide file tree
Showing 5 changed files with 260 additions and 139 deletions.
2 changes: 1 addition & 1 deletion chaosengine-experiments/chaosengine-kubernetes/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<dependency>
<groupId>io.kubernetes</groupId>
<artifactId>client-java</artifactId>
<version>9.0.2</version>
<version>10.0.0</version>
<scope>compile</scope>
</dependency>
<!-- Kubernetes Dependencies -->
Expand Down
16 changes: 14 additions & 2 deletions ...gine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public class KubernetesPodContainer extends Container {
private String podName;
@Identifier(order = 2)
private String namespace;
private Map<String, String> labels = new HashMap<>();
private final Map<String, String> labels = new HashMap<>();
private boolean isBackedByController = false;
private KubernetesPlatform kubernetesPlatform;
@Identifier(order = 3)
Expand All @@ -52,7 +52,8 @@ public class KubernetesPodContainer extends Container {
private String ownerName;
private Collection<String> subcontainers = new HashSet<>();
private String targetedSubcontainer;
private Callable<ContainerHealth> replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this);
private final Callable<ContainerHealth> replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this);
private String parentNode;

private KubernetesPodContainer () {
super();
Expand Down Expand Up @@ -82,6 +83,10 @@ public String getOwnerName () {
return ownerName;
}

public String getParentNode () {
return parentNode;
}

@Override
public void startExperiment (Experiment experiment) {
this.targetedSubcontainer = null;
Expand Down Expand Up @@ -153,6 +158,7 @@ public static final class KubernetesPodContainerBuilder {
private String ownerKind;
private String ownerName;
private Collection<String> subcontainers;
private String parentNode;

private KubernetesPodContainerBuilder () {
}
Expand Down Expand Up @@ -211,6 +217,11 @@ public KubernetesPodContainerBuilder withUUID (String uuid) {
return this;
}

public KubernetesPodContainerBuilder withParentNode (String parentNode) {
this.parentNode = parentNode;
return this;
}

public KubernetesPodContainer build () {
KubernetesPodContainer kubernetesPodContainer = new KubernetesPodContainer();
kubernetesPodContainer.uuid = this.uuid;
Expand All @@ -223,6 +234,7 @@ public KubernetesPodContainer build () {
kubernetesPodContainer.ownerKind = ControllerKind.mapFromString(this.ownerKind);
kubernetesPodContainer.ownerName = ownerName;
kubernetesPodContainer.subcontainers = this.subcontainers;
kubernetesPodContainer.parentNode = this.parentNode;
try {
kubernetesPodContainer.setMappedDiagnosticContext();
kubernetesPodContainer.log.info("Created new Kubernetes Pod Container object");
Expand Down
119 changes: 74 additions & 45 deletions ...aosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@

import static com.thales.chaos.constants.DataDogConstants.DATADOG_CONTAINER_KEY;
import static com.thales.chaos.exception.enums.KubernetesChaosErrorCode.K8S_API_ERROR;
import static java.util.function.Predicate.not;
import static net.logstash.logback.argument.StructuredArguments.v;

@Component
Expand All @@ -57,31 +58,35 @@
public class KubernetesPlatform extends Platform implements ShellBasedExperiment<KubernetesPodContainer> {
@Autowired
private ContainerManager containerManager;

@Autowired
private ApiClient apiClient;
private String namespace = "default";
private final ApiClient apiClient;
static final String DEFAULT_NAMESPACE = "default";
static final String UNKNOWN_PARENT_NODE = "UNKNOWN";
private Collection<String> namespaces = List.of(DEFAULT_NAMESPACE);

@Autowired
KubernetesPlatform (ApiClient apiClient) {
this.apiClient = apiClient;
log.info("Kubernetes Platform created");
public Collection<String> getNamespaces () {
return namespaces;
}

public String getNamespace () {
return namespace;
public void setNamespaces (String namespaces) {
this.namespaces = Optional.of(Arrays.stream(namespaces.split(","))
.filter(Objects::nonNull)
.filter(not(String::isEmpty))
.collect(Collectors.toList()))
.filter(l -> !l.isEmpty())
.orElse(List.of(DEFAULT_NAMESPACE));
}

public void setNamespace (String namespace) {
this.namespace = namespace;
@Autowired
KubernetesPlatform (ApiClient apiClient) {
this.apiClient = apiClient;
log.info("Kubernetes Platform created");
}

public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContainer) {
try {
Optional<Boolean> podExists = podExists(kubernetesPodContainer);
if (podExists.isEmpty()) {
return ContainerHealth.RUNNING_EXPERIMENT;
} else if (!podExists.get()) {
boolean podExists = podExists(kubernetesPodContainer);
if (!podExists) {
return ContainerHealth.DOES_NOT_EXIST;
}
V1Pod result = getCoreV1Api().readNamespacedPodStatus(kubernetesPodContainer.getPodName(),
Expand All @@ -103,31 +108,50 @@ public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContaine
return ContainerHealth.RUNNING_EXPERIMENT;
}

Optional<Boolean> podExists (KubernetesPodContainer kubernetesPodContainer) {
boolean podExists (KubernetesPodContainer kubernetesPodContainer) {
String podUuid = kubernetesPodContainer.getUuid();
if (podUuid == null) return Optional.empty();
try {
Boolean podExists = listAllPodsInNamespace().getItems()
.stream()
.map(V1Pod::getMetadata)
.filter(Objects::nonNull)
.map(V1ObjectMeta::getUid)
.anyMatch(podUuid::equals);
log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists);
return Optional.of(podExists);
} catch (ApiException e) {
log.debug("Exception when checking container existence", e);
return Optional.empty();
if (podUuid == null) {
return false;
}
boolean podExists = listAllPodsInNamespace(kubernetesPodContainer.getNamespace()).getItems()
.stream()
.map(V1Pod::getMetadata)
.filter(Objects::nonNull)
.map(V1ObjectMeta::getUid)
.anyMatch(podUuid::equals);
log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists);
return podExists;
}

@JsonIgnore
CoreV1Api getCoreV1Api () {
return new CoreV1Api(apiClient);
}

private V1PodList listAllPodsInNamespace () throws ApiException {
return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false);
private V1PodList listAllPodsInNamespace (String namespace) {
try {
return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false);
} catch (ApiException e) {
log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e);
return new V1PodList();
}
}

@Override
public PlatformHealth getPlatformHealth () {
if (namespaces.stream().map(this::canListPodsInNamespace).anyMatch(canList -> canList.equals(false))) {
return PlatformHealth.FAILED;
}
if (namespaces.stream()
.map(this::listAllPodsInNamespace)
.map(V1PodList::getItems)
.flatMap(List::stream)
.collect(Collectors.toList())
.isEmpty()) {
log.warn("No PODs detected in specified namespaces {}", namespaces);
return PlatformHealth.DEGRADED;
}
return PlatformHealth.OK;
}

@Override
Expand All @@ -151,25 +175,28 @@ public PlatformLevel getPlatformLevel () {
return PlatformLevel.PAAS;
}

@Override
public PlatformHealth getPlatformHealth () {
private boolean canListPodsInNamespace (String namespace) {
try {
V1PodList pods = listAllPodsInNamespace();
return (!pods.getItems().isEmpty()) ? PlatformHealth.OK : PlatformHealth.DEGRADED;
} catch (ApiException e) {
log.error("Kubernetes Platform health check failed", e);
return PlatformHealth.FAILED;
getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false);
} catch (Exception e) {
log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e);
return false;
}
return true;
}

@Override
protected List<Container> generateRoster () {
final List<Container> containerList = new ArrayList<>();
try {
V1PodList pods = listAllPodsInNamespace();
containerList.addAll(pods.getItems().stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet()));
List<V1Pod> pods = namespaces.stream()
.map(this::listAllPodsInNamespace)
.map(V1PodList::getItems)
.flatMap(List::stream)
.collect(Collectors.toList());
containerList.addAll(pods.stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet()));
return containerList;
} catch (ApiException e) {
} catch (Exception e) {
log.error("Could not generate Kubernetes roster", e);
return containerList;
}
Expand All @@ -178,7 +205,7 @@ protected List<Container> generateRoster () {
@Override
public boolean isContainerRecycled (Container container) {
KubernetesPodContainer kubernetesPodContainer = (KubernetesPodContainer) container;
if (podExists(kubernetesPodContainer).orElse(false)) return isContainerRestarted(kubernetesPodContainer,
if (podExists(kubernetesPodContainer)) return isContainerRestarted(kubernetesPodContainer,
((KubernetesPodContainer) container).getTargetedSubcontainer());
return isDesiredReplicas(kubernetesPodContainer);
}
Expand All @@ -195,11 +222,11 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) {
.withKubernetesPlatform(this)
.isBackedByController(CollectionUtils.isNotEmpty(pod.getMetadata()
.getOwnerReferences()))
.withOwnerKind(Optional.of(pod.getMetadata().getOwnerReferences())
.withOwnerKind(Optional.ofNullable(pod.getMetadata().getOwnerReferences())
.flatMap(list -> list.stream().findFirst())
.map(V1OwnerReference::getKind)
.orElse(""))
.withOwnerName(Optional.of(pod.getMetadata().getOwnerReferences())
.withOwnerName(Optional.ofNullable(pod.getMetadata().getOwnerReferences())
.flatMap(list -> list.stream().findFirst())
.map(V1OwnerReference::getName)
.orElse(""))
Expand All @@ -210,6 +237,8 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) {
.flatMap(Collection::stream)
.map(V1Container::getName)
.collect(Collectors.toList()))
.withParentNode(Optional.ofNullable(pod.getSpec().getNodeName())
.orElse(UNKNOWN_PARENT_NODE))
.build();
log.info("Found new Kubernetes Pod Container {}", v(DATADOG_CONTAINER_KEY, container));
containerManager.offer(container);
Expand Down Expand Up @@ -247,7 +276,7 @@ private boolean isContainerRestarted (KubernetesPodContainer container, String s
}

public ContainerHealth replicaSetRecovered (KubernetesPodContainer kubernetesPodContainer) {
return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer).orElse(false) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT;
return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT;
}

/**
Expand Down
Loading

0 comments on commit b87d548

Please sign in to comment.