diff --git a/chaosengine-experiments/chaosengine-kubernetes/pom.xml b/chaosengine-experiments/chaosengine-kubernetes/pom.xml index 6a035b124..589051e38 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/pom.xml +++ b/chaosengine-experiments/chaosengine-kubernetes/pom.xml @@ -15,7 +15,7 @@ io.kubernetes client-java - 9.0.2 + 10.0.0 compile diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java index ee9f08901..9bc900036 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/container/impl/KubernetesPodContainer.java @@ -43,7 +43,7 @@ public class KubernetesPodContainer extends Container { private String podName; @Identifier(order = 2) private String namespace; - private Map labels = new HashMap<>(); + private final Map labels = new HashMap<>(); private boolean isBackedByController = false; private KubernetesPlatform kubernetesPlatform; @Identifier(order = 3) @@ -52,7 +52,8 @@ public class KubernetesPodContainer extends Container { private String ownerName; private Collection subcontainers = new HashSet<>(); private String targetedSubcontainer; - private Callable replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this); + private final Callable replicaSetRecovered = () -> kubernetesPlatform.replicaSetRecovered(this); + private String parentNode; private KubernetesPodContainer () { super(); @@ -82,6 +83,10 @@ public String getOwnerName () { return ownerName; } + public String getParentNode () { + return parentNode; + } + @Override public void startExperiment (Experiment experiment) { this.targetedSubcontainer = null; @@ -153,6 +158,7 @@ public static final class KubernetesPodContainerBuilder { private String ownerKind; private String ownerName; private Collection subcontainers; + private String parentNode; private KubernetesPodContainerBuilder () { } @@ -211,6 +217,11 @@ public KubernetesPodContainerBuilder withUUID (String uuid) { return this; } + public KubernetesPodContainerBuilder withParentNode (String parentNode) { + this.parentNode = parentNode; + return this; + } + public KubernetesPodContainer build () { KubernetesPodContainer kubernetesPodContainer = new KubernetesPodContainer(); kubernetesPodContainer.uuid = this.uuid; @@ -223,6 +234,7 @@ public KubernetesPodContainer build () { kubernetesPodContainer.ownerKind = ControllerKind.mapFromString(this.ownerKind); kubernetesPodContainer.ownerName = ownerName; kubernetesPodContainer.subcontainers = this.subcontainers; + kubernetesPodContainer.parentNode = this.parentNode; try { kubernetesPodContainer.setMappedDiagnosticContext(); kubernetesPodContainer.log.info("Created new Kubernetes Pod Container object"); diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java index 636846f28..10fa7b538 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/main/java/com/thales/chaos/platform/impl/KubernetesPlatform.java @@ -49,6 +49,7 @@ import static com.thales.chaos.constants.DataDogConstants.DATADOG_CONTAINER_KEY; import static com.thales.chaos.exception.enums.KubernetesChaosErrorCode.K8S_API_ERROR; +import static java.util.function.Predicate.not; import static net.logstash.logback.argument.StructuredArguments.v; @Component @@ -57,31 +58,35 @@ public class KubernetesPlatform extends Platform implements ShellBasedExperiment { @Autowired private ContainerManager containerManager; - @Autowired - private ApiClient apiClient; - private String namespace = "default"; + private final ApiClient apiClient; + static final String DEFAULT_NAMESPACE = "default"; + static final String UNKNOWN_PARENT_NODE = "UNKNOWN"; + private Collection namespaces = List.of(DEFAULT_NAMESPACE); - @Autowired - KubernetesPlatform (ApiClient apiClient) { - this.apiClient = apiClient; - log.info("Kubernetes Platform created"); + public Collection getNamespaces () { + return namespaces; } - public String getNamespace () { - return namespace; + public void setNamespaces (String namespaces) { + this.namespaces = Optional.of(Arrays.stream(namespaces.split(",")) + .filter(Objects::nonNull) + .filter(not(String::isEmpty)) + .collect(Collectors.toList())) + .filter(l -> !l.isEmpty()) + .orElse(List.of(DEFAULT_NAMESPACE)); } - public void setNamespace (String namespace) { - this.namespace = namespace; + @Autowired + KubernetesPlatform (ApiClient apiClient) { + this.apiClient = apiClient; + log.info("Kubernetes Platform created"); } public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContainer) { try { - Optional podExists = podExists(kubernetesPodContainer); - if (podExists.isEmpty()) { - return ContainerHealth.RUNNING_EXPERIMENT; - } else if (!podExists.get()) { + boolean podExists = podExists(kubernetesPodContainer); + if (!podExists) { return ContainerHealth.DOES_NOT_EXIST; } V1Pod result = getCoreV1Api().readNamespacedPodStatus(kubernetesPodContainer.getPodName(), @@ -103,22 +108,19 @@ public ContainerHealth checkHealth (KubernetesPodContainer kubernetesPodContaine return ContainerHealth.RUNNING_EXPERIMENT; } - Optional podExists (KubernetesPodContainer kubernetesPodContainer) { + boolean podExists (KubernetesPodContainer kubernetesPodContainer) { String podUuid = kubernetesPodContainer.getUuid(); - if (podUuid == null) return Optional.empty(); - try { - Boolean podExists = listAllPodsInNamespace().getItems() - .stream() - .map(V1Pod::getMetadata) - .filter(Objects::nonNull) - .map(V1ObjectMeta::getUid) - .anyMatch(podUuid::equals); - log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists); - return Optional.of(podExists); - } catch (ApiException e) { - log.debug("Exception when checking container existence", e); - return Optional.empty(); + if (podUuid == null) { + return false; } + boolean podExists = listAllPodsInNamespace(kubernetesPodContainer.getNamespace()).getItems() + .stream() + .map(V1Pod::getMetadata) + .filter(Objects::nonNull) + .map(V1ObjectMeta::getUid) + .anyMatch(podUuid::equals); + log.debug("Kubernetes POD {} exists = {}", kubernetesPodContainer.getPodName(), podExists); + return podExists; } @JsonIgnore @@ -126,8 +128,30 @@ CoreV1Api getCoreV1Api () { return new CoreV1Api(apiClient); } - private V1PodList listAllPodsInNamespace () throws ApiException { - return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + private V1PodList listAllPodsInNamespace (String namespace) { + try { + return getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + } catch (ApiException e) { + log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e); + return new V1PodList(); + } + } + + @Override + public PlatformHealth getPlatformHealth () { + if (namespaces.stream().map(this::canListPodsInNamespace).anyMatch(canList -> canList.equals(false))) { + return PlatformHealth.FAILED; + } + if (namespaces.stream() + .map(this::listAllPodsInNamespace) + .map(V1PodList::getItems) + .flatMap(List::stream) + .collect(Collectors.toList()) + .isEmpty()) { + log.warn("No PODs detected in specified namespaces {}", namespaces); + return PlatformHealth.DEGRADED; + } + return PlatformHealth.OK; } @Override @@ -151,25 +175,28 @@ public PlatformLevel getPlatformLevel () { return PlatformLevel.PAAS; } - @Override - public PlatformHealth getPlatformHealth () { + private boolean canListPodsInNamespace (String namespace) { try { - V1PodList pods = listAllPodsInNamespace(); - return (!pods.getItems().isEmpty()) ? PlatformHealth.OK : PlatformHealth.DEGRADED; - } catch (ApiException e) { - log.error("Kubernetes Platform health check failed", e); - return PlatformHealth.FAILED; + getCoreV1Api().listNamespacedPod(namespace, "true", false, "", "", "", 0, "", 0, false); + } catch (Exception e) { + log.error("Cannot list pods in namespace {}: {} ", namespace, e.getMessage(), e); + return false; } + return true; } @Override protected List generateRoster () { final List containerList = new ArrayList<>(); try { - V1PodList pods = listAllPodsInNamespace(); - containerList.addAll(pods.getItems().stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet())); + List pods = namespaces.stream() + .map(this::listAllPodsInNamespace) + .map(V1PodList::getItems) + .flatMap(List::stream) + .collect(Collectors.toList()); + containerList.addAll(pods.stream().map(this::fromKubernetesAPIPod).collect(Collectors.toSet())); return containerList; - } catch (ApiException e) { + } catch (Exception e) { log.error("Could not generate Kubernetes roster", e); return containerList; } @@ -178,7 +205,7 @@ protected List generateRoster () { @Override public boolean isContainerRecycled (Container container) { KubernetesPodContainer kubernetesPodContainer = (KubernetesPodContainer) container; - if (podExists(kubernetesPodContainer).orElse(false)) return isContainerRestarted(kubernetesPodContainer, + if (podExists(kubernetesPodContainer)) return isContainerRestarted(kubernetesPodContainer, ((KubernetesPodContainer) container).getTargetedSubcontainer()); return isDesiredReplicas(kubernetesPodContainer); } @@ -195,11 +222,11 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) { .withKubernetesPlatform(this) .isBackedByController(CollectionUtils.isNotEmpty(pod.getMetadata() .getOwnerReferences())) - .withOwnerKind(Optional.of(pod.getMetadata().getOwnerReferences()) + .withOwnerKind(Optional.ofNullable(pod.getMetadata().getOwnerReferences()) .flatMap(list -> list.stream().findFirst()) .map(V1OwnerReference::getKind) .orElse("")) - .withOwnerName(Optional.of(pod.getMetadata().getOwnerReferences()) + .withOwnerName(Optional.ofNullable(pod.getMetadata().getOwnerReferences()) .flatMap(list -> list.stream().findFirst()) .map(V1OwnerReference::getName) .orElse("")) @@ -210,6 +237,8 @@ KubernetesPodContainer fromKubernetesAPIPod (V1Pod pod) { .flatMap(Collection::stream) .map(V1Container::getName) .collect(Collectors.toList())) + .withParentNode(Optional.ofNullable(pod.getSpec().getNodeName()) + .orElse(UNKNOWN_PARENT_NODE)) .build(); log.info("Found new Kubernetes Pod Container {}", v(DATADOG_CONTAINER_KEY, container)); containerManager.offer(container); @@ -247,7 +276,7 @@ private boolean isContainerRestarted (KubernetesPodContainer container, String s } public ContainerHealth replicaSetRecovered (KubernetesPodContainer kubernetesPodContainer) { - return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer).orElse(false) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT; + return isDesiredReplicas(kubernetesPodContainer) && !podExists(kubernetesPodContainer) ? ContainerHealth.NORMAL : ContainerHealth.RUNNING_EXPERIMENT; } /** diff --git a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java index 218889ca4..b5e645197 100644 --- a/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java +++ b/chaosengine-experiments/chaosengine-kubernetes/src/test/java/com/thales/chaos/platform/impl/KubernetesPlatformTest.java @@ -34,6 +34,7 @@ import io.kubernetes.client.openapi.models.*; import junit.framework.TestCase; import org.apache.http.HttpStatus; +import org.hamcrest.collection.IsIterableContainingInAnyOrder; import org.joda.time.DateTime; import org.junit.Assert; import org.junit.Before; @@ -79,22 +80,29 @@ public class KubernetesPlatformTest { @Mock private AppsV1Api appsV1Api; - @Before - public void setUp () { - doReturn(coreApi).when(platform).getCoreApi(); - doReturn(coreV1Api).when(platform).getCoreV1Api(); - doReturn(appsV1Api).when(platform).getAppsV1Api(); - platform.setNamespace(NAMESPACE_NAME); + private static V1PodList getV1PodList (boolean isBackedByController, int numberOfPods) { + List ownerReferences = null; + if (isBackedByController) { + ownerReferences = new ArrayList<>(); + ownerReferences.add(new V1OwnerReferenceBuilder().withNewController("mycontroller").build()); + } + V1ObjectMeta metadata = new V1ObjectMetaBuilder().withUid(randomUUID().toString()) + .withName(POD_NAME) + .withNamespace(NAMESPACE_NAME) + .withLabels(new HashMap<>()) + .withOwnerReferences(ownerReferences) + .build(); + V1Pod pod = new V1Pod(); + pod.setMetadata(metadata); + pod.setSpec(new V1PodSpec().containers(Collections.singletonList(new V1Container().name(randomUUID().toString())))); + V1PodList list = new V1PodList(); + for (int i = 0; i < numberOfPods; i++) list.addItemsItem(pod); + return list; } @Test public void testPodWithoutOwnerCannotBeTested () throws Exception { - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), + when(coreV1Api.listNamespacedPod(anyString(), anyString(), anyBoolean(), anyString(), anyString(), anyString(), anyInt(), anyString(), anyInt(), @@ -139,31 +147,14 @@ public void testContainerHealthWithException () throws ApiException { anyString(), anyInt(), anyBoolean())).thenThrow(new ApiException()); - assertEquals("Error while checking container presence", - ContainerHealth.RUNNING_EXPERIMENT, - platform.checkHealth((KubernetesPodContainer) platform.getRoster().get(0))); } - @Test - public void testPodExists () throws ApiException { - V1PodList v1PodList = getV1PodList(true); - V1Pod pod = v1PodList.getItems().get(0); - KubernetesPodContainer kubernetesPodContainer = platform.fromKubernetesAPIPod(pod); - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), - anyInt(), - anyString(), - anyInt(), - anyBoolean())).thenReturn(v1PodList) - .thenReturn(new V1PodList()) - .thenThrow(new ApiException(new IOException())); - assertTrue("POD exists", platform.podExists(kubernetesPodContainer).orElseThrow()); - assertFalse("POD does not exist", platform.podExists(kubernetesPodContainer).orElseThrow()); - assertTrue("IO Exception", platform.podExists(kubernetesPodContainer).isEmpty()); + @Before + public void setUp () { + doReturn(coreApi).when(platform).getCoreApi(); + doReturn(coreV1Api).when(platform).getCoreV1Api(); + doReturn(appsV1Api).when(platform).getAppsV1Api(); + platform.setNamespaces(NAMESPACE_NAME); } @Test @@ -197,6 +188,21 @@ public void testPlatformHealth () throws ApiException { assertEquals(PlatformHealth.OK, platform.getPlatformHealth()); } + @Test + public void testPlatformHealthCannotListPods () throws ApiException { + when(coreV1Api.listNamespacedPod(anyString(), + anyString(), + anyBoolean(), + anyString(), + anyString(), + anyString(), + anyInt(), + anyString(), + anyInt(), + anyBoolean())).thenThrow(new ApiException()); + assertEquals(PlatformHealth.FAILED, platform.getPlatformHealth()); + } + @Test public void testPodWithOwnerCanBeTested () throws Exception { when(coreV1Api.listNamespacedPod(anyString(), @@ -365,34 +371,10 @@ public void testCheckDesiredReplicasReplicationController () throws ApiException } @Test - public void testPlatformHealthNotAvailable () throws ApiException { - when(coreV1Api.listNamespacedPod(anyString(), - anyString(), - anyBoolean(), - anyString(), - anyString(), - anyString(), - anyInt(), - anyString(), - anyInt(), - anyBoolean())).thenThrow(new ApiException()); - assertEquals(PlatformHealth.FAILED, platform.getPlatformHealth()); - } - - @Test - public void testContainerHealthDoesNotExist () throws ApiException { - V1PodList list = new V1PodList(); - V1Pod pod = mock(V1Pod.class); - V1ObjectMeta metadata = mock(V1ObjectMeta.class); - V1PodSpec spec = mock(V1PodSpec.class); - when(metadata.getUid()).thenReturn(randomUUID().toString()); - when(pod.getMetadata()).thenReturn(metadata); - when(pod.getSpec()).thenReturn(spec); - list.addItemsItem(pod); - KubernetesPodContainer kubernetesPodContainer = KubernetesPodContainer.builder() - .withUUID(randomUUID().toString()) - .withOwnerKind("") - .build(); + public void testPodExists () throws ApiException { + V1PodList v1PodList = getV1PodList(true); + V1Pod pod = v1PodList.getItems().get(0); + KubernetesPodContainer kubernetesPodContainer = platform.fromKubernetesAPIPod(pod); when(coreV1Api.listNamespacedPod(anyString(), anyString(), anyBoolean(), @@ -402,8 +384,12 @@ public void testContainerHealthDoesNotExist () throws ApiException { anyInt(), anyString(), anyInt(), - anyBoolean())).thenReturn(list); - assertEquals(ContainerHealth.DOES_NOT_EXIST, platform.checkHealth(kubernetesPodContainer)); + anyBoolean())).thenReturn(v1PodList) + .thenReturn(new V1PodList()) + .thenThrow(new ApiException(new IOException())); + assertTrue("POD exists", platform.podExists(kubernetesPodContainer)); + assertFalse("POD does not exist", platform.podExists(kubernetesPodContainer)); + assertFalse("IO Exception", platform.podExists(kubernetesPodContainer)); } @Test @@ -505,28 +491,32 @@ public void testContainerHealthWithSeveralContainerOneUnhealthy () throws ApiExc platform.checkHealth((KubernetesPodContainer) platform.getRoster().get(0))); } - private static V1PodList getV1PodList (boolean isBackedByController, int numberOfPods) { - List ownerReferences = new ArrayList<>(); - if (isBackedByController) { - ownerReferences.add(new V1OwnerReferenceBuilder().withNewController("mycontroller").build()); - } - V1ObjectMeta metadata = new V1ObjectMetaBuilder().withUid(randomUUID().toString()) - .withName(POD_NAME) - .withNamespace(NAMESPACE_NAME) - .withLabels(new HashMap<>()) - .withOwnerReferences(ownerReferences) - .build(); - V1Pod pod = new V1Pod(); - pod.setMetadata(metadata); - pod.setSpec(new V1PodSpec().containers(Collections.singletonList(new V1Container().name(randomUUID().toString())))); - V1PodList list = new V1PodList(); - for (int i = 0; i < numberOfPods; i++) list.addItemsItem(pod); - return list; - } - @Test - public void testGetNamespace () { - assertEquals("mynamespace", platform.getNamespace()); + public void testContainerHealthDoesNotExist () throws ApiException { + V1PodList list = new V1PodList(); + V1Pod pod = mock(V1Pod.class); + V1ObjectMeta metadata = mock(V1ObjectMeta.class); + V1PodSpec spec = mock(V1PodSpec.class); + when(metadata.getUid()).thenReturn(randomUUID().toString()); + when(pod.getMetadata()).thenReturn(metadata); + when(pod.getSpec()).thenReturn(spec); + list.addItemsItem(pod); + KubernetesPodContainer kubernetesPodContainer = KubernetesPodContainer.builder() + .withNamespace(NAMESPACE_NAME) + .withUUID(randomUUID().toString()) + .withOwnerKind("") + .build(); + when(coreV1Api.listNamespacedPod(anyString(), + anyString(), + anyBoolean(), + anyString(), + anyString(), + anyString(), + anyInt(), + anyString(), + anyInt(), + anyBoolean())).thenReturn(list); + assertEquals(ContainerHealth.DOES_NOT_EXIST, platform.checkHealth(kubernetesPodContainer)); } private static V1PodList getV1PodList (boolean isBackedByController) { @@ -847,6 +837,7 @@ public void testIsContainerRecycled () throws ApiException { String uid = randomUUID().toString(); String containerName = randomUUID().toString(); KubernetesPodContainer kubernetesPodContainer = spy(KubernetesPodContainer.builder() + .withNamespace(NAMESPACE_NAME) .withUUID(uid) .withSubcontainers(List.of( containerName)) @@ -922,7 +913,7 @@ public void testIsContainerRecycledAPIError () throws ApiException { .withOwnerKind(REPLICA_SET.toString()) .withSubcontainers(Set.of(subContainer)) .build(); - doReturn(Optional.of(Boolean.TRUE)).when(platform).podExists(kubernetesPodContainer); + doReturn(true).when(platform).podExists(kubernetesPodContainer); doThrow(new ApiException()).when(coreV1Api).readNamespacedPodStatus(any(), any(), any()); platform.isContainerRecycled(kubernetesPodContainer); } @@ -948,7 +939,16 @@ public void testRecycleContainer () throws ApiException { public void testGetConnectedShellClient () throws IOException, ApiException { KubernetesPodContainer kubernetesPodContainer = mock(KubernetesPodContainer.class); platform.getConnectedShellClient(kubernetesPodContainer); + } + @Test + public void testSetNamespaces () { + platform.setNamespaces(""); + assertThat(platform.getNamespaces(), + IsIterableContainingInAnyOrder.containsInAnyOrder(KubernetesPlatform.DEFAULT_NAMESPACE)); + platform.setNamespaces("default,application"); + assertThat(platform.getNamespaces(), + IsIterableContainingInAnyOrder.containsInAnyOrder("default", "application")); } @Configuration @@ -966,7 +966,7 @@ KubernetesPlatform kubernetesPlatform () { } private class TestProcess extends Process { - private InputStream is; + private final InputStream is; public TestProcess (InputStream is) { this.is = is; diff --git a/docs/markdown/Experiment_Modules/kubernetes_experiments.md b/docs/markdown/Experiment_Modules/kubernetes_experiments.md index d1555b4bc..4135893a4 100644 --- a/docs/markdown/Experiment_Modules/kubernetes_experiments.md +++ b/docs/markdown/Experiment_Modules/kubernetes_experiments.md @@ -14,7 +14,7 @@ The official Kubernetes Java Client is used to interact with the cluster. | | | | --- | --- | | Resource | | -| Version | 9.0.2 | +| Version | 10.0.0 | | Maven Repositories | | ## Configuration @@ -26,7 +26,7 @@ Environment variables that control how the Chaos Engine interacts with Kubernete | kubernetes | The presence of this key enables Kubernetes module. | N/A | Yes | | kubernetes.url | Kubernetes server API url e.g. | None | Yes | | kubernetes.token | JWT token assigned to service account. You can get the value by running `kubectl describe secret name_of_your_secret` | None | Yes | -| kubernetes.namespace | K8S namespace where experiments should be performed | `default` | Yes | +| kubernetes.namespaces | Comma-separated list of namespaces where experiments should be performed | `default` | Yes | | kubernetes.debug | Enables debug log of Kubernetes java client | `false` | No | | kubernetes.validateSSL | Enables validation of sever side certificates | `false` | No | @@ -36,6 +36,7 @@ A service account with a role binding needs to be created in order to access spe Please replace the {{namespace}} fillers with the appropriate values and apply to your cluster. +### Experiments on single namespace **chaos-engine-service-account.yaml** ```yaml @@ -110,7 +111,86 @@ subjects: namespace: {{namespace}} ``` -You can retrieve the token by running `kubectl describe secret chaos-engine -n {{namespace}}` +You can retrieve the token by running `kubectl describe secret chaos-engine -n {{namespace}}` + +### Experiments on multiple namespaces + +When your experiment targets are located in multiple namespaces, +you need to bind roles allowing access to appropriate namespace to your service account. +Or you can simply create a cluster role and binding by running below yaml. + + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: chaos-engine-crole +rules: +- apiGroups: + - apps + resources: + - daemonsets + - daemonsets/status + - deployments + - deployments/status + - replicasets + - replicasets/status + - statefulsets + - statefulsets/status + verbs: + - get + - list +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + + +- apiGroups: + - "" + resources: + - pods + - pods/status + - replicationcontrollers/status + verbs: + - get + - list + +- apiGroups: + - "" + resources: + - pods/exec + verbs: + - create + - get + +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: chaos-engine-rolebinding +roleRef: + kind: ClusterRole + name: chaos-engine-crole + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: chaos-engine-serviceaccount + namespace: {{namespace}} + +--- + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: chaos-engine-serviceaccount + namespace: {{namespace}} + +``` + ### Verify Service Account Setting