Skip to content

Commit

Permalink
ci: Improve Validate package (#2485)
Browse files Browse the repository at this point in the history 
* ci: improve validate package

* ci: add v6 hns validate

* chore: address comments

Signed-off-by: John Payne <89417863+jpayne3506@users.noreply.github.com>

* ci: capture hns json object

* chore: lint fix

* ci: kubeproxy restart in CreateValidator

* fix: validateRestartNetwork for hybrid clusters

* ci: cleanup RestartKubeProxyService changes

* chore: lint fix

* chore: address comments

---------

Signed-off-by: John Payne <89417863+jpayne3506@users.noreply.github.com>
  • Loading branch information
@jpayne3506
jpayne3506 authored Jan 25, 2024
1 parent 8acc551 commit faa2b73
Show file tree
Hide file tree
Showing 6 changed files with 186 additions and 108 deletions.
13 changes: 9 additions & 4 deletions test/integration/datapath/datapath_windows_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (

"github.com/Azure/azure-container-networking/test/internal/datapath"
"github.com/Azure/azure-container-networking/test/internal/kubernetes"
"github.com/Azure/azure-container-networking/test/validate"
"github.com/stretchr/testify/require"
apiv1 "k8s.io/api/core/v1"
)
Expand Down Expand Up @@ -57,9 +56,15 @@ func setupWindowsEnvironment(t *testing.T) {
clientset := kubernetes.MustGetClientset()

if *restartKubeproxy {
validator, err := validate.CreateValidator(ctx, clientset, restConfig, *podNamespace, "cniv2", false, "windows")
require.NoError(t, err)
err = validator.RestartKubeProxyService(ctx)
privilegedDaemonSet := kubernetes.MustParseDaemonSet(kubernetes.PrivilegedDaemonSetPath)
daemonsetClient := clientset.AppsV1().DaemonSets(kubernetes.PrivilegedNamespace)
kubernetes.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet)

// Ensures that pods have been replaced if test is re-run after failure
if err := kubernetes.WaitForPodDaemonset(ctx, clientset, kubernetes.PrivilegedNamespace, privilegedDaemonSet.Name, kubernetes.PrivilegedLabelSelector); err != nil {
require.NoError(t, err)
}
err := kubernetes.RestartKubeProxyService(ctx, clientset, kubernetes.PrivilegedNamespace, kubernetes.PrivilegedLabelSelector, restConfig)
require.NoError(t, err)
}

Expand Down
47 changes: 43 additions & 4 deletions test/internal/kubernetes/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,13 @@ const (
SubnetNameLabel = "kubernetes.azure.com/podnetwork-subnet"

// RetryAttempts is the number of times to retry a test.
RetryAttempts = 90
RetryDelay = 10 * time.Second
DeleteRetryAttempts = 12
DeleteRetryDelay = 5 * time.Second
RetryAttempts = 90
RetryDelay = 10 * time.Second
DeleteRetryAttempts = 12
DeleteRetryDelay = 5 * time.Second
PrivilegedDaemonSetPath = "../manifests/load/privileged-daemonset-windows.yaml"
PrivilegedLabelSelector = "app=privileged-daemonset"
PrivilegedNamespace = "kube-system"
)

var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file")
Expand Down Expand Up @@ -403,6 +406,9 @@ func ExecCmdOnPod(ctx context.Context, clientset *kubernetes.Clientset, namespac
if err != nil {
return []byte{}, errors.Wrapf(err, "error in executing command %s", cmd)
}
if len(stdout.Bytes()) == 0 {
log.Printf("Warning: %v had 0 bytes returned from command - %v", podName, cmd)
}

return stdout.Bytes(), nil
}
Expand Down Expand Up @@ -465,3 +471,36 @@ func MustRestartDaemonset(ctx context.Context, clientset *kubernetes.Clientset,
_, err = clientset.AppsV1().DaemonSets(namespace).Update(ctx, ds, metav1.UpdateOptions{})
return errors.Wrapf(err, "failed to update ds %s", daemonsetName)
}

// Restarts kubeproxy on windows nodes from an existing privileged daemonset
func RestartKubeProxyService(ctx context.Context, clientset *kubernetes.Clientset, privilegedNamespace, privilegedLabelSelector string, config *rest.Config) error {
restartKubeProxyCmd := []string{"powershell", "Restart-service", "kubeproxy"}

nodes, err := GetNodeList(ctx, clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
node := nodes.Items[index]
if node.Status.NodeInfo.OperatingSystem != string(corev1.Windows) {
continue
}
// get the privileged pod
pod, err := GetPodsByNode(ctx, clientset, privilegedNamespace, privilegedLabelSelector, node.Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod on node %s", node.Name)
}

if len(pod.Items) == 0 {
return errors.Errorf("there are no privileged pods on node - %v", node.Name)
}
privilegedPod := pod.Items[0]
// exec into the pod and restart kubeproxy
_, err = ExecCmdOnPod(ctx, clientset, privilegedNamespace, privilegedPod.Name, restartKubeProxyCmd, config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod %s on node %s", privilegedPod.Name, node.Name)
}
}
return nil
}
70 changes: 46 additions & 24 deletions test/validate/linux_validate.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,14 @@
package validate

import (
"context"
"encoding/json"

"github.com/Azure/azure-container-networking/cns"
restserver "github.com/Azure/azure-container-networking/cns/restserver"
acnk8s "github.com/Azure/azure-container-networking/test/internal/kubernetes"
"github.com/pkg/errors"
corev1 "k8s.io/api/core/v1"
)

const (
Expand All @@ -15,12 +18,12 @@ const (
)

var (
restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"}
cnsManagedStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"}
azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"}
azureVnetIpamStateCmd = []string{"bash", "-c", "cat /var/run/azure-vnet-ipam.json"}
ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"}
cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"}
restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"}
cnsManagedStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"}
azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"}
azureVnetIpamStateCmd = []string{"bash", "-c", "cat /var/run/azure-vnet-ipam.json"}
ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"}
cnsCachedAssignedIPStateCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"}
)

type stateFileIpsFunc func([]byte) (map[string]string, error)
Expand All @@ -29,18 +32,18 @@ var linuxChecksMap = map[string][]check{
"cilium": {
{"cns", cnsManagedStateFileIps, cnsLabelSelector, privilegedNamespace, cnsManagedStateFileCmd}, // cns configmap "ManageEndpointState": true, | Endpoints managed in CNS State File
{"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd},
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd},
},
"cniv1": {
{"azure-vnet", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd},
{"azure-vnet-ipam", azureVnetIpamStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamStateCmd},
},
"cniv2": {
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd},
{"azure-vnet", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, // cns configmap "ManageEndpointState": false, | Endpoints managed in CNI State File
},
"dualstack": {
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd},
{"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd},
{"azure dualstackoverlay", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd},
},
}
Expand Down Expand Up @@ -156,21 +159,6 @@ func ciliumStateFileIps(result []byte) (map[string]string, error) {
return ciliumPodIps, nil
}

func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
var cnsLocalCache CNSLocalCache

err := json.Unmarshal(result, &cnsLocalCache)
if err != nil {
return nil, errors.Wrapf(err, "failed to unmarshal cns local cache")
}

cnsPodIps := make(map[string]string)
for index := range cnsLocalCache.IPConfigurationStatus {
cnsPodIps[cnsLocalCache.IPConfigurationStatus[index].IPAddress] = cnsLocalCache.IPConfigurationStatus[index].PodInfo.Name()
}
return cnsPodIps, nil
}

func azureVnetStateIps(result []byte) (map[string]string, error) {
var azureVnetResult AzureCniState
err := json.Unmarshal(result, &azureVnetResult)
Expand Down Expand Up @@ -212,3 +200,37 @@ func azureVnetIpamStateIps(result []byte) (map[string]string, error) {
}
return azureVnetIpamPodIps, nil
}

// Linux only function
func (v *Validator) validateRestartNetwork(ctx context.Context) error {
nodes, err := acnk8s.GetNodeList(ctx, v.clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
node := nodes.Items[index]
if node.Status.NodeInfo.OperatingSystem != string(corev1.Linux) {
continue
}
// get the privileged pod
pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, node.Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}
if len(pod.Items) == 0 {
return errors.Errorf("there are no privileged pods on node - %v", node.Name)
}
privilegedPod := pod.Items[0]
// exec into the pod to get the state file
_, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privilegedPod.Name, restartNetworkCmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod %s on node %s", privilegedPod.Name, node.Name)
}
err = acnk8s.WaitForPodsRunning(ctx, v.clientset, "", "")
if err != nil {
return errors.Wrapf(err, "failed to wait for pods running")
}
}
return nil
}
8 changes: 5 additions & 3 deletions test/validate/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,16 @@ import (
)

func compareIPs(expected map[string]string, actual []string) error {
if len(expected) != len(actual) {
return errors.Errorf("len of expected IPs != len of actual IPs, expected: %+v, actual: %+v", expected, actual)
}
expectedLen := len(expected)

for _, ip := range actual {
if _, ok := expected[ip]; !ok {
return errors.Errorf("actual ip %s is unexpected, expected: %+v, actual: %+v", ip, expected, actual)
}
delete(expected, ip)
}
if expectedLen != len(actual) {
return errors.Errorf("len of expected IPs != len of actual IPs, expected: %+v, actual: %+v | Remaining, potentially leaked, IP(s) on state file - %v", expectedLen, len(actual), expected)
}

return nil
Expand Down
83 changes: 26 additions & 57 deletions test/validate/validate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package validate

import (
"context"
"encoding/json"
"log"

acnk8s "github.com/Azure/azure-container-networking/test/internal/kubernetes"
Expand Down Expand Up @@ -72,6 +73,10 @@ func CreateValidator(ctx context.Context, clientset *kubernetes.Clientset, confi
switch os {
case "windows":
checks = windowsChecksMap[cni]
err := acnk8s.RestartKubeProxyService(ctx, clientset, privilegedNamespace, privilegedLabelSelector, config)
if err != nil {
return nil, errors.Wrapf(err, "failed to restart kubeproxy")
}
case "linux":
checks = linuxChecksMap[cni]
default:
Expand Down Expand Up @@ -99,7 +104,7 @@ func (v *Validator) Validate(ctx context.Context) error {
if v.os == "linux" {
// We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706
log.Printf("Validating the restart network scenario")
err = v.ValidateRestartNetwork(ctx)
err = v.validateRestartNetwork(ctx)
if err != nil {
return errors.Wrapf(err, "failed to validate restart network scenario")
}
Expand All @@ -117,33 +122,6 @@ func (v *Validator) ValidateStateFile(ctx context.Context) error {
return nil
}

func (v *Validator) ValidateRestartNetwork(ctx context.Context) error {
nodes, err := acnk8s.GetNodeList(ctx, v.clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
// get the privileged pod
pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}

privelegedPod := pod.Items[0]
// exec into the pod to get the state file
_, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod - %s", privelegedPod.Name)
}
err = acnk8s.WaitForPodsRunning(ctx, v.clientset, "", "")
if err != nil {
return errors.Wrapf(err, "failed to wait for pods running")
}
}
return nil
}

func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error {
log.Printf("Validating %s state file", checkType)
nodes, err := acnk8s.GetNodeListByLabelSelector(ctx, v.clientset, nodeSelectorMap[v.os])
Expand All @@ -157,6 +135,9 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}
if len(pod.Items) == 0 {
return errors.Errorf("there are no privileged pods on node - %v", nodes.Items[index].Name)
}
podName := pod.Items[0].Name
// exec into the pod to get the state file
result, err := acnk8s.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config)
Expand All @@ -165,7 +146,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu
}
filePodIps, err := stateFileIps(result)
if err != nil {
return errors.Wrapf(err, "failed to get pod ips from state file")
return errors.Wrapf(err, "failed to get pod ips from state file on node %v", nodes.Items[index].Name)
}
if len(filePodIps) == 0 && v.restartCase {
log.Printf("No pods found on node %s", nodes.Items[index].Name)
Expand All @@ -175,7 +156,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu
podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index])

if err := compareIPs(filePodIps, podIps); err != nil {
return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name)
return errors.Wrapf(err, "State file validation failed for %s on node %s", checkType, nodes.Items[index].Name)
}
}
log.Printf("State file validation for %s passed", checkType)
Expand Down Expand Up @@ -257,36 +238,24 @@ func (v *Validator) ValidateDualStackControlPlane(ctx context.Context) error {
return nil
}

func (v *Validator) RestartKubeProxyService(ctx context.Context) error {
nodes, err := acnk8s.GetNodeList(ctx, v.clientset)
if err != nil {
return errors.Wrapf(err, "failed to get node list")
}

for index := range nodes.Items {
node := nodes.Items[index]
if node.Status.NodeInfo.OperatingSystem != string(corev1.Windows) {
continue
}
// get the privileged pod
pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name)
if err != nil {
return errors.Wrapf(err, "failed to get privileged pod")
}

privelegedPod := pod.Items[0]
// exec into the pod and restart kubeproxy
_, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartKubeProxyCmd, v.config)
if err != nil {
return errors.Wrapf(err, "failed to exec into privileged pod - %s", privelegedPod.Name)
}
}
return nil
}

func (v *Validator) Cleanup(ctx context.Context) {
// deploy privileged pod
privilegedDaemonSet := acnk8s.MustParseDaemonSet(privilegedDaemonSetPathMap[v.os])
daemonsetClient := v.clientset.AppsV1().DaemonSets(privilegedNamespace)
acnk8s.MustDeleteDaemonset(ctx, daemonsetClient, privilegedDaemonSet)
}

func cnsCacheStateFileIps(result []byte) (map[string]string, error) {
var cnsLocalCache CNSLocalCache

err := json.Unmarshal(result, &cnsLocalCache)
if err != nil {
return nil, errors.Wrapf(err, "failed to unmarshal cns local cache")
}

cnsPodIps := make(map[string]string)
for index := range cnsLocalCache.IPConfigurationStatus {
cnsPodIps[cnsLocalCache.IPConfigurationStatus[index].IPAddress] = cnsLocalCache.IPConfigurationStatus[index].PodInfo.Name()
}
return cnsPodIps, nil
}
Loading

0 comments on commit faa2b73

Please sign in to comment.