Skip to content

Commit

Permalink
move e2e to v1beta1 file and add workflow
Browse files Browse the repository at this point in the history
Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>
  • Loading branch information
lujiajing1126 committed Sep 15, 2024
1 parent 844d656 commit 6ebfb5b
Show file tree
Hide file tree
Showing 3 changed files with 213 additions and 128 deletions.
85 changes: 85 additions & 0 deletions .github/workflows/e2e-multi-network-provider.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: E2E-Multiple-NetworkProvider

on:
push:
branches:
- master
- release-*
pull_request: {}
workflow_dispatch: {}

env:
# Common versions
GO_VERSION: '1.17'
KIND_IMAGE: 'kindest/node:v1.23.3'
KIND_CLUSTER_NAME: 'ci-testing'

jobs:

rollout:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/kind-action@v1.2.0
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
kubectl apply -f ./test/e2e/test_data/customNetworkProvider/istio_crd.yaml
kubectl apply -f ./test/e2e/test_data/customNetworkProvider/lua_script_configmap.yaml
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='Canary rollout with multiple network providers' test/e2e
retVal=$?
# kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
128 changes: 0 additions & 128 deletions test/e2e/rollout_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2807,134 +2807,6 @@ var _ = SIGDescribe("Rollout", func() {
})
})

KruiseDescribe("Canary rollout with multiple network providers", func() {
It("V1->V2: Route traffic with header/queryParams/path matches and weight using rollout for Istio and Ingress", func() {
By("Creating Rollout...")
rollout := &v1beta1.Rollout{}
Expect(ReadYamlToObject("./test_data/customNetworkProvider/rollout_with_multi_trafficrouting.yaml", rollout)).ToNot(HaveOccurred())
CreateObject(rollout)

By("Creating workload and waiting for all pods ready...")
// service
service := &v1.Service{}
Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
CreateObject(service)
// istio api
vs := &unstructured.Unstructured{}
Expect(ReadYamlToObject("./test_data/customNetworkProvider/virtualservice_without_destinationrule.yaml", vs)).ToNot(HaveOccurred())
vs.SetAPIVersion("networking.istio.io/v1alpha3")
vs.SetKind("VirtualService")
CreateObject(vs)
// ingress
ingress := &netv1.Ingress{}
Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred())
CreateObject(ingress)
// workload
workload := &apps.Deployment{}
Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
workload.Spec.Replicas = utilpointer.Int32(4)
CreateObject(workload)
WaitDeploymentAllPodsReady(workload)

// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy))
By("check rollout status & paused success")

// v1 -> v2, start rollout action
newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
workload.Spec.Template.Spec.Containers[0].Env = newEnvs
UpdateDeployment(workload)
By("Update deployment env NODE_NAME from(version1) -> to(version2), routing traffic with header agent:pc to new version pods")
time.Sleep(time.Second * 2)

Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeTrue())
// wait step 1 complete
WaitRolloutCanaryStepPaused(rollout.Name, 1)
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1))
Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1))
// check virtualservice spec
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec := `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"match":[{"uri":{"prefix":"/pc"}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"match":[{"queryParams":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"match":[{"headers":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"route":[{"destination":{"host":"echoserver"}}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
// check original spec annotation
expectedAnno := `{"spec":{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}}`
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
// check canary-ingress spec
cIngress := &netv1.Ingress{}
Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("user-agent"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-value", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("pc"))

// resume rollout canary
ResumeRolloutCanaryV1beta1(rollout.Name)
By("Resume rollout, and wait next step(2), routing 50% traffic to new version pods")
WaitRolloutCanaryStepPaused(rollout.Name, 2)
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2))
Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2))
// check virtualservice spec
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":50},{"destination":{"host":"echoserver-canary"},"weight":50}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
// check original spec annotation
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
// check canary-ingress spec
Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic)))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-value", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())

// resume rollout
ResumeRolloutCanaryV1beta1(rollout.Name)
WaitRolloutStatusPhase(rollout.Name, v1alpha1.RolloutPhaseHealthy)
By("rollout completed, and check")
// check ingress & service & virtualservice & deployment

// ingress
Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred())
cIngress = &netv1.Ingress{}
Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred())
// virtualservice
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(""))
// service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
cService := &v1.Service{}
Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
// deployment
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeFalse())
Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
if env.Name == "NODE_NAME" {
Expect(env.Value).Should(Equal("version2"))
}
}
// check progressing succeed
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
cond := getRolloutConditionV1beta1(rollout.Status, v1beta1.RolloutConditionProgressing)
Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted))
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
cond = getRolloutConditionV1beta1(rollout.Status, v1beta1.RolloutConditionSucceeded)
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
})
})

KruiseDescribe("DaemonSet canary rollout", func() {
It("DaemonSet V1->V2: 1,100% Succeeded", func() {
By("Creating Rollout...")
Expand Down
128 changes: 128 additions & 0 deletions test/e2e/rollout_v1beta1_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ import (
netv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/util/retry"
Expand Down Expand Up @@ -3982,6 +3983,133 @@ var _ = SIGDescribe("Rollout v1beta1", func() {
})
})

KruiseDescribe("Canary rollout with multiple network providers", func() {
It("V1->V2: Route traffic with header/queryParams/path matches and weight using rollout for Istio and Ingress", func() {
By("Creating Rollout...")
rollout := &v1beta1.Rollout{}
Expect(ReadYamlToObject("./test_data/customNetworkProvider/rollout_with_multi_trafficrouting.yaml", rollout)).ToNot(HaveOccurred())
CreateObject(rollout)

By("Creating workload and waiting for all pods ready...")
// service
service := &v1.Service{}
Expect(ReadYamlToObject("./test_data/rollout/service.yaml", service)).ToNot(HaveOccurred())
CreateObject(service)
// istio api
vs := &unstructured.Unstructured{}
Expect(ReadYamlToObject("./test_data/customNetworkProvider/virtualservice_without_destinationrule.yaml", vs)).ToNot(HaveOccurred())
vs.SetAPIVersion("networking.istio.io/v1alpha3")
vs.SetKind("VirtualService")
CreateObject(vs)
// ingress
ingress := &netv1.Ingress{}
Expect(ReadYamlToObject("./test_data/rollout/nginx_ingress.yaml", ingress)).ToNot(HaveOccurred())
CreateObject(ingress)
// workload
workload := &apps.Deployment{}
Expect(ReadYamlToObject("./test_data/rollout/deployment.yaml", workload)).ToNot(HaveOccurred())
workload.Spec.Replicas = utilpointer.Int32(4)
CreateObject(workload)
WaitDeploymentAllPodsReady(workload)

// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.Phase).Should(Equal(v1beta1.RolloutPhaseHealthy))
By("check rollout status & paused success")

// v1 -> v2, start rollout action
newEnvs := mergeEnvVar(workload.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{Name: "NODE_NAME", Value: "version2"})
workload.Spec.Template.Spec.Containers[0].Env = newEnvs
UpdateDeployment(workload)
By("Update deployment env NODE_NAME from(version1) -> to(version2), routing traffic with header agent:pc to new version pods")
time.Sleep(time.Second * 2)

Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeTrue())
// wait step 1 complete
WaitRolloutCanaryStepPaused(rollout.Name, 1)
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 1))
Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 1))
// check virtualservice spec
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec := `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"match":[{"uri":{"prefix":"/pc"}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"match":[{"queryParams":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"match":[{"headers":{"user-agent":{"exact":"pc"}}}],"route":[{"destination":{"host":"echoserver-canary"}}]},{"route":[{"destination":{"host":"echoserver"}}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
// check original spec annotation
expectedAnno := `{"spec":{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}}`
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
// check canary-ingress spec
cIngress := &netv1.Ingress{}
Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("user-agent"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-value", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("pc"))

// resume rollout canary
ResumeRolloutCanary(rollout.Name)
By("Resume rollout, and wait next step(2), routing 50% traffic to new version pods")
WaitRolloutCanaryStepPaused(rollout.Name, 2)
// check rollout status
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
Expect(rollout.Status.CanaryStatus.CanaryReplicas).Should(BeNumerically("==", 2))
Expect(rollout.Status.CanaryStatus.CanaryReadyReplicas).Should(BeNumerically("==", 2))
// check virtualservice spec
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"},"weight":50},{"destination":{"host":"echoserver-canary"},"weight":50}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
// check original spec annotation
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(expectedAnno))
// check canary-ingress spec
Expect(GetObject(service.Name+"-canary", cIngress)).NotTo(HaveOccurred())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary", nginxIngressAnnotationDefaultPrefix)]).Should(Equal("true"))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-weight", nginxIngressAnnotationDefaultPrefix)]).Should(Equal(removePercentageSign(*rollout.Spec.Strategy.Canary.Steps[1].Traffic)))
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())
Expect(cIngress.Annotations[fmt.Sprintf("%s/canary-by-header-value", nginxIngressAnnotationDefaultPrefix)]).Should(BeEmpty())

// resume rollout
ResumeRolloutCanary(rollout.Name)
WaitRolloutStatusPhase(rollout.Name, v1beta1.RolloutPhaseHealthy)
By("rollout completed, and check")
// check ingress & service & virtualservice & deployment

// ingress
Expect(GetObject(ingress.Name, ingress)).NotTo(HaveOccurred())
cIngress = &netv1.Ingress{}
Expect(GetObject(fmt.Sprintf("%s-canary", ingress.Name), cIngress)).To(HaveOccurred())
// virtualservice
Expect(GetObject(vs.GetName(), vs)).NotTo(HaveOccurred())
expectedSpec = `{"gateways":["nginx-gateway"],"hosts":["*"],"http":[{"route":[{"destination":{"host":"echoserver"}}]}]}`
Expect(util.DumpJSON(vs.Object["spec"])).Should(Equal(expectedSpec))
Expect(vs.GetAnnotations()[OriginalSpecAnnotation]).Should(Equal(""))
// service
Expect(GetObject(service.Name, service)).NotTo(HaveOccurred())
Expect(service.Spec.Selector[apps.DefaultDeploymentUniqueLabelKey]).Should(Equal(""))
cService := &v1.Service{}
Expect(GetObject(fmt.Sprintf("%s-canary", service.Name), cService)).To(HaveOccurred())
// deployment
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
Expect(workload.Spec.Paused).Should(BeFalse())
Expect(workload.Status.UpdatedReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.Replicas).Should(BeNumerically("==", *workload.Spec.Replicas))
Expect(workload.Status.ReadyReplicas).Should(BeNumerically("==", *workload.Spec.Replicas))
for _, env := range workload.Spec.Template.Spec.Containers[0].Env {
if env.Name == "NODE_NAME" {
Expect(env.Value).Should(Equal("version2"))
}
}
// check progressing succeed
Expect(GetObject(rollout.Name, rollout)).NotTo(HaveOccurred())
cond := getRolloutConditionV1beta1(rollout.Status, v1beta1.RolloutConditionProgressing)
Expect(cond.Reason).Should(Equal(v1beta1.ProgressingReasonCompleted))
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionFalse)))
cond = getRolloutConditionV1beta1(rollout.Status, v1beta1.RolloutConditionSucceeded)
Expect(string(cond.Status)).Should(Equal(string(metav1.ConditionTrue)))
Expect(GetObject(workload.Name, workload)).NotTo(HaveOccurred())
WaitRolloutWorkloadGeneration(rollout.Name, workload.Generation)
})
})
})

func removePercentageSign(input string) string {
Expand Down

0 comments on commit 6ebfb5b

Please sign in to comment.