Skip to content

Commit

Permalink
Add composite provider to support multiple network providers (#224)
Browse files Browse the repository at this point in the history
* add composite provider

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* fix nginx lua script and add E2E

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* fix test case

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* revert image

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* fix indent

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* follow latest interface change

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

* move e2e to v1beta1 file and add workflow

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>

---------

Signed-off-by: Megrez Lu <lujiajing1126@gmail.com>
  • Loading branch information
lujiajing1126 authored Nov 1, 2024
1 parent f0363f2 commit 6854752
Show file tree
Hide file tree
Showing 8 changed files with 358 additions and 21 deletions.
85 changes: 85 additions & 0 deletions .github/workflows/e2e-multi-network-provider.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
name: E2E-Multiple-NetworkProvider

on:
push:
branches:
- master
- release-*
pull_request: {}
workflow_dispatch: {}

env:
# Common versions
GO_VERSION: '1.17'
KIND_IMAGE: 'kindest/node:v1.23.3'
KIND_CLUSTER_NAME: 'ci-testing'

jobs:

rollout:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
submodules: true
- name: Setup Go
uses: actions/setup-go@v2
with:
go-version: ${{ env.GO_VERSION }}
- name: Setup Kind Cluster
uses: helm/kind-action@v1.2.0
with:
node_image: ${{ env.KIND_IMAGE }}
cluster_name: ${{ env.KIND_CLUSTER_NAME }}
config: ./test/kind-conf.yaml
- name: Build image
run: |
export IMAGE="openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID}"
docker build --pull --no-cache . -t $IMAGE
kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
- name: Install Kruise Rollout
run: |
set -ex
kubectl cluster-info
IMG=openkruise/kruise-rollout:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
for ((i=1;i<10;i++));
do
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
set -e
if [ "$PODS" -eq "1" ]; then
break
fi
sleep 3
done
set +e
PODS=$(kubectl get pod -n kruise-rollout | grep '1/1' | wc -l)
kubectl get node -o yaml
kubectl get all -n kruise-rollout -o yaml
set -e
if [ "$PODS" -eq "1" ]; then
echo "Wait for kruise-rollout ready successfully"
else
echo "Timeout to wait for kruise-rollout ready"
exit 1
fi
- name: Run E2E Tests
run: |
export KUBECONFIG=/home/runner/.kube/config
kubectl apply -f ./test/e2e/test_data/customNetworkProvider/istio_crd.yaml
kubectl apply -f ./test/e2e/test_data/customNetworkProvider/lua_script_configmap.yaml
make ginkgo
set +e
./bin/ginkgo -timeout 60m -v --focus='Canary rollout with multiple network providers' test/e2e
retVal=$?
# kubectl get pod -n kruise-rollout --no-headers | grep manager | awk '{print $1}' | xargs kubectl logs -n kruise-rollout
restartCount=$(kubectl get pod -n kruise-rollout --no-headers | awk '{print $4}')
if [ "${restartCount}" -eq "0" ];then
echo "Kruise-rollout has not restarted"
else
kubectl get pod -n kruise-rollout --no-headers
echo "Kruise-rollout has restarted, abort!!!"
kubectl get pod -n kruise-rollout --no-headers| awk '{print $1}' | xargs kubectl logs -p -n kruise-rollout
exit 1
fi
exit $retVal
24 changes: 13 additions & 11 deletions lua_configuration/trafficrouting_ingress/nginx.lua
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,21 @@ end
-- headers & cookie apis
-- traverse matches
for _,match in ipairs(obj.matches) do
local header = match.headers[1]
-- cookie
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
-- if regular expression
if ( header.type == "RegularExpression" )
if match.headers and next(match.headers) ~= nil then
local header = match.headers[1]
-- cookie
if ( header.name == "canary-by-cookie" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
annotations["nginx.ingress.kubernetes.io/canary-by-cookie"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
annotations["nginx.ingress.kubernetes.io/canary-by-header"] = header.name
-- if regular expression
if ( header.type == "RegularExpression" )
then
annotations["nginx.ingress.kubernetes.io/canary-by-header-pattern"] = header.value
else
annotations["nginx.ingress.kubernetes.io/canary-by-header-value"] = header.value
end
end
end
end
Expand Down
26 changes: 22 additions & 4 deletions pkg/trafficrouting/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -336,8 +336,9 @@ func (m *Manager) PatchStableService(c *TrafficRoutingContext) (bool, error) {

func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, cService string) (network.NetworkProvider, error) {
trafficRouting := con.ObjectRef[0]
networkProviders := make([]network.NetworkProvider, 0, 3)
if trafficRouting.CustomNetworkRefs != nil {
return custom.NewCustomController(c, custom.Config{
np, innerErr := custom.NewCustomController(c, custom.Config{
Key: con.Key,
RolloutNs: con.Namespace,
CanaryService: cService,
Expand All @@ -347,27 +348,44 @@ func newNetworkProvider(c client.Client, con *TrafficRoutingContext, sService, c
//only set for CustomController, never work for Ingress and Gateway
DisableGenerateCanaryService: con.DisableGenerateCanaryService,
})
if innerErr != nil {
return nil, innerErr
}
networkProviders = append(networkProviders, np)
}
if trafficRouting.Ingress != nil {
return ingress.NewIngressTrafficRouting(c, ingress.Config{
np, innerErr := ingress.NewIngressTrafficRouting(c, ingress.Config{
Key: con.Key,
Namespace: con.Namespace,
CanaryService: cService,
StableService: sService,
TrafficConf: trafficRouting.Ingress,
OwnerRef: con.OwnerRef,
})
if innerErr != nil {
return nil, innerErr
}
networkProviders = append(networkProviders, np)
}
if trafficRouting.Gateway != nil {
return gateway.NewGatewayTrafficRouting(c, gateway.Config{
np, innerErr := gateway.NewGatewayTrafficRouting(c, gateway.Config{
Key: con.Key,
Namespace: con.Namespace,
CanaryService: cService,
StableService: sService,
TrafficConf: trafficRouting.Gateway,
})
if innerErr != nil {
return nil, innerErr
}
networkProviders = append(networkProviders, np)
}
if len(networkProviders) == 0 {
return nil, fmt.Errorf("TrafficRouting current only supports Ingress, Gateway API and CustomNetworkRefs")
} else if len(networkProviders) == 1 {
return networkProviders[0], nil
}
return nil, fmt.Errorf("TrafficRouting current only support Ingress or Gateway API")
return network.CompositeController(networkProviders), nil
}

func (m *Manager) createCanaryService(c *TrafficRoutingContext, cService string, spec corev1.ServiceSpec) (*corev1.Service, error) {
Expand Down
66 changes: 66 additions & 0 deletions pkg/trafficrouting/network/composite.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
/*
Copyright 2024 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package network

import (
"context"

"k8s.io/apimachinery/pkg/util/validation/field"

"github.com/openkruise/rollouts/api/v1beta1"
)

var (
_ NetworkProvider = (CompositeController)(nil)
)

// CompositeController is a set of NetworkProvider
type CompositeController []NetworkProvider

func (c CompositeController) Initialize(ctx context.Context) error {
for _, provider := range c {
if err := provider.Initialize(ctx); err != nil {
return err
}
}
return nil
}

func (c CompositeController) EnsureRoutes(ctx context.Context, strategy *v1beta1.TrafficRoutingStrategy) (bool, error) {
done := true
for _, provider := range c {
if innerDone, innerErr := provider.EnsureRoutes(ctx, strategy); innerErr != nil {
return false, innerErr
} else if !innerDone {
done = false
}
}
return done, nil
}

func (c CompositeController) Finalise(ctx context.Context) (bool, error) {
modified := false
errList := field.ErrorList{}
for _, provider := range c {
if updated, innerErr := provider.Finalise(ctx); innerErr != nil {
errList = append(errList, field.InternalError(field.NewPath("FinalizeChildNetworkProvider"), innerErr))
} else if updated {
modified = true
}
}
return modified, errList.ToAggregate()
}
11 changes: 5 additions & 6 deletions pkg/trafficrouting/network/interface.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,11 @@ type NetworkProvider interface {
// Initialize only determine if the network resources(ingress & gateway api) exist.
// If error is nil, then the network resources exist.
Initialize(ctx context.Context) error
// EnsureRoutes check and set canary weight and matches.
// weight indicates percentage of traffic to canary service, and range of values[0,100]
// matches indicates A/B Testing release for headers, cookies
// 1. check if canary has been set desired weight.
// 2. If not, set canary desired weight
// When the first set weight is returned false, mainly to give the provider some time to process, only when again ensure, will return true
// EnsureRoutes check and set routes, e.g. canary weight and match conditions.
// 1. Canary weight specifies the relative proportion of traffic to be forwarded to the canary service within the range of [0,100]
// 2. Match conditions indicates rules to be satisfied for A/B testing scenarios, such as header, cookie, queryParams etc.
// Return true if and only if the route resources have been correctly updated and does not change in this round of reconciliation.
// Otherwise, return false to wait for the eventual consistency.
EnsureRoutes(ctx context.Context, strategy *v1beta1.TrafficRoutingStrategy) (bool, error)
// Finalise will do some cleanup work after the canary rollout complete, such as delete canary ingress.
// if error is nil, the return bool value means if the resources are modified
Expand Down
Loading

0 comments on commit 6854752

Please sign in to comment.