From fbac9760661b7b5f02846d4e4b1a44cfcb8b67a1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 23 Jan 2024 09:44:22 -0600 Subject: [PATCH 01/31] deps: bump k8s.io/klog/v2 from 2.120.0 to 2.120.1 (#2532) Bumps [k8s.io/klog/v2](https://github.com/kubernetes/klog) from 2.120.0 to 2.120.1. - [Release notes](https://github.com/kubernetes/klog/releases) - [Changelog](https://github.com/kubernetes/klog/blob/main/RELEASE.md) - [Commits](https://github.com/kubernetes/klog/compare/v2.120.0...v2.120.1) --- updated-dependencies: - dependency-name: k8s.io/klog/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f8a0cd7a9ed..56306b30a62 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( k8s.io/apimachinery v0.28.5 k8s.io/client-go v0.28.5 k8s.io/klog v1.0.0 - k8s.io/klog/v2 v2.120.0 + k8s.io/klog/v2 v2.120.1 k8s.io/utils v0.0.0-20230726121419-3b25d923346b sigs.k8s.io/controller-runtime v0.16.3 ) diff --git a/go.sum b/go.sum index 19214fe3995..d8dc7c8bd8f 100644 --- a/go.sum +++ b/go.sum @@ -468,8 +468,8 @@ k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= -k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= -k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= From ab5dfe0de94cb914d887db0a81e486440fcc49e4 Mon Sep 17 00:00:00 2001 From: Matthew Long <61910737+thatmattlong@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:59:33 -0800 Subject: [PATCH 02/31] feat: add imds client (#2537) * feat: add imds client * fix: lint --- cns/imds/client.go | 132 +++++++++++++++++++++++++ cns/imds/client_test.go | 70 +++++++++++++ cns/imds/testdata/computeMetadata.json | 130 ++++++++++++++++++++++++ 3 files changed, 332 insertions(+) create mode 100644 cns/imds/client.go create mode 100644 cns/imds/client_test.go create mode 100644 cns/imds/testdata/computeMetadata.json diff --git a/cns/imds/client.go b/cns/imds/client.go new file mode 100644 index 00000000000..5ab43e76ba0 --- /dev/null +++ b/cns/imds/client.go @@ -0,0 +1,132 @@ +// Copyright 2024 Microsoft. All rights reserved. +// MIT License + +package imds + +import ( + "context" + "encoding/json" + "net/http" + "net/url" + + "github.com/avast/retry-go/v4" + "github.com/pkg/errors" +) + +// see docs for IMDS here: https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service + +// Client returns metadata about the VM by querying IMDS +type Client struct { + cli *http.Client + config clientConfig +} + +// clientConfig holds config options for a Client +type clientConfig struct { + endpoint string + retryAttempts uint +} + +type ClientOption func(*clientConfig) + +// Endpoint overrides the default endpoint for a Client +func Endpoint(endpoint string) ClientOption { + return func(c *clientConfig) { + c.endpoint = endpoint + } +} + +// RetryAttempts overrides the default retry attempts for the client +func RetryAttempts(attempts uint) ClientOption { + return func(c *clientConfig) { + c.retryAttempts = attempts + } +} + +const ( + vmUniqueIDProperty = "vmId" + imdsComputePath = "/metadata/instance/compute?api-version=2021-01-01&format=json" + metadataHeaderKey = "Metadata" + metadataHeaderValue = "true" + defaultRetryAttempts = 10 + defaultIMDSEndpoint = "http://169.254.169.254" +) + +var ( + ErrVMUniqueIDNotFound = errors.New("vm unique ID not found") + ErrUnexpectedStatusCode = errors.New("imds returned an unexpected status code") +) + +// NewClient creates a new imds client +func NewClient(opts ...ClientOption) *Client { + config := clientConfig{ + endpoint: defaultIMDSEndpoint, + } + + for _, o := range opts { + o(&config) + } + + return &Client{ + cli: &http.Client{}, + config: config, + } +} + +func (c *Client) GetVMUniqueID(ctx context.Context) (string, error) { + var vmUniqueID string + err := retry.Do(func() error { + computeDoc, err := c.getInstanceComputeMetadata(ctx) + if err != nil { + return errors.Wrap(err, "error getting IMDS compute metadata") + } + vmUniqueIDUntyped := computeDoc[vmUniqueIDProperty] + var ok bool + vmUniqueID, ok = vmUniqueIDUntyped.(string) + if !ok { + return errors.New("unable to parse IMDS compute metadata, vmId property is not a string") + } + return nil + }, retry.Context(ctx), retry.Attempts(c.config.retryAttempts), retry.DelayType(retry.BackOffDelay)) + if err != nil { + return "", errors.Wrap(err, "exhausted retries querying IMDS compute metadata") + } + + if vmUniqueID == "" { + return "", ErrVMUniqueIDNotFound + } + + return vmUniqueID, nil +} + +func (c *Client) getInstanceComputeMetadata(ctx context.Context) (map[string]any, error) { + imdsComputeURL, err := url.JoinPath(c.config.endpoint, imdsComputePath) + if err != nil { + return nil, errors.Wrap(err, "unable to build path to IMDS compute metadata") + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsComputeURL, http.NoBody) + if err != nil { + return nil, errors.Wrap(err, "error building IMDS http request") + } + + // IMDS requires the "Metadata: true" header + req.Header.Add(metadataHeaderKey, metadataHeaderValue) + + resp, err := c.cli.Do(req) + if err != nil { + return nil, errors.Wrap(err, "error querying IMDS") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, errors.Wrapf(ErrUnexpectedStatusCode, "unexpected status code %d", resp.StatusCode) + } + + var m map[string]any + if err := json.NewDecoder(resp.Body).Decode(&m); err != nil { + return nil, errors.Wrap(err, "error decoding IMDS response as json") + } + + return m, nil +} diff --git a/cns/imds/client_test.go b/cns/imds/client_test.go new file mode 100644 index 00000000000..c0c03b40a3f --- /dev/null +++ b/cns/imds/client_test.go @@ -0,0 +1,70 @@ +// Copyright 2024 Microsoft. All rights reserved. +// MIT License + +package imds_test + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "testing" + + "github.com/Azure/azure-container-networking/cns/imds" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetVMUniqueID(t *testing.T) { + computeMetadata, err := os.ReadFile("testdata/computeMetadata.json") + require.NoError(t, err, "error reading testdata compute metadata file") + + mockIMDSServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // request header "Metadata: true" must be present + metadataHeader := r.Header.Get("Metadata") + assert.Equal(t, "true", metadataHeader) + w.WriteHeader(http.StatusOK) + _, writeErr := w.Write(computeMetadata) + require.NoError(t, writeErr, "error writing response") + })) + defer mockIMDSServer.Close() + + imdsClient := imds.NewClient(imds.Endpoint(mockIMDSServer.URL)) + vmUniqueID, err := imdsClient.GetVMUniqueID(context.Background()) + require.NoError(t, err, "error querying testserver") + + require.Equal(t, "55b8499d-9b42-4f85-843f-24ff69f4a643", vmUniqueID) +} + +func TestGetVMUniqueIDInvalidEndpoint(t *testing.T) { + imdsClient := imds.NewClient(imds.Endpoint(string([]byte{0x7f})), imds.RetryAttempts(1)) + _, err := imdsClient.GetVMUniqueID(context.Background()) + require.Error(t, err, "expected invalid path") +} + +func TestIMDSInternalServerError(t *testing.T) { + mockIMDSServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // request header "Metadata: true" must be present + w.WriteHeader(http.StatusInternalServerError) + })) + defer mockIMDSServer.Close() + + imdsClient := imds.NewClient(imds.Endpoint(mockIMDSServer.URL), imds.RetryAttempts(1)) + + _, err := imdsClient.GetVMUniqueID(context.Background()) + require.ErrorIs(t, err, imds.ErrUnexpectedStatusCode, "expected internal server error") +} + +func TestIMDSInvalidJSON(t *testing.T) { + mockIMDSServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte("not json")) + require.NoError(t, err) + })) + defer mockIMDSServer.Close() + + imdsClient := imds.NewClient(imds.Endpoint(mockIMDSServer.URL), imds.RetryAttempts(1)) + + _, err := imdsClient.GetVMUniqueID(context.Background()) + require.Error(t, err, "expected json decoding error") +} diff --git a/cns/imds/testdata/computeMetadata.json b/cns/imds/testdata/computeMetadata.json new file mode 100644 index 00000000000..f59af37850d --- /dev/null +++ b/cns/imds/testdata/computeMetadata.json @@ -0,0 +1,130 @@ +{ + "azEnvironment": "AzurePublicCloud", + "customData": "", + "evictionPolicy": "", + "isHostCompatibilityLayerVm": "false", + "licenseType": "", + "location": "westus2", + "name": "aks-nodepool1-25781205-vmss_0", + "offer": "", + "osProfile": { + "adminUsername": "azureuser", + "computerName": "aks-nodepool1-25781205-vmss000000", + "disablePasswordAuthentication": "true" + }, + "osType": "Linux", + "placementGroupId": "078e7cc3-c76f-46d2-91ff-64c4dbdd0d7c", + "plan": { + "name": "", + "product": "", + "publisher": "" + }, + "platformFaultDomain": "0", + "platformUpdateDomain": "0", + "priority": "", + "provider": "Microsoft.Compute", + "publicKeys": [], + "publisher": "", + "resourceGroupName": "MC_matlong-test-imds_matlong-test-imds_westus2", + "resourceId": "/subscriptions/9b8218f9-902a-4d20-a65c-e98acec5362f/resourceGroups/MC_matlong-test-imds_matlong-test-imds_westus2/providers/Microsoft.Compute/virtualMachineScaleSets/aks-nodepool1-25781205-vmss/virtualMachines/0", + "securityProfile": { + "secureBootEnabled": "false", + "virtualTpmEnabled": "false" + }, + "sku": "", + "storageProfile": { + "dataDisks": [], + "imageReference": { + "id": "/subscriptions/109a5e88-712a-48ae-9078-9ca8b3c81345/resourceGroups/AKS-Ubuntu/providers/Microsoft.Compute/galleries/AKSUbuntu/images/2204gen2containerd/versions/202401.09.0", + "offer": "", + "publisher": "", + "sku": "", + "version": "" + }, + "osDisk": { + "caching": "ReadWrite", + "createOption": "FromImage", + "diffDiskSettings": { + "option": "" + }, + "diskSizeGB": "128", + "encryptionSettings": { + "enabled": "false" + }, + "image": { + "uri": "" + }, + "managedDisk": { + "id": "/subscriptions/9b8218f9-902a-4d20-a65c-e98acec5362f/resourceGroups/MC_matlong-test-imds_matlong-test-imds_westus2/providers/Microsoft.Compute/disks/aks-nodepool1-257812aks-nodepool1-2578120disk1_7f5fbaaac4d7480b9f78b5da0206643c", + "storageAccountType": "Premium_LRS" + }, + "name": "aks-nodepool1-257812aks-nodepool1-2578120disk1_7f5fbaaac4d7480b9f78b5da0206643c", + "osType": "Linux", + "vhd": { + "uri": "" + }, + "writeAcceleratorEnabled": "false" + }, + "resourceDisk": { + "size": "14336" + } + }, + "subscriptionId": "9b8218f9-902a-4d20-a65c-e98acec5362f", + "tags": "aks-managed-azure-cni-overlay:true;aks-managed-consolidated-additional-properties:d9dfbd53-b974-11ee-bba9-b29ebea78e50;aks-managed-coordination:true;aks-managed-createOperationID:f9031ed6-4bd9-47e8-908f-90c6429c0790;aks-managed-creationSource:vmssclient-aks-nodepool1-25781205-vmss;aks-managed-kubeletIdentityClientID:58ada700-3bba-437e-b6a2-71df9a5148b5;aks-managed-orchestrator:Kubernetes:1.27.7;aks-managed-poolName:nodepool1;aks-managed-resourceNameSuffix:22973824;aks-managed-ssh-access:LocalUser;azsecpack:nonprod;platformsettings.host_environment.service.platform_optedin_for_rootcerts:true", + "tagsList": [ + { + "name": "aks-managed-azure-cni-overlay", + "value": "true" + }, + { + "name": "aks-managed-consolidated-additional-properties", + "value": "d9dfbd53-b974-11ee-bba9-b29ebea78e50" + }, + { + "name": "aks-managed-coordination", + "value": "true" + }, + { + "name": "aks-managed-createOperationID", + "value": "f9031ed6-4bd9-47e8-908f-90c6429c0790" + }, + { + "name": "aks-managed-creationSource", + "value": "vmssclient-aks-nodepool1-25781205-vmss" + }, + { + "name": "aks-managed-kubeletIdentityClientID", + "value": "58ada700-3bba-437e-b6a2-71df9a5148b5" + }, + { + "name": "aks-managed-orchestrator", + "value": "Kubernetes:1.27.7" + }, + { + "name": "aks-managed-poolName", + "value": "nodepool1" + }, + { + "name": "aks-managed-resourceNameSuffix", + "value": "22973824" + }, + { + "name": "aks-managed-ssh-access", + "value": "LocalUser" + }, + { + "name": "azsecpack", + "value": "nonprod" + }, + { + "name": "platformsettings.host_environment.service.platform_optedin_for_rootcerts", + "value": "true" + } + ], + "userData": "", + "version": "202401.09.0", + "vmId": "55b8499d-9b42-4f85-843f-24ff69f4a643", + "vmScaleSetName": "aks-nodepool1-25781205-vmss", + "vmSize": "Standard_DS2_v2", + "zone": "" +} \ No newline at end of file From 1f577562a6bf200a294497dd9d9885f925ae26e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 10:10:24 -0600 Subject: [PATCH 03/31] deps: bump github.com/prometheus/common from 0.45.0 to 0.46.0 (#2524) Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.45.0 to 0.46.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.45.0...v0.46.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 7 +++---- go.sum | 14 ++++++-------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 56306b30a62..1e7c5a92d03 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/common v0.45.0 + github.com/prometheus/common v0.46.0 github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.3 github.com/spf13/afero v1.11.0 // indirect @@ -109,8 +109,8 @@ require ( go.opencensus.io v0.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 - golang.org/x/net v0.19.0 // indirect - golang.org/x/oauth2 v0.15.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.5.0 // indirect @@ -139,7 +139,6 @@ require ( github.com/containerd/containerd v1.6.26 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/golang-jwt/jwt/v5 v5.0.0 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/rootless-containers/rootlesskit v1.1.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect diff --git a/go.sum b/go.sum index d8dc7c8bd8f..be7b81f5fb3 100644 --- a/go.sum +++ b/go.sum @@ -186,8 +186,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= github.com/microsoft/go-winio v0.4.17 h1:mvYE47XnSE/F5NAiM1TkmuSNfqg5f4fH1Yo7+if9qc4= @@ -229,8 +227,8 @@ github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlk github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -327,11 +325,11 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From c48e67c74a8882635d4e244d1df99e4286385080 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:12:42 +0000 Subject: [PATCH 04/31] deps: bump github.com/google/uuid from 1.5.0 to 1.6.0 (#2542) Bumps [github.com/google/uuid](https://github.com/google/uuid) from 1.5.0 to 1.6.0. - [Release notes](https://github.com/google/uuid/releases) - [Changelog](https://github.com/google/uuid/blob/master/CHANGELOG.md) - [Commits](https://github.com/google/uuid/compare/v1.5.0...v1.6.0) --- updated-dependencies: - dependency-name: github.com/google/uuid dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1e7c5a92d03..aa55dc96855 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 - github.com/google/uuid v1.5.0 + github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-version v1.6.0 github.com/microsoft/ApplicationInsights-Go v0.4.4 diff --git a/go.sum b/go.sum index be7b81f5fb3..0a525538b2a 100644 --- a/go.sum +++ b/go.sum @@ -139,8 +139,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= From 524859a3299d7901e6ccf6f1333f294207426b90 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:12:52 +0000 Subject: [PATCH 05/31] deps: bump github.com/Azure/azure-sdk-for-go/sdk/azidentity from 1.4.0 to 1.5.1 (#2522) deps: bump github.com/Azure/azure-sdk-for-go/sdk/azidentity Bumps [github.com/Azure/azure-sdk-for-go/sdk/azidentity](https://github.com/Azure/azure-sdk-for-go) from 1.4.0 to 1.5.1. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.4.0...sdk/internal/v1.5.1) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azidentity dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index aa55dc96855..7c49eef4bdc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21 require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 github.com/Masterminds/semver v1.5.0 github.com/Microsoft/go-winio v0.6.1 @@ -56,7 +56,7 @@ require ( code.cloudfoundry.org/clock v1.0.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/containerd/cgroups v1.1.0 // indirect @@ -94,7 +94,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/common v0.46.0 github.com/prometheus/procfs v0.12.0 // indirect @@ -138,7 +138,7 @@ require ( require ( github.com/containerd/containerd v1.6.26 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect - github.com/golang-jwt/jwt/v5 v5.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.2.0 // indirect github.com/rootless-containers/rootlesskit v1.1.1 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect diff --git a/go.sum b/go.sum index 0a525538b2a..c8d2a4e47ef 100644 --- a/go.sum +++ b/go.sum @@ -4,16 +4,16 @@ code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2 code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1 h1:WpB/QDNLpMw72xHJc34BNNykqSOeEJDAWkhf0u12/Jk= -github.com/AzureAD/microsoft-authentication-library-for-go v1.1.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -98,8 +98,8 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= -github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.0 h1:d/ix8ftRUorsN+5eMIlF4T6J8CAt9rch3My2winC1Jw= +github.com/golang-jwt/jwt/v5 v5.2.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= @@ -214,8 +214,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaR github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -355,12 +355,12 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= From 7ae0fa5a03327daf9da964ea2b47160e4f3bfb39 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Jan 2024 19:13:16 +0000 Subject: [PATCH 06/31] deps: bump google.golang.org/grpc from 1.60.1 to 1.61.0 (#2543) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.60.1 to 1.61.0. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.60.1...v1.61.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7c49eef4bdc..51abe43000f 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,7 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sys v0.16.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect - google.golang.org/grpc v1.60.1 + google.golang.org/grpc v1.61.0 google.golang.org/protobuf v1.32.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 k8s.io/api v0.28.5 diff --git a/go.sum b/go.sum index c8d2a4e47ef..349d36e0337 100644 --- a/go.sum +++ b/go.sum @@ -413,8 +413,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From c976cfaaeb5ea2df56543f3097bdea0c0cd61173 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 25 Jan 2024 10:21:53 -0600 Subject: [PATCH 07/31] deps: bump github.com/Azure/azure-container-networking from 1.5.18 to 1.5.19 in /azure-ipam (#2539) deps: bump github.com/Azure/azure-container-networking in /azure-ipam Bumps [github.com/Azure/azure-container-networking](https://github.com/Azure/azure-container-networking) from 1.5.18 to 1.5.19. - [Release notes](https://github.com/Azure/azure-container-networking/releases) - [Commits](https://github.com/Azure/azure-container-networking/compare/v1.5.18...v1.5.19) --- updated-dependencies: - dependency-name: github.com/Azure/azure-container-networking dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- azure-ipam/go.mod | 2 +- azure-ipam/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-ipam/go.mod b/azure-ipam/go.mod index 94762d0c944..e697aa8ad50 100644 --- a/azure-ipam/go.mod +++ b/azure-ipam/go.mod @@ -3,7 +3,7 @@ module github.com/Azure/azure-container-networking/azure-ipam go 1.21 require ( - github.com/Azure/azure-container-networking v1.5.18 + github.com/Azure/azure-container-networking v1.5.19 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.4.0 github.com/pkg/errors v0.9.1 diff --git a/azure-ipam/go.sum b/azure-ipam/go.sum index a3db83ef10d..0167d9020cd 100644 --- a/azure-ipam/go.sum +++ b/azure-ipam/go.sum @@ -2,8 +2,8 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -github.com/Azure/azure-container-networking v1.5.18 h1:PBFO1ON4yvgw1NKmzvpUuSAVTqzK1wtCmzdyZ3kE1ic= -github.com/Azure/azure-container-networking v1.5.18/go.mod h1:AJ5ZTZ0UBNBBIS3DAr883DrdHnCJRR3vBpPMDSCZ+nA= +github.com/Azure/azure-container-networking v1.5.19 h1:vdSUU0EjyUu5ePdJyiBT2bOUH24hFI6sB6eWocvfXr4= +github.com/Azure/azure-container-networking v1.5.19/go.mod h1:T0wq4BcGMX+S0ue3grruyej/C7GbhjN2B2ciHxqPNeg= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= From 0c9cfb612619150bed5df5369a8f37e5f6ea3c6b Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 25 Jan 2024 08:33:46 -0800 Subject: [PATCH 08/31] ci: E2E Framework [Kubernetes steps] [3/6] (#2529) * add kubernetes steps * update port forwarder * go mod * revert port-forwarding changes to be taken in future --- go.mod | 3 +- go.sum | 6 +- .../kubernetes/create-agnhost-statefulset.go | 162 +++++++++++ .../kubernetes/create-kapinger-deployment.go | 256 ++++++++++++++++++ .../kubernetes/create-network-policy.go | 114 ++++++++ .../framework/kubernetes/create-resource.go | 209 ++++++++++++++ .../framework/kubernetes/delete-resource.go | 162 +++++++++++ test/e2e/framework/kubernetes/exec-pod.go | 89 ++++++ .../e2e/framework/kubernetes/port-forward.txt | 163 +++++++++++ .../framework/kubernetes/wait-pod-ready.go | 64 +++++ test/integration/portforward.go | 2 +- 11 files changed, 1226 insertions(+), 4 deletions(-) create mode 100644 test/e2e/framework/kubernetes/create-agnhost-statefulset.go create mode 100644 test/e2e/framework/kubernetes/create-kapinger-deployment.go create mode 100644 test/e2e/framework/kubernetes/create-network-policy.go create mode 100644 test/e2e/framework/kubernetes/create-resource.go create mode 100644 test/e2e/framework/kubernetes/delete-resource.go create mode 100644 test/e2e/framework/kubernetes/exec-pod.go create mode 100644 test/e2e/framework/kubernetes/port-forward.txt create mode 100644 test/e2e/framework/kubernetes/wait-pod-ready.go diff --git a/go.mod b/go.mod index 51abe43000f..52d4d339df8 100644 --- a/go.mod +++ b/go.mod @@ -123,7 +123,7 @@ require ( gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/component-base v0.28.3 // indirect + k8s.io/component-base v0.28.5 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect @@ -132,6 +132,7 @@ require ( require ( golang.org/x/sync v0.6.0 gotest.tools/v3 v3.5.1 + k8s.io/kubectl v0.28.5 sigs.k8s.io/yaml v1.4.0 ) diff --git a/go.sum b/go.sum index 349d36e0337..ecd557e43e9 100644 --- a/go.sum +++ b/go.sum @@ -462,14 +462,16 @@ k8s.io/apimachinery v0.28.5 h1:EEj2q1qdTcv2p5wl88KavAn3VlFRjREgRu8Sm/EuMPY= k8s.io/apimachinery v0.28.5/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= k8s.io/client-go v0.28.5 h1:6UNmc33vuJhh3+SAOEKku3QnKa+DtPKGnhO2MR0IEbk= k8s.io/client-go v0.28.5/go.mod h1:+pt086yx1i0HAlHzM9S+RZQDqdlzuXFl4hY01uhpcpA= -k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= -k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/component-base v0.28.5 h1:uFCW7USa8Fpme8dVtn2ZrdVaUPBRDwYJ+kNrV9OO1Cc= +k8s.io/component-base v0.28.5/go.mod h1:gw2d8O28okS9RrsPuJnD2mFl2It0HH9neHiGi2xoXcY= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kubectl v0.28.5 h1:jq8xtiCCZPR8Cl/Qe1D7bLU0h8KtcunwfROqIekCUeU= +k8s.io/kubectl v0.28.5/go.mod h1:9WiwzqeKs3vLiDtEQPbjhqqysX+BIVMLt7C7gN+T5w8= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go new file mode 100644 index 00000000000..6be316c1447 --- /dev/null +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -0,0 +1,162 @@ +package k8s + +import ( + "context" + "fmt" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod") + +const ( + AgnhostHTTPPort = 80 + AgnhostReplicas = 1 + + defaultTimeoutSeconds = 300 + defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 60 + defaultHTTPClientTimeout = 2 * time.Second +) + +type CreateAgnhostStatefulSet struct { + AgnhostName string + AgnhostNamespace string + KubeConfigFilePath string +} + +func (c *CreateAgnhostStatefulSet) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", c.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), defaultTimeoutSeconds*time.Second) + defer cancel() + + agnhostStatefulest := c.getAgnhostDeployment() + + err = CreateResource(ctx, agnhostStatefulest, clientset) + if err != nil { + return fmt.Errorf("error agnhost component: %w", err) + } + + selector, exists := agnhostStatefulest.Spec.Selector.MatchLabels["app"] + if !exists { + return fmt.Errorf("missing label \"app=%s\" from agnhost statefulset: %w", c.AgnhostName, ErrLabelMissingFromPod) + } + + labelSelector := fmt.Sprintf("app=%s", selector) + err = WaitForPodReady(ctx, clientset, c.AgnhostNamespace, labelSelector) + if err != nil { + return fmt.Errorf("error waiting for agnhost pod to be ready: %w", err) + } + + return nil +} + +func (c *CreateAgnhostStatefulSet) Prevalidate() error { + return nil +} + +func (c *CreateAgnhostStatefulSet) Postvalidate() error { + return nil +} + +func (c *CreateAgnhostStatefulSet) getAgnhostDeployment() *appsv1.StatefulSet { + reps := int32(AgnhostReplicas) + + return &appsv1.StatefulSet{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: c.AgnhostName, + Namespace: c.AgnhostNamespace, + }, + Spec: appsv1.StatefulSetSpec{ + Replicas: &reps, + Selector: &metaV1.LabelSelector{ + MatchLabels: map[string]string{ + "app": c.AgnhostName, + "k8s-app": "agnhost", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metaV1.ObjectMeta{ + Labels: map[string]string{ + "app": c.AgnhostName, + "k8s-app": "agnhost", + }, + Annotations: map[string]string{ + "policy.cilium.io/proxy-visibility": "", + }, + }, + + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + // prefer an even spread across the cluster to avoid scheduling on the same node + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: MaxAffinityWeight, + PodAffinityTerm: v1.PodAffinityTerm{ + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metaV1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": "agnhost", + }, + }, + }, + }, + }, + }, + }, + Containers: []v1.Container{ + { + Name: c.AgnhostName, + Image: "acnpublic.azurecr.io/agnhost:2.40", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "memory": resource.MustParse("20Mi"), + }, + Limits: v1.ResourceList{ + "memory": resource.MustParse("20Mi"), + }, + }, + Command: []string{ + "/agnhost", + }, + Args: []string{ + "serve-hostname", + "--http", + "--port", + strconv.Itoa(AgnhostHTTPPort), + }, + + Ports: []v1.ContainerPort{ + { + ContainerPort: AgnhostHTTPPort, + }, + }, + Env: []v1.EnvVar{}, + }, + }, + }, + }, + }, + } +} diff --git a/test/e2e/framework/kubernetes/create-kapinger-deployment.go b/test/e2e/framework/kubernetes/create-kapinger-deployment.go new file mode 100644 index 00000000000..b65732d2ee8 --- /dev/null +++ b/test/e2e/framework/kubernetes/create-kapinger-deployment.go @@ -0,0 +1,256 @@ +package k8s + +import ( + "context" + "fmt" + "strconv" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/resource" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + KapingerHTTPPort = 8080 + KapingerTCPPort = 8085 + KapingerUDPPort = 8086 + MaxAffinityWeight = 100 +) + +type CreateKapingerDeployment struct { + KapingerNamespace string + KapingerReplicas string + KubeConfigFilePath string +} + +func (c *CreateKapingerDeployment) Run() error { + _, err := strconv.Atoi(c.KapingerReplicas) + if err != nil { + return fmt.Errorf("error converting replicas to int for Kapinger replicas: %w", err) + } + + config, err := clientcmd.BuildConfigFromFlags("", c.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + resources := []runtime.Object{ + c.GetKapingerService(), + c.GetKapingerServiceAccount(), + c.GetKapingerClusterRole(), + c.GetKapingerClusterRoleBinding(), + c.GetKapingerDeployment(), + } + + for i := range resources { + err = CreateResource(ctx, resources[i], clientset) + if err != nil { + return fmt.Errorf("error kapinger component: %w", err) + } + } + + return nil +} + +func (c *CreateKapingerDeployment) Prevalidate() error { + return nil +} + +func (c *CreateKapingerDeployment) Postvalidate() error { + return nil +} + +func (c *CreateKapingerDeployment) GetKapingerDeployment() *appsv1.Deployment { + replicas, err := strconv.ParseInt(c.KapingerReplicas, 10, 32) + if err != nil { + fmt.Println("Error converting replicas to int for Kapinger replicas: ", err) + return nil + } + reps := int32(replicas) + + return &appsv1.Deployment{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Deployment", + APIVersion: "apps/v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "kapinger", + Namespace: c.KapingerNamespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &reps, + Selector: &metaV1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "kapinger", + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metaV1.ObjectMeta{ + Labels: map[string]string{ + "app": "kapinger", + "server": "good", + }, + }, + + Spec: v1.PodSpec{ + Affinity: &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + // prefer an even spread across the cluster to avoid scheduling on the same node + PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{ + { + Weight: MaxAffinityWeight, + PodAffinityTerm: v1.PodAffinityTerm{ + TopologyKey: "kubernetes.io/hostname", + LabelSelector: &metaV1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "kapinger", + }, + }, + }, + }, + }, + }, + }, + ServiceAccountName: "kapinger-sa", + Containers: []v1.Container{ + { + Name: "kapinger", + Image: "acnpublic.azurecr.io/kapinger:be57650", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "memory": resource.MustParse("20Mi"), + }, + Limits: v1.ResourceList{ + "memory": resource.MustParse("20Mi"), + }, + }, + Ports: []v1.ContainerPort{ + { + ContainerPort: KapingerHTTPPort, + }, + }, + Env: []v1.EnvVar{ + { + Name: "TARGET_TYPE", + Value: "service", + }, + { + Name: "HTTP_PORT", + Value: strconv.Itoa(KapingerHTTPPort), + }, + { + Name: "TCP_PORT", + Value: strconv.Itoa(KapingerTCPPort), + }, + { + Name: "UDP_PORT", + Value: strconv.Itoa(KapingerUDPPort), + }, + }, + }, + }, + }, + }, + }, + } +} + +func (c *CreateKapingerDeployment) GetKapingerService() *v1.Service { + return &v1.Service{ + TypeMeta: metaV1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "kapinger-service", + Namespace: c.KapingerNamespace, + Labels: map[string]string{ + "app": "kapinger", + }, + }, + Spec: v1.ServiceSpec{ + Selector: map[string]string{ + "app": "kapinger", + }, + Ports: []v1.ServicePort{ + { + Port: KapingerHTTPPort, + Protocol: v1.ProtocolTCP, + TargetPort: intstr.FromInt(KapingerHTTPPort), + }, + }, + }, + } +} + +func (c *CreateKapingerDeployment) GetKapingerServiceAccount() *v1.ServiceAccount { + return &v1.ServiceAccount{ + TypeMeta: metaV1.TypeMeta{ + Kind: "ServiceAccount", + APIVersion: "v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "kapinger-sa", + Namespace: c.KapingerNamespace, + }, + } +} + +func (c *CreateKapingerDeployment) GetKapingerClusterRole() *rbacv1.ClusterRole { + return &rbacv1.ClusterRole{ + TypeMeta: metaV1.TypeMeta{ + Kind: "ClusterRole", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "kapinger-role", + Namespace: c.KapingerNamespace, + }, + Rules: []rbacv1.PolicyRule{ + { + APIGroups: []string{""}, + Resources: []string{"services", "pods"}, + Verbs: []string{"get", "list"}, + }, + }, + } +} + +func (c *CreateKapingerDeployment) GetKapingerClusterRoleBinding() *rbacv1.ClusterRoleBinding { + return &rbacv1.ClusterRoleBinding{ + TypeMeta: metaV1.TypeMeta{ + Kind: "ClusterRoleBinding", + APIVersion: "rbac.authorization.k8s.io/v1", + }, + ObjectMeta: metaV1.ObjectMeta{ + Name: "kapinger-rolebinding", + Namespace: c.KapingerNamespace, + }, + Subjects: []rbacv1.Subject{ + { + Kind: "ServiceAccount", + Name: "kapinger-sa", + Namespace: c.KapingerNamespace, + }, + }, + RoleRef: rbacv1.RoleRef{ + APIGroup: "rbac.authorization.k8s.io", + Kind: "ClusterRole", + Name: "kapinger-role", + }, + } +} diff --git a/test/e2e/framework/kubernetes/create-network-policy.go b/test/e2e/framework/kubernetes/create-network-policy.go new file mode 100644 index 00000000000..fa1171a257e --- /dev/null +++ b/test/e2e/framework/kubernetes/create-network-policy.go @@ -0,0 +1,114 @@ +package k8s + +import ( + "context" + "fmt" + "strings" + + networkingv1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + Egress = "egress" + Ingress = "ingress" +) + +type CreateDenyAllNetworkPolicy struct { + NetworkPolicyNamespace string + KubeConfigFilePath string + DenyAllLabelSelector string +} + +func (c *CreateDenyAllNetworkPolicy) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", c.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + agnhostStatefulest := getNetworkPolicy(c.NetworkPolicyNamespace, c.DenyAllLabelSelector) + err = CreateResource(ctx, agnhostStatefulest, clientset) + if err != nil { + return fmt.Errorf("error creating simple deny-all network policy: %w", err) + } + + return nil +} + +func getNetworkPolicy(namespace, labelSelector string) *networkingv1.NetworkPolicy { + labelSelectorSlice := strings.Split(labelSelector, "=") + return &networkingv1.NetworkPolicy{ + ObjectMeta: metav1.ObjectMeta{ + Name: "deny-all", + Namespace: namespace, + }, + Spec: networkingv1.NetworkPolicySpec{ + PodSelector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + labelSelectorSlice[0]: labelSelectorSlice[1], + }, + }, + PolicyTypes: []networkingv1.PolicyType{ + networkingv1.PolicyTypeIngress, + networkingv1.PolicyTypeEgress, + }, + Egress: []networkingv1.NetworkPolicyEgressRule{}, + Ingress: []networkingv1.NetworkPolicyIngressRule{}, + }, + } +} + +func (c *CreateDenyAllNetworkPolicy) Prevalidate() error { + return nil +} + +func (c *CreateDenyAllNetworkPolicy) Postvalidate() error { + return nil +} + +type DeleteDenyAllNetworkPolicy struct { + NetworkPolicyNamespace string + KubeConfigFilePath string + DenyAllLabelSelector string +} + +func (d *DeleteDenyAllNetworkPolicy) Run() error { + config, err := clientcmd.BuildConfigFromFlags("", d.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + agnhostStatefulest := getNetworkPolicy(d.NetworkPolicyNamespace, d.DenyAllLabelSelector) + err = DeleteResource(ctx, agnhostStatefulest, clientset) + if err != nil { + return fmt.Errorf("error creating simple deny-all network policy: %w", err) + } + + return nil +} + +func (d *DeleteDenyAllNetworkPolicy) Prevalidate() error { + return nil +} + +func (d *DeleteDenyAllNetworkPolicy) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/kubernetes/create-resource.go b/test/e2e/framework/kubernetes/create-resource.go new file mode 100644 index 00000000000..8e7eafda461 --- /dev/null +++ b/test/e2e/framework/kubernetes/create-resource.go @@ -0,0 +1,209 @@ +package k8s + +import ( + "context" + "fmt" + "log" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" +) + +var ( + ErrUnknownResourceType = fmt.Errorf("unknown resource type") + ErrCreateNilResource = fmt.Errorf("cannot create nil resource") +) + +func CreateResource(ctx context.Context, obj runtime.Object, clientset *kubernetes.Clientset) error { //nolint:gocyclo //this is just boilerplate code + if obj == nil { + return ErrCreateNilResource + } + + switch o := obj.(type) { + case *appsv1.DaemonSet: + log.Printf("Creating/Updating DaemonSet \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().DaemonSets(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create DaemonSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update DaemonSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *appsv1.Deployment: + log.Printf("Creating/Updating Deployment \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().Deployments(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create Deployment \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update Deployment \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *appsv1.StatefulSet: + log.Printf("Creating/Updating StatefulSet \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().StatefulSets(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create StatefulSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update StatefulSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *v1.Service: + log.Printf("Creating/Updating Service \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().Services(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create Service \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update Service \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *v1.ServiceAccount: + log.Printf("Creating/Updating ServiceAccount \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().ServiceAccounts(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ServiceAccount \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update ServiceAccount \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.Role: + log.Printf("Creating/Updating Role \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.RbacV1().Roles(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create Role \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update Role \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.RoleBinding: + log.Printf("Creating/Updating RoleBinding \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.RbacV1().RoleBindings(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create RoleBinding \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update RoleBinding \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.ClusterRole: + log.Printf("Creating/Updating ClusterRole \"%s\"...\n", o.Name) + client := clientset.RbacV1().ClusterRoles() + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ClusterRole \"%s\": %w", o.Name, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update ClusterRole \"%s\": %w", o.Name, err) + } + + case *rbacv1.ClusterRoleBinding: + log.Printf("Creating/Updating ClusterRoleBinding \"%s\"...\n", o.Name) + client := clientset.RbacV1().ClusterRoleBindings() + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ClusterRoleBinding \"%s\": %w", o.Name, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update ClusterRoleBinding \"%s\": %w", o.Name, err) + } + + case *v1.ConfigMap: + log.Printf("Creating/Updating ConfigMap \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().ConfigMaps(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create ConfigMap \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update ConfigMap \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *networkingv1.NetworkPolicy: + log.Printf("Creating/Updating NetworkPolicy \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.NetworkingV1().NetworkPolicies(o.Namespace) + _, err := client.Get(ctx, o.Name, metaV1.GetOptions{}) + if errors.IsNotFound(err) { + _, err = client.Create(ctx, o, metaV1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create NetworkPolicy \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + return nil + } + _, err = client.Update(ctx, o, metaV1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to create/update NetworkPolicy \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + default: + return fmt.Errorf("unknown object type: %T, err: %w", obj, ErrUnknownResourceType) + } + return nil +} diff --git a/test/e2e/framework/kubernetes/delete-resource.go b/test/e2e/framework/kubernetes/delete-resource.go new file mode 100644 index 00000000000..d0b6a0adc52 --- /dev/null +++ b/test/e2e/framework/kubernetes/delete-resource.go @@ -0,0 +1,162 @@ +package k8s + +import ( + "context" + "fmt" + "log" + + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + networkingv1 "k8s.io/api/networking/v1" + rbacv1 "k8s.io/api/rbac/v1" + "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes" +) + +var ErrDeleteNilResource = fmt.Errorf("cannot create nil resource") + +func DeleteResource(ctx context.Context, obj runtime.Object, clientset *kubernetes.Clientset) error { //nolint:gocyclo //this is just boilerplate code + if obj == nil { + return ErrCreateNilResource + } + + switch o := obj.(type) { + case *appsv1.DaemonSet: + log.Printf("Deleting DaemonSet \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().DaemonSets(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("DaemonSet \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete DaemonSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *appsv1.Deployment: + log.Printf("Creating/Updating Deployment \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().Deployments(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("Deployment \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete Deployment \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *appsv1.StatefulSet: + log.Printf("Creating/Updating StatefulSet \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.AppsV1().StatefulSets(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("StatefulSet \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete StatefulSet \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *v1.Service: + log.Printf("Creating/Updating Service \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().Services(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("Service \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete Service \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *v1.ServiceAccount: + log.Printf("Creating/Updating ServiceAccount \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().ServiceAccounts(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("ServiceAccount \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete ServiceAccount \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.Role: + log.Printf("Creating/Updating Role \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.RbacV1().Roles(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("Role \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete Role \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.RoleBinding: + log.Printf("Creating/Updating RoleBinding \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.RbacV1().RoleBindings(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("RoleBinding \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete RoleBinding \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.ClusterRole: + log.Printf("Creating/Updating ClusterRole \"%s\"...\n", o.Name) + client := clientset.RbacV1().ClusterRoles() + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("ClusterRole \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete ClusterRole \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *rbacv1.ClusterRoleBinding: + log.Printf("Creating/Updating ClusterRoleBinding \"%s\"...\n", o.Name) + client := clientset.RbacV1().ClusterRoleBindings() + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("ClusterRoleBinding \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete ClusterRoleBinding \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *v1.ConfigMap: + log.Printf("Creating/Updating ConfigMap \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.CoreV1().ConfigMaps(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("ConfigMap \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete ConfigMap \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + case *networkingv1.NetworkPolicy: + log.Printf("Creating/Updating NetworkPolicy \"%s\" in namespace \"%s\"...\n", o.Name, o.Namespace) + client := clientset.NetworkingV1().NetworkPolicies(o.Namespace) + err := client.Delete(ctx, o.Name, metaV1.DeleteOptions{}) + if err != nil { + if errors.IsNotFound(err) { + log.Printf("NetworkPolicy \"%s\" in namespace \"%s\" does not exist\n", o.Name, o.Namespace) + return nil + } + return fmt.Errorf("failed to delete NetworkPolicy \"%s\" in namespace \"%s\": %w", o.Name, o.Namespace, err) + } + + default: + return fmt.Errorf("unknown object type: %T, err: %w", obj, ErrUnknownResourceType) + } + return nil +} diff --git a/test/e2e/framework/kubernetes/exec-pod.go b/test/e2e/framework/kubernetes/exec-pod.go new file mode 100644 index 00000000000..aaf71c49459 --- /dev/null +++ b/test/e2e/framework/kubernetes/exec-pod.go @@ -0,0 +1,89 @@ +package k8s + +import ( + "context" + "fmt" + "log" + "os" + "strings" + + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/remotecommand" + "k8s.io/kubectl/pkg/scheme" +) + +const ExecSubResources = "exec" + +type ExecInPod struct { + PodNamespace string + KubeConfigFilePath string + PodName string + Command string +} + +func (e *ExecInPod) Run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + err := ExecPod(ctx, e.KubeConfigFilePath, e.PodNamespace, e.PodName, e.Command) + if err != nil { + return fmt.Errorf("error executing command [%s]: %w", e.Command, err) + } + + return nil +} + +func (e *ExecInPod) Prevalidate() error { + return nil +} + +func (e *ExecInPod) Postvalidate() error { + return nil +} + +func ExecPod(ctx context.Context, kubeConfigFilePath, namespace, podName, command string) error { + config, err := clientcmd.BuildConfigFromFlags("", kubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("error creating Kubernetes client: %w", err) + } + + req := clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName). + Namespace(namespace).SubResource(ExecSubResources) + option := &v1.PodExecOptions{ + Command: strings.Fields(command), + Stdin: true, + Stdout: true, + Stderr: true, + TTY: false, + } + + req.VersionedParams( + option, + scheme.ParameterCodec, + ) + + exec, err := remotecommand.NewSPDYExecutor(config, "POST", req.URL()) + if err != nil { + return fmt.Errorf("error creating executor: %w", err) + } + + log.Printf("executing command \"%s\" on pod \"%s\" in namespace \"%s\"...", command, podName, namespace) + err = exec.StreamWithContext(ctx, remotecommand.StreamOptions{ + Stdin: os.Stdin, + Stdout: os.Stdout, + Stderr: os.Stderr, + }) + + if err != nil { + return fmt.Errorf("error executing command: %w", err) + } + + return nil +} diff --git a/test/e2e/framework/kubernetes/port-forward.txt b/test/e2e/framework/kubernetes/port-forward.txt new file mode 100644 index 00000000000..7dcabe496da --- /dev/null +++ b/test/e2e/framework/kubernetes/port-forward.txt @@ -0,0 +1,163 @@ +// todo: matmerr, this is just going to remain broken until it can be validated with scenarios pr + +package k8s + +import ( + "context" + "fmt" + "log" + "net/http" + "strconv" + "time" + + k8s "github.com/Azure/azure-container-networking/test/integration" + "github.com/Azure/azure-container-networking/test/internal/retry" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +const ( + defaultTimeoutSeconds = 300 + defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 60 + defaultHTTPClientTimeout = 2 * time.Second +) + +var ( + ErrNoPodWithLabelFound = fmt.Errorf("no pod with label found with matching pod affinity") + + defaultRetrier = retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} +) + +type PortForward struct { + Namespace string + LabelSelector string + LocalPort string + RemotePort string + KubeConfigFilePath string + OptionalLabelAffinity string + + // local properties + pf *k8s.PortForwarder + portForwardHandle k8s.PortForwardStreamHandle +} + +func (p *PortForward) Run() error { + lport, _ := strconv.Atoi(p.LocalPort) + rport, _ := strconv.Atoi(p.RemotePort) + + pctx := context.Background() + portForwardCtx, cancel := context.WithTimeout(pctx, defaultTimeoutSeconds*time.Second) + defer cancel() + + config, err := clientcmd.BuildConfigFromFlags("", p.KubeConfigFilePath) + if err != nil { + return fmt.Errorf("error building kubeconfig: %w", err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return fmt.Errorf("could not create clientset: %w", err) + } + + p.pf, err = k8s.NewPortForwarder(config) + if err != nil { + return fmt.Errorf("could not create port forwarder: %w", err) + } + + // if we have an optional label affinity, find a pod with that label, on the same node as a pod with the label selector + targetPodName := "" + if p.OptionalLabelAffinity != "" { + // get all pods with label + targetPodName, err = p.findPodsWithAffinity(pctx, clientset) + if err != nil { + return fmt.Errorf("could not find pod with affinity: %w", err) + } + } + + portForwardFn := func() error { + log.Printf("attempting port forward to a pod with label \"%s\", in namespace \"%s\"...\n", p.LabelSelector, p.Namespace) + + var handle k8s.PortForwardStreamHandle + + // if we have a pod name (likely from affinity above), use it, otherwise use label selector + if targetPodName != "" { + handle, err = p.pf.ForwardWithPodName(pctx, p.Namespace, targetPodName, lport, rport) + if err != nil { + return fmt.Errorf("could not start port forward: %w", err) + } + } else { + handle, err = p.pf.ForwardWithLabelSelector(pctx, p.Namespace, p.LabelSelector, lport, rport) + if err != nil { + return fmt.Errorf("could not start port forward: %w", err) + } + } + + // verify port forward succeeded + client := http.Client{ + Timeout: defaultHTTPClientTimeout, + } + resp, err := client.Get(handle.URL()) //nolint + if err != nil { + log.Printf("port forward validation HTTP request to %s failed: %v\n", handle.URL(), err) + handle.Stop() + return fmt.Errorf("port forward validation HTTP request to %s failed: %w", handle.URL(), err) + } + defer resp.Body.Close() + + log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", handle.URL(), resp.Status) + p.portForwardHandle = handle + + return nil + } + + if err = defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil { + return fmt.Errorf("could not start port forward within %ds: %w", defaultTimeoutSeconds, err) + } + log.Printf("successfully port forwarded to \"%s\"\n", p.portForwardHandle.URL()) + return nil +} + +func (p *PortForward) findPodsWithAffinity(ctx context.Context, clientset *kubernetes.Clientset) (string, error) { + targetPods, errAffinity := clientset.CoreV1().Pods(p.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: p.LabelSelector, + FieldSelector: "status.phase=Running", + }) + if errAffinity != nil { + return "", fmt.Errorf("could not list pods in %q with label %q: %w", p.Namespace, p.LabelSelector, errAffinity) + } + + affinityPods, errAffinity := clientset.CoreV1().Pods(p.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: p.OptionalLabelAffinity, + FieldSelector: "status.phase=Running", + }) + if errAffinity != nil { + return "", fmt.Errorf("could not list affinity pods in %q with label %q: %w", p.Namespace, p.OptionalLabelAffinity, errAffinity) + } + + // keep track of where the affinity pods are scheduled + affinityNodes := make(map[string]bool) + for i := range affinityPods.Items { + affinityNodes[affinityPods.Items[i].Spec.NodeName] = true + } + + // if a pod is found on the same node as an affinity pod, use it + for i := range targetPods.Items { + if affinityNodes[targetPods.Items[i].Spec.NodeName] { + // found a pod with the specified label, on a node with the optional label affinity + return targetPods.Items[i].Name, nil + } + } + + return "", fmt.Errorf("could not find a pod with label \"%s\", on a node that also has a pod with label \"%s\": %w", p.LabelSelector, p.OptionalLabelAffinity, ErrNoPodWithLabelFound) +} + +func (p *PortForward) Prevalidate() error { + return nil +} + +func (p *PortForward) Postvalidate() error { + p.portForwardHandle.Stop() + return nil +} diff --git a/test/e2e/framework/kubernetes/wait-pod-ready.go b/test/e2e/framework/kubernetes/wait-pod-ready.go new file mode 100644 index 00000000000..41a9f1d4016 --- /dev/null +++ b/test/e2e/framework/kubernetes/wait-pod-ready.go @@ -0,0 +1,64 @@ +package k8s + +import ( + "context" + "fmt" + "log" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" +) + +const ( + RetryTimeoutPodsReady = 5 * time.Minute + RetryIntervalPodsReady = 5 * time.Second +) + +func WaitForPodReady(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelSelector string) error { + podReadyMap := make(map[string]bool) + + conditionFunc := wait.ConditionWithContextFunc(func(context.Context) (bool, error) { + // get a list of all cilium pods + var podList *corev1.PodList + podList, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{LabelSelector: labelSelector}) + if err != nil { + return false, fmt.Errorf("error listing Pods: %w", err) + } + + if len(podList.Items) == 0 { + log.Printf("no pods found in namespace \"%s\" with label \"%s\"", namespace, labelSelector) + return false, nil + } + + // check each indviidual pod to see if it's in Running state + for i := range podList.Items { + var pod *corev1.Pod + pod, err = clientset.CoreV1().Pods(namespace).Get(ctx, podList.Items[i].Name, metav1.GetOptions{}) + if err != nil { + return false, fmt.Errorf("error getting Pod: %w", err) + } + + // Check the Pod phase + if pod.Status.Phase != corev1.PodRunning { + log.Printf("pod \"%s\" is not in Running state yet. Waiting...\n", pod.Name) + return false, nil + } + if !podReadyMap[pod.Name] { + log.Printf("pod \"%s\" is in Running state\n", pod.Name) + podReadyMap[pod.Name] = true + } + } + log.Printf("all pods in namespace \"%s\" with label \"%s\" are in Running state\n", namespace, labelSelector) + return true, nil + }) + + // wait until all cilium pods are in Running state condition is true + err := wait.PollUntilContextCancel(ctx, RetryIntervalPodsReady, true, conditionFunc) + if err != nil { + return fmt.Errorf("error waiting for pods in namespace \"%s\" with label \"%s\" to be in Running state: %w", namespace, labelSelector, err) + } + return nil +} diff --git a/test/integration/portforward.go b/test/integration/portforward.go index 605bd706021..5170b91dc79 100644 --- a/test/integration/portforward.go +++ b/test/integration/portforward.go @@ -81,7 +81,7 @@ func (p *PortForwarder) Forward(ctx context.Context) error { return fmt.Errorf("no pods found in %q with label %q", p.opts.Namespace, p.opts.LabelSelector) //nolint:goerr113 //no specific handling expected } - randomIndex := rand.Intn(len(pods.Items)) + randomIndex := rand.Intn(len(pods.Items)) //nolint:gosec //this is going to be revised in the future anyways, avoid random pods podName := pods.Items[randomIndex].Name portForwardURL := p.clientset.CoreV1().RESTClient().Post(). Resource("pods"). From ef2ae4be38fe8d582e10140f7064504e3cf3abbb Mon Sep 17 00:00:00 2001 From: Mike Zappa Date: Thu, 25 Jan 2024 10:53:55 -0700 Subject: [PATCH 09/31] Enable Hubble in PR pipeline with additional stage (#2534) enable hubble in pr with additional stage --- .pipelines/pipeline.yaml | 12 + .../cilium-overlay-e2e-job-template.yaml | 86 ++++++++ .../cilium-overlay-e2e-step-template.yaml | 206 ++++++++++++++++++ .../cilium-agent/templates/daemonset.tpl | 14 +- .../cilium-operator/templates/deployment.tpl | 2 +- 5 files changed, 312 insertions(+), 8 deletions(-) create mode 100644 .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml create mode 100644 .pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 1a9d8c3583e..8603f749bbd 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -420,6 +420,18 @@ stages: k8sVersion: "" dependsOn: "test" + # Cilium Overlay with hubble E2E tests + - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml + parameters: + name: "cilium_h_overlay_e2e" + displayName: Cilium on AKS Overlay with Hubble + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cilwhleovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "test" + testHubble: true + # Azure Overlay E2E tests - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-job-template.yaml parameters: diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml new file mode 100644 index 00000000000..01ab0823ebc --- /dev/null +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml @@ -0,0 +1,86 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + os: "linux" + testHubble: false + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cilium Overlay Test Suite - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: cilium-overlay-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + testHubble: ${{ parameters.testHubble }} + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + dependsOn: ${{ parameters.name }} + datapath: true + dns: true + portforward: true + service: true + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml new file mode 100644 index 00000000000..0e59910c8b5 --- /dev/null +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -0,0 +1,206 @@ +parameters: + name: "" + clusterName: "" + testHubble: false + +steps: + - bash: | + echo $UID + sudo rm -rf $(System.DefaultWorkingDirectory)/* + displayName: "Set up OS environment" + + - checkout: self + + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-config/cilium-config-hubble.yaml + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-agent/files + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-operator/files + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl | kubectl apply -f - + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl | kubectl apply -f - + # Use different file directories for nightly and current cilium version + name: "installCilium" + displayName: "Install Cilium on AKS Overlay" + + - script: | + echo "install cilium CLI" + if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then + echo "Cilium Agent Version ${BASH_REMATCH[0]}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt) + else + echo "Cilium Agent Version ${CILIUM_VERSION_TAG}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) + fi + CLI_ARCH=amd64 + if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum + sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin + rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + cilium status + cilium version + name: "installCiliumCLI" + displayName: "Install Cilium CLI" + + - script: | + echo "Start Azilium E2E Tests on Overlay Cluster" + if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ] + then + CNS=$(CNS_VERSION) IPAM=$(AZURE_IPAM_VERSION) && echo "Running nightly" + else + CNS=$(make cns-version) IPAM=$(make azure-ipam-version) + fi + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true + retryCountOnTaskFailure: 3 + name: "aziliumTest" + displayName: "Run Azilium E2E on AKS Overlay" + + - script: | + echo "Status of the nodes and pods after the test" + kubectl get nodes -o wide + kubectl get pods -A -o wide + echo "Logs will be available as a build artifact" + ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ + echo $ARTIFACT_DIR + sudo rm -rf $ARTIFACT_DIR + sudo mkdir $ARTIFACT_DIR + sudo cp test/integration/logs/* $ARTIFACT_DIR + name: "GetLogs" + displayName: "Get logs" + condition: always() + + - task: PublishBuildArtifacts@1 + inputs: + artifactName: test-output + pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" + condition: always() + + - script: | + kubectl get pods -A + echo "Waiting < 2 minutes for cilium to be ready" + # Ensure Cilium is ready Xm\Xs + cilium status --wait --wait-duration 2m + retryCountOnTaskFailure: 3 + name: "CiliumStatus" + displayName: "Cilium Status" + + - script: | + echo "Run Cilium Connectivity Tests" + cilium status + cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' + retryCountOnTaskFailure: 3 + name: "ciliumConnectivityTests" + displayName: "Run Cilium Connectivity Tests" + + - ${{ if eq( parameters['testHubble'], true) }}: + - script: | + echo "enable Hubble metrics server" + kubectl apply -f test/integration/manifests/cilium/hubble/hubble-peer-svc.yaml + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-config/cilium-config-hubble.yaml + kubectl rollout restart ds cilium -n kube-system + echo "wait <3 minutes for pods to be ready after restart" + kubectl rollout status ds cilium -n kube-system --timeout=3m + kubectl get pods -Aowide + echo "verify Hubble metrics endpoint is usable" + go test ./test/integration/networkobservability -v -tags=networkobservability + retryCountOnTaskFailure: 3 + name: "HubbleConnectivityTests" + displayName: "Run Hubble Connectivity Tests" + + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl get pod -owide -A + # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. + # Saves 17 minutes + kubectl delete deploy -n cilium-test echo-external-node + if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then + echo "Check cilium identities in cilium-test namepsace during nightly run" + echo "expect the identities to be deleted when the namespace is deleted" + kubectl get ciliumidentity | grep cilium-test + fi + make test-validate-state + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns cilium-test + kubectl get pod -owide -A + make test-validate-state + name: "validatePods" + displayName: "Validate Pods" + + - script: | + if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then + kubectl get pod -owide -n cilium-test + echo "wait for pod and cilium identity deletion in cilium-test namespace" + ns="cilium-test" + while true; do + pods=$(kubectl get pods -n $ns --no-headers=true 2>/dev/null) + if [[ -z "$pods" ]]; then + echo "No pods found" + break + fi + sleep 2s + done + sleep 20s + echo "Verify cilium identities are deleted from cilium-test" + checkIdentity="$(kubectl get ciliumidentity -o json | grep cilium-test | jq -e 'length == 0')" + if [[ -n $checkIdentity ]]; then + echo "##[error]Cilium Identities still present in cilium-test namespace" + else + printf -- "Identities deleted from cilium-test namespace\n" + fi + else + echo "skip cilium identities check for PR pipeline" + fi + name: "CiliumIdentities" + displayName: "Verify Cilium Identities Deletion" + + - script: | + echo "validate pod IP assignment before CNS restart" + kubectl get pod -owide -A + make test-validate-state + echo "restart CNS" + kubectl rollout restart ds azure-cns -n kube-system + kubectl rollout status ds azure-cns -n kube-system + kubectl get pod -owide -A + echo "validate pods after CNS restart" + make test-validate-state + name: "restartCNS" + displayName: "Restart CNS and validate pods" + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" + + - script: | + ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ + echo $ARTIFACT_DIR + sudo rm -rf $ARTIFACT_DIR + sudo rm -rf test/integration/logs + name: "Cleanupartifactdir" + displayName: "Cleanup artifact dir" + condition: always() diff --git a/test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl b/test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl index a710c233602..964461d0a92 100644 --- a/test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl +++ b/test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl @@ -66,7 +66,7 @@ spec: fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent livenessProbe: failureThreshold: 10 @@ -163,7 +163,7 @@ spec: hostNetwork: true initContainers: - name: install-cni-binaries - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent command: - "/install-plugin.sh" @@ -192,7 +192,7 @@ spec: value: /run/cilium/cgroupv2 - name: BIN_PATH value: /opt/cni/bin - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent name: mount-cgroup resources: {} @@ -224,7 +224,7 @@ spec: env: - name: BIN_PATH value: /opt/cni/bin - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent name: apply-sysctl-overwrites resources: {} @@ -252,7 +252,7 @@ spec: - /bin/bash - -c - -- - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent name: mount-bpf-fs resources: {} @@ -279,7 +279,7 @@ spec: key: clean-cilium-bpf-state name: cilium-config optional: true - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent name: clean-cilium-state resources: @@ -338,7 +338,7 @@ spec: name: host-usr-lib readOnly: true - name: block-wireserver - image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/cilium:1.14.4 imagePullPolicy: IfNotPresent command: - /bin/bash diff --git a/test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl b/test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl index 2842221eee1..f6130163720 100644 --- a/test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl +++ b/test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl @@ -29,7 +29,7 @@ spec: spec: containers: - name: cilium-operator - image: $CILIUM_IMAGE_REGISTRY/cilium/operator-generic:$CILIUM_VERSION_TAG + image: $CILIUM_IMAGE_REGISTRY/cilium/operator-generic:1.14.4 imagePullPolicy: IfNotPresent command: - cilium-operator-generic From b908af591b811cfcd513c8d4fd0ba9a80e6e1260 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Thu, 25 Jan 2024 11:49:00 -0800 Subject: [PATCH 10/31] ci: E2E Framework [Azure steps] [4/6] (#2528) e2e azure commands --- go.mod | 5 + go.sum | 15 ++ .../framework/azure/create-cilium-cluster.go | 213 ++++++++++++++++++ test/e2e/framework/azure/create-cluster.go | 115 ++++++++++ test/e2e/framework/azure/create-rg.go | 48 ++++ test/e2e/framework/azure/create-vnet.go | 110 +++++++++ test/e2e/framework/azure/delete-cluster.go | 48 ++++ test/e2e/framework/azure/delete-rg.go | 52 +++++ test/e2e/framework/azure/enable-ama.go | 117 ++++++++++ test/e2e/framework/azure/get-kubeconfig.go | 53 +++++ 10 files changed, 776 insertions(+) create mode 100644 test/e2e/framework/azure/create-cilium-cluster.go create mode 100644 test/e2e/framework/azure/create-cluster.go create mode 100644 test/e2e/framework/azure/create-rg.go create mode 100644 test/e2e/framework/azure/create-vnet.go create mode 100644 test/e2e/framework/azure/delete-cluster.go create mode 100644 test/e2e/framework/azure/delete-rg.go create mode 100644 test/e2e/framework/azure/enable-ama.go create mode 100644 test/e2e/framework/azure/get-kubeconfig.go diff --git a/go.mod b/go.mod index 52d4d339df8..5f93e41cf49 100644 --- a/go.mod +++ b/go.mod @@ -130,6 +130,11 @@ require ( ) require ( + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 golang.org/x/sync v0.6.0 gotest.tools/v3 v3.5.1 k8s.io/kubectl v0.28.5 diff --git a/go.sum b/go.sum index ecd557e43e9..03025387cfe 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,21 @@ github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1 h1:EHlWMHGZaYhRHWNbawWxEzlZ2Eh8aTXNs8t7iAibq+I= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4 v4.7.0-beta.1/go.mod h1:noQIdW75SiQFB3mSFJBr4iRRH83S9skaFiBv4C0uEs0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0 h1:MRPU8Bge2f9tkfG3PCr4vEnqXl8XOSjlhuK3l+8Hvkc= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard v1.2.0/go.mod h1:xYrOYxajQvXMlp6M1E3amlaqPDXspyJxmjqTsGo6Jmw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0 h1:PTFGRSlMKCQelWwxUyYVEUqseBJVemLyqWJjvMyt0do= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v2 v2.0.0/go.mod h1:LRr2FzBTQlONPPa5HREE5+RjSCTXl7BwOvYOaWTqCaI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0 h1:Ds0KRF8ggpEGg4Vo42oX1cIt/IfOhHWJBikksZbVxeg= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor v0.11.0/go.mod h1:jj6P8ybImR+5topJ+eH6fgcemSFBmU6/6bFF8KkwuDI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0 h1:9CrwzqQ+e8EqD+A2bh547GjBU4K0o30FhiTB981LFNI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0/go.mod h1:Wfx7a5UHfOLG6O4NZ7Q0BPZUYwvlNCBR/OlIBpP3dlA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go new file mode 100644 index 00000000000..671a25945bb --- /dev/null +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -0,0 +1,213 @@ +package azure + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/manifests" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/kubectl/pkg/scheme" +) + +var ( + ErrResourceNameTooLong = fmt.Errorf("resource name too long") + ErrEmptyFile = fmt.Errorf("empty file") +) + +type CreateBYOCiliumCluster struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string + VnetName string + SubnetName string + PodCidr string + DNSServiceIP string + ServiceCidr string +} + +func printjson(data interface{}) { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(data); err != nil { + panic(err) + } +} + +func (c *CreateBYOCiliumCluster) Prevalidate() error { + for _, dir := range manifests.CiliumV14Directories { + files, err := manifests.CiliumManifests.ReadDir(dir) + if err != nil { + return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err) + } + + for _, file := range files { + b, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name())) + if err != nil { + return fmt.Errorf("error reading manifest file %s from embed: %w", file.Name(), err) + } + if len(b) == 0 { + return fmt.Errorf("manifest file %s from embed is empty: %w", file.Name(), ErrEmptyFile) + } + + } + } + + if len(c.ResourceGroupName) > 80 { //nolint:gomnd // 80 is the max length for resource group names + return fmt.Errorf("resource group name for nodes cannot exceed 80 characters: %w", ErrResourceNameTooLong) + } + + return nil +} + +func (c *CreateBYOCiliumCluster) Postvalidate() error { + return nil +} + +func (c *CreateBYOCiliumCluster) Run() error { + // Start with default cluster template + ciliumCluster := GetStarterClusterTemplate(c.Location) + ciliumCluster.Properties.NetworkProfile.NetworkPlugin = to.Ptr(armcontainerservice.NetworkPluginNone) + ciliumCluster.Properties.NetworkProfile.NetworkPluginMode = to.Ptr(armcontainerservice.NetworkPluginModeOverlay) + ciliumCluster.Properties.NetworkProfile.PodCidr = to.Ptr(c.PodCidr) + ciliumCluster.Properties.NetworkProfile.DNSServiceIP = to.Ptr(c.DNSServiceIP) + ciliumCluster.Properties.NetworkProfile.ServiceCidr = to.Ptr(c.ServiceCidr) + subnetkey := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s/subnets/%s", c.SubscriptionID, c.ResourceGroupName, c.VnetName, c.SubnetName) + ciliumCluster.Properties.AgentPoolProfiles[0].VnetSubnetID = to.Ptr(subnetkey) + + // Set the kubeproxy config + kubeProxyConfig := armcontainerservice.NetworkProfileKubeProxyConfig{ + Mode: to.Ptr(armcontainerservice.ModeIPVS), + Enabled: to.Ptr(false), + IpvsConfig: to.Ptr(armcontainerservice.NetworkProfileKubeProxyConfigIpvsConfig{ + Scheduler: to.Ptr(armcontainerservice.IpvsSchedulerLeastConnection), + TCPTimeoutSeconds: to.Ptr(int32(900)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + TCPFinTimeoutSeconds: to.Ptr(int32(120)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + UDPTimeoutSeconds: to.Ptr(int32(300)), //nolint:gomnd // set by existing kube-proxy in hack/aks/kube-proxy.json + }), + } + + log.Printf("using kube-proxy config:\n") + printjson(kubeProxyConfig) + ciliumCluster.Properties.NetworkProfile.KubeProxyConfig = to.Ptr(kubeProxyConfig) + + // Deploy cluster + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create az client: %w", err) + } + + log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + + poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil) + if err != nil { + return fmt.Errorf("failed to finish the create cluster request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to create cluster: %w", err) + } + + // get kubeconfig + log.Printf("getting kubeconfig for cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + clientset, err := c.getKubeConfig() + if err != nil { + return fmt.Errorf("failed to get kubeconfig for cluster \"%s\": %w", c.ClusterName, err) + } + + // Deploy the cilium components once the cluster is created + log.Printf("deploying cilium components to cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) + err = c.deployCiliumComponents(clientset) + if err != nil { + return fmt.Errorf("failed to deploy cilium components: %w", err) + } + + // wait for cilium pods to be ready + err = k8s.WaitForPodReady(ctx, clientset, "kube-system", "k8s-app=cilium") + if err != nil { + return fmt.Errorf("failed to wait for cilium pods to be ready: %w", err) + } + + return nil +} + +func (c *CreateBYOCiliumCluster) getKubeConfig() (*kubernetes.Clientset, error) { + // create temporary directory for kubeconfig, as we need access to deploy cilium things + dir, err := os.MkdirTemp("", "cilium-e2e") + if err != nil { + return nil, fmt.Errorf("failed to create temporary directory to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + // reuse getKubeConfig job + kubeconfigpath := dir + "/kubeconfig" + getKubeconfigJob := GetAKSKubeConfig{ + ClusterName: c.ClusterName, + SubscriptionID: c.SubscriptionID, + ResourceGroupName: c.ResourceGroupName, + Location: c.Location, + KubeConfigFilePath: kubeconfigpath, + } + + err = getKubeconfigJob.Run() + if err != nil { + return nil, fmt.Errorf("failed to get kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + + config, err := clientcmd.BuildConfigFromFlags("", kubeconfigpath) + if err != nil { + return nil, fmt.Errorf("failed to build kubeconfig to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, fmt.Errorf("failed to create Kubernetes client to deploy cilium components on cluster \"%s\": %w", c.ClusterName, err) + } + return clientset, nil +} + +func (c *CreateBYOCiliumCluster) deployCiliumComponents(clientset *kubernetes.Clientset) error { + // traverse the predefined Cilium component folders + for _, dir := range manifests.CiliumV14Directories { + + files, err := manifests.CiliumManifests.ReadDir(dir) + if err != nil { + return fmt.Errorf("error reading manifest directory %s from embed: %w", dir, err) + } + + for _, file := range files { + yamlFile, err := manifests.CiliumManifests.ReadFile(fmt.Sprintf("%s/%s", dir, file.Name())) + if err != nil { + return fmt.Errorf("error reading YAML file: %w", err) + } + + // Decode the YAML file into a Kubernetes object + decode := scheme.Codecs.UniversalDeserializer().Decode + obj, _, err := decode([]byte(yamlFile), nil, nil) + if err != nil { + return fmt.Errorf("error decoding YAML file: %w", err) + } + + // create the resource + err = k8s.CreateResource(context.Background(), obj, clientset) + if err != nil { + return fmt.Errorf("error creating resource: %w", err) + } + } + } + + return nil +} diff --git a/test/e2e/framework/azure/create-cluster.go b/test/e2e/framework/azure/create-cluster.go new file mode 100644 index 00000000000..54250bbabc6 --- /dev/null +++ b/test/e2e/framework/azure/create-cluster.go @@ -0,0 +1,115 @@ +package azure + +import ( + "context" + "fmt" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +const ( + MaxNumberOfNodes = 3 + MaxPodsPerNode = 250 + AgentSKU = "Standard_D4s_v3" +) + +var defaultClusterCreateTimeout = 30 * time.Minute + +type CreateCluster struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string +} + +func (c *CreateCluster) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, GetStarterClusterTemplate(c.Location), nil) + if err != nil { + return fmt.Errorf("failed to finish the create cluster request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the create cluster result: %w", err) + } + + return nil +} + +func GetStarterClusterTemplate(location string) armcontainerservice.ManagedCluster { + id := armcontainerservice.ResourceIdentityTypeSystemAssigned + return armcontainerservice.ManagedCluster{ + Location: to.Ptr(location), + Tags: map[string]*string{ + "archv2": to.Ptr(""), + "tier": to.Ptr("production"), + }, + Properties: &armcontainerservice.ManagedClusterProperties{ + AddonProfiles: map[string]*armcontainerservice.ManagedClusterAddonProfile{}, + /* Moving this to a separate stage to enable AMA since it takes some time to provision + AzureMonitorProfile: &armcontainerservice.ManagedClusterAzureMonitorProfile{ + Metrics: &armcontainerservice.ManagedClusterAzureMonitorProfileMetrics{ + Enabled: to.Ptr(true), + }, + }, + */ + AgentPoolProfiles: []*armcontainerservice.ManagedClusterAgentPoolProfile{ + { + Type: to.Ptr(armcontainerservice.AgentPoolTypeVirtualMachineScaleSets), + AvailabilityZones: []*string{to.Ptr("1")}, + Count: to.Ptr[int32](MaxNumberOfNodes), + EnableNodePublicIP: to.Ptr(false), + Mode: to.Ptr(armcontainerservice.AgentPoolModeSystem), + OSType: to.Ptr(armcontainerservice.OSTypeLinux), + ScaleDownMode: to.Ptr(armcontainerservice.ScaleDownModeDelete), + VMSize: to.Ptr(AgentSKU), + Name: to.Ptr("nodepool1"), + MaxPods: to.Ptr(int32(MaxPodsPerNode)), + }, + }, + KubernetesVersion: to.Ptr(""), + DNSPrefix: to.Ptr("dnsprefix1"), + EnablePodSecurityPolicy: to.Ptr(false), + EnableRBAC: to.Ptr(true), + LinuxProfile: nil, + NetworkProfile: &armcontainerservice.NetworkProfile{ + LoadBalancerSKU: to.Ptr(armcontainerservice.LoadBalancerSKUStandard), + OutboundType: to.Ptr(armcontainerservice.OutboundTypeLoadBalancer), + NetworkPlugin: to.Ptr(armcontainerservice.NetworkPluginAzure), + }, + WindowsProfile: &armcontainerservice.ManagedClusterWindowsProfile{ + AdminPassword: to.Ptr("replacePassword1234$"), + AdminUsername: to.Ptr("azureuser"), + }, + }, + Identity: &armcontainerservice.ManagedClusterIdentity{ + Type: &id, + }, + + SKU: &armcontainerservice.ManagedClusterSKU{ + Name: to.Ptr(armcontainerservice.ManagedClusterSKUName("Base")), + Tier: to.Ptr(armcontainerservice.ManagedClusterSKUTierStandard), + }, + } +} + +func (c *CreateCluster) Prevalidate() error { + return nil +} + +func (c *CreateCluster) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/create-rg.go b/test/e2e/framework/azure/create-rg.go new file mode 100644 index 00000000000..efa1d88ea55 --- /dev/null +++ b/test/e2e/framework/azure/create-rg.go @@ -0,0 +1,48 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +type CreateResourceGroup struct { + SubscriptionID string + ResourceGroupName string + Location string +} + +func (c *CreateResourceGroup) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armresources.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create resource group client: %w", err) + } + log.Printf("creating resource group %s in location %s...", c.ResourceGroupName, c.Location) + + _, err = clientFactory.NewResourceGroupsClient().CreateOrUpdate(ctx, c.ResourceGroupName, armresources.ResourceGroup{ + Location: to.Ptr(c.Location), + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request: %w", err) + } + + log.Printf("resource group created %s in location %s", c.ResourceGroupName, c.Location) + return nil +} + +func (c *CreateResourceGroup) Prevalidate() error { + return nil +} + +func (c *CreateResourceGroup) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/create-vnet.go b/test/e2e/framework/azure/create-vnet.go new file mode 100644 index 00000000000..0149a14c8d0 --- /dev/null +++ b/test/e2e/framework/azure/create-vnet.go @@ -0,0 +1,110 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5" +) + +const FlowTimeoutInMinutes = 10 + +type CreateVNet struct { + SubscriptionID string + ResourceGroupName string + Location string + VnetName string + VnetAddressSpace string +} + +func (c *CreateVNet) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armnetwork.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("creating vnet \"%s\" in resource group \"%s\"...", c.VnetName, c.ResourceGroupName) + + poller, err := clientFactory.NewVirtualNetworksClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.VnetName, armnetwork.VirtualNetwork{ + Location: to.Ptr(c.Location), + Properties: &armnetwork.VirtualNetworkPropertiesFormat{ + AddressSpace: &armnetwork.AddressSpace{ + AddressPrefixes: []*string{ + to.Ptr(c.VnetAddressSpace), + }, + }, + FlowTimeoutInMinutes: to.Ptr[int32](FlowTimeoutInMinutes), + }, + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request for create vnet: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for create vnet: %w", err) + } + return nil +} + +func (c *CreateVNet) Prevalidate() error { + return nil +} + +func (c *CreateVNet) Postvalidate() error { + return nil +} + +type CreateSubnet struct { + SubscriptionID string + ResourceGroupName string + Location string + VnetName string + SubnetName string + SubnetAddressSpace string +} + +func (c *CreateSubnet) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armnetwork.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("creating subnet \"%s\" in vnet \"%s\" in resource group \"%s\"...", c.SubnetName, c.VnetName, c.ResourceGroupName) + + poller, err := clientFactory.NewSubnetsClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.VnetName, c.SubnetName, armnetwork.Subnet{ + Properties: &armnetwork.SubnetPropertiesFormat{ + AddressPrefix: to.Ptr(c.SubnetAddressSpace), + }, + }, nil) + if err != nil { + return fmt.Errorf("failed to finish the request for create subnet: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for create subnet: %w", err) + } + return nil +} + +func (c *CreateSubnet) Prevalidate() error { + return nil +} + +func (c *CreateSubnet) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-cluster.go b/test/e2e/framework/azure/delete-cluster.go new file mode 100644 index 00000000000..674a2fbb620 --- /dev/null +++ b/test/e2e/framework/azure/delete-cluster.go @@ -0,0 +1,48 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +type DeleteCluster struct { + ClusterName string + SubscriptionID string + ResourceGroupName string + Location string +} + +func (d *DeleteCluster) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armcontainerservice.NewClientFactory(d.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + log.Printf("deleting cluster %s in resource group %s...", d.ClusterName, d.ResourceGroupName) + poller, err := clientFactory.NewManagedClustersClient().BeginDelete(ctx, d.ResourceGroupName, d.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to finish the request: %w", err) + } + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result: %w", err) + } + return nil +} + +func (d *DeleteCluster) Prevalidate() error { + return nil +} + +func (d *DeleteCluster) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/delete-rg.go b/test/e2e/framework/azure/delete-rg.go new file mode 100644 index 00000000000..0d65df6d5fe --- /dev/null +++ b/test/e2e/framework/azure/delete-rg.go @@ -0,0 +1,52 @@ +package azure + +import ( + "context" + "fmt" + "log" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" +) + +type DeleteResourceGroup struct { + SubscriptionID string + ResourceGroupName string + Location string +} + +func (d *DeleteResourceGroup) Run() error { + log.Printf("deleting resource group \"%s\"...", d.ResourceGroupName) + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armresources.NewClientFactory(d.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create resource group client: %w", err) + } + + forceDeleteType := "Microsoft.Compute/virtualMachines,Microsoft.Compute/virtualMachineScaleSets" + poller, err := clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) + if err != nil { + return fmt.Errorf("failed to finish the delete resource group request: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to pull the result for delete resource group: %w", err) + } + + log.Printf("resource group \"%s\" deleted successfully", d.ResourceGroupName) + return nil +} + +func (d *DeleteResourceGroup) Prevalidate() error { + return nil +} + +func (d *DeleteResourceGroup) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/enable-ama.go b/test/e2e/framework/azure/enable-ama.go new file mode 100644 index 00000000000..e6bdaf4e618 --- /dev/null +++ b/test/e2e/framework/azure/enable-ama.go @@ -0,0 +1,117 @@ +package azure + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/dashboard/armdashboard" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/monitor/armmonitor" +) + +const fileperms = 0o600 + +type CreateAzureMonitor struct { + SubscriptionID string + ResourceGroupName string + Location string + ClusterName string +} + +func (c *CreateAzureMonitor) Run() error { + log.Printf(`this will deploy azure monitor workspace and grafana, but as of 1/9/2024, the api docs don't show how to do +az aks update --enable-azure-monitor-metrics \ +-n $NAME \ +-g $CLUSTER_RESOURCE_GROUP \ +--azure-monitor-workspace-resource-id $AZMON_RESOURCE_ID \ +--grafana-resource-id $GRAFANA_RESOURCE_ID +`) + + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + + ctx := context.Background() + amaClientFactory, err := armmonitor.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create azure monitor workspace client: %w", err) + } + log.Printf("creating resource group %s in location %s...", c.ResourceGroupName, c.Location) + + // create azure monitor + _, err = amaClientFactory.NewAzureMonitorWorkspacesClient().Create(ctx, c.ResourceGroupName, "test", armmonitor.AzureMonitorWorkspaceResource{ + Location: &c.Location, + }, &armmonitor.AzureMonitorWorkspacesClientCreateOptions{}) + if err != nil { + return fmt.Errorf("failed to azure monitor workspace: %w", err) + } + + // Create grafana + + granafaClientFactory, err := armdashboard.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create grafana client: %w", err) + } + + _, err = granafaClientFactory.NewGrafanaClient().BeginCreate(ctx, c.ResourceGroupName, "test", armdashboard.ManagedGrafana{}, &armdashboard.GrafanaClientBeginCreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create grafana: %w", err) + } + + log.Printf("azure monitor workspace %s in location %s", c.ResourceGroupName, c.Location) + + // update aks cluster + + ctx, cancel := context.WithTimeout(context.Background(), defaultClusterCreateTimeout) + defer cancel() + aksClientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + + cluster, err := aksClientFactory.NewManagedClustersClient().Get(ctx, c.ResourceGroupName, c.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to get cluster to enable AMA: %w", err) + } + + // enable Azure Monitor Metrics + cluster.Properties.AzureMonitorProfile.Metrics.Enabled = to.Ptr(true) + + // Marshal the struct into a JSON byte array with indentation + jsonData, err := json.MarshalIndent(cluster, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal cluster to JSON for AMA: %w", err) + } + + // Write the JSON data to a file + err = os.WriteFile("cluster.json", jsonData, fileperms) + if err != nil { + return fmt.Errorf("failed to write cluster JSON to file for AMA: %w", err) + } + + poller, err := aksClientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, GetStarterClusterTemplate(c.Location), nil) + if err != nil { + return fmt.Errorf("failed to finish the update cluster request for AMA: %w", err) + } + + _, err = poller.PollUntilDone(ctx, nil) + if err != nil { + return fmt.Errorf("failed to enable AMA on cluster %s: %w", *cluster.Name, err) + } + + return nil +} + +func (c *CreateAzureMonitor) Prevalidate() error { + return nil +} + +func (c *CreateAzureMonitor) Postvalidate() error { + return nil +} diff --git a/test/e2e/framework/azure/get-kubeconfig.go b/test/e2e/framework/azure/get-kubeconfig.go new file mode 100644 index 00000000000..08cb8414036 --- /dev/null +++ b/test/e2e/framework/azure/get-kubeconfig.go @@ -0,0 +1,53 @@ +package azure + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v4" +) + +const KubeConfigPerms = 0o600 + +type GetAKSKubeConfig struct { + ClusterName string + SubscriptionID string + ResourceGroupName string + Location string + KubeConfigFilePath string +} + +func (c *GetAKSKubeConfig) Run() error { + cred, err := azidentity.NewAzureCLICredential(nil) + if err != nil { + return fmt.Errorf("failed to obtain a credential: %w", err) + } + ctx := context.Background() + clientFactory, err := armcontainerservice.NewClientFactory(c.SubscriptionID, cred, nil) + if err != nil { + return fmt.Errorf("failed to create client: %w", err) + } + res, err := clientFactory.NewManagedClustersClient().ListClusterUserCredentials(ctx, c.ResourceGroupName, c.ClusterName, nil) + if err != nil { + return fmt.Errorf("failed to finish the get managed cluster client request: %w", err) + } + + err = os.WriteFile(c.KubeConfigFilePath, []byte(res.Kubeconfigs[0].Value), KubeConfigPerms) + if err != nil { + return fmt.Errorf("failed to write kubeconfig to file \"%s\": %w", c.KubeConfigFilePath, err) + } + + log.Printf("kubeconfig for cluster \"%s\" in resource group \"%s\" written to \"%s\"\n", c.ClusterName, c.ResourceGroupName, c.KubeConfigFilePath) + return nil +} + +func (c *GetAKSKubeConfig) Prevalidate() error { + return nil +} + +func (c *GetAKSKubeConfig) Postvalidate() error { + return nil +} From 1fc2ba07ed42b0b968bbc9db7833386add78fd96 Mon Sep 17 00:00:00 2001 From: Matthew Long <61910737+thatmattlong@users.noreply.github.com> Date: Thu, 25 Jan 2024 11:50:31 -0800 Subject: [PATCH 11/31] feat: create or update nodeinfo crd at cns startup for multitenancy (#2545) * feat: create or update nodeinfo crd at cns startup for multitenancy * fix: spelling on variable name nodeInfoErr --- cns/service/main.go | 47 ++++++++++++- crd/multitenancy/client.go | 26 ++++++++ crd/multitenancy/client_test.go | 115 ++++++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 crd/multitenancy/client_test.go diff --git a/cns/service/main.go b/cns/service/main.go index 94a8cfd878f..fec9a25194f 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -31,6 +31,7 @@ import ( "github.com/Azure/azure-container-networking/cns/fsnotify" "github.com/Azure/azure-container-networking/cns/healthserver" "github.com/Azure/azure-container-networking/cns/hnsclient" + "github.com/Azure/azure-container-networking/cns/imds" "github.com/Azure/azure-container-networking/cns/ipampool" cssctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/clustersubnetstate" mtpncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/multitenantpodnetworkconfig" @@ -46,6 +47,7 @@ import ( acn "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/crd" cssv1alpha1 "github.com/Azure/azure-container-networking/crd/clustersubnetstate/api/v1alpha1" + "github.com/Azure/azure-container-networking/crd/multitenancy" mtv1alpha1 "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig" "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" @@ -68,9 +70,11 @@ import ( kuberuntime "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/cache" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "sigs.k8s.io/controller-runtime/pkg/healthz" ctrlzap "sigs.k8s.io/controller-runtime/pkg/log/zap" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" @@ -1215,8 +1219,9 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if _, ok := node.Labels[configuration.LabelNodeSwiftV2]; ok { cnsconfig.EnableSwiftV2 = true cnsconfig.WatchPods = true - // TODO(rbtr): create the NodeInfo for Swift V2 - // register the noop mtpnc reconciler to populate the cache + if nodeInfoErr := createOrUpdateNodeInfoCRD(ctx, kubeConfig, node); nodeInfoErr != nil { + return errors.Wrap(nodeInfoErr, "error creating or updating nodeinfo crd") + } } var podInfoByIPProvider cns.PodInfoByIPProvider @@ -1458,3 +1463,41 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn logger.Printf("Initialized SyncHostNCVersion loop.") return nil } + +// createOrUpdateNodeInfoCRD polls imds to learn the VM Unique ID and then creates or updates the NodeInfo CRD +// with that vm unique ID +func createOrUpdateNodeInfoCRD(ctx context.Context, restConfig *rest.Config, node *corev1.Node) error { + imdsCli := imds.NewClient() + vmUniqueID, err := imdsCli.GetVMUniqueID(ctx) + if err != nil { + return errors.Wrap(err, "error getting vm unique ID from imds") + } + + directcli, err := client.New(restConfig, client.Options{Scheme: multitenancy.Scheme}) + if err != nil { + return errors.Wrap(err, "failed to create ctrl client") + } + + nodeInfoCli := multitenancy.NodeInfoClient{ + Cli: directcli, + } + + nodeInfo := &mtv1alpha1.NodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: node.Name, + }, + Spec: mtv1alpha1.NodeInfoSpec{ + VMUniqueID: vmUniqueID, + }, + } + + if err := controllerutil.SetOwnerReference(node, nodeInfo, multitenancy.Scheme); err != nil { + return errors.Wrap(err, "failed to set nodeinfo owner reference to node") + } + + if err := nodeInfoCli.CreateOrUpdate(ctx, nodeInfo, "azure-cns"); err != nil { + return errors.Wrap(err, "error ensuring nodeinfo CRD exists and is up-to-date") + } + + return nil +} diff --git a/crd/multitenancy/client.go b/crd/multitenancy/client.go index 797553b7de1..bfd7a0061e2 100644 --- a/crd/multitenancy/client.go +++ b/crd/multitenancy/client.go @@ -14,6 +14,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" ) // Scheme is a runtime scheme containing the client-go scheme and the MTPNC/NI scheme. @@ -190,3 +191,28 @@ func (i *Installer) InstallOrUpdatePodNetworkInstance(ctx context.Context) (*v1. } return current, nil } + +type NodeInfoClient struct { + Cli client.Client +} + +func (n *NodeInfoClient) CreateOrUpdate(ctx context.Context, nodeInfo *v1alpha1.NodeInfo, fieldOwner string) error { + if err := n.Cli.Create(ctx, nodeInfo); err != nil { + if !apierrors.IsAlreadyExists(err) { + return errors.Wrap(err, "error creating nodeinfo crd") + } + if err := n.Cli.Patch(ctx, &v1alpha1.NodeInfo{ + TypeMeta: metav1.TypeMeta{ + APIVersion: v1alpha1.GroupVersion.String(), + Kind: "NodeInfo", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: nodeInfo.Name, + }, + Spec: nodeInfo.Spec, + }, client.Apply, client.ForceOwnership, client.FieldOwner(fieldOwner)); err != nil { + return errors.Wrap(err, "error patching nodeinfo crd") + } + } + return nil +} diff --git a/crd/multitenancy/client_test.go b/crd/multitenancy/client_test.go new file mode 100644 index 00000000000..cf50ed4ed49 --- /dev/null +++ b/crd/multitenancy/client_test.go @@ -0,0 +1,115 @@ +package multitenancy_test + +import ( + "context" + "errors" + "testing" + + "github.com/Azure/azure-container-networking/crd/multitenancy" + "github.com/Azure/azure-container-networking/crd/multitenancy/api/v1alpha1" + "github.com/stretchr/testify/require" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type mockClient struct { + client.Client + createFunc func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error + patchFunc func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error +} + +func (m *mockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return m.createFunc(ctx, obj, opts...) +} + +func (m *mockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return m.patchFunc(ctx, obj, patch, opts...) +} + +func TestCreateNodeInfo(t *testing.T) { + cli := multitenancy.NodeInfoClient{ + Cli: &mockClient{ + createFunc: func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return nil + }, + }, + } + + err := cli.CreateOrUpdate(context.Background(), &v1alpha1.NodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-node", + }, + }, "field-owner") + + require.NoError(t, err, "unexpected error creating nodeinfo crd") +} + +func TestUpdateNodeInfo(t *testing.T) { + cli := multitenancy.NodeInfoClient{ + Cli: &mockClient{ + createFunc: func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return apierrors.NewAlreadyExists(schema.GroupResource{}, obj.GetName()) + }, + patchFunc: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return nil + }, + }, + } + + err := cli.CreateOrUpdate(context.Background(), &v1alpha1.NodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-node", + }, + }, "field-owner") + + require.NoError(t, err, "unexpected error creating nodeinfo crd") +} + +func TestCreateNodeInfoInternalServerError(t *testing.T) { + someInternalError := errors.New("some internal error") //nolint:goerr113 // dynamic error is fine here + cli := multitenancy.NodeInfoClient{ + Cli: &mockClient{ + createFunc: func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return apierrors.NewInternalError(someInternalError) + }, + }, + } + + err := cli.CreateOrUpdate(context.Background(), &v1alpha1.NodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-node", + }, + }, "field-owner") + + require.Error(t, err, "expected error") + // NewInternalError doesn't wrap the error, so assert that the final error string at least preserves + // the original error message + require.Contains(t, err.Error(), someInternalError.Error()) +} + +func TestPatchNodeInfoInternalServerError(t *testing.T) { + someInternalError := errors.New("some internal error") //nolint:goerr113 // dynamic error is fine here + cli := multitenancy.NodeInfoClient{ + Cli: &mockClient{ + createFunc: func(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + return apierrors.NewAlreadyExists(schema.GroupResource{}, obj.GetName()) + }, + patchFunc: func(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + return apierrors.NewInternalError(someInternalError) + }, + }, + } + + err := cli.CreateOrUpdate(context.Background(), &v1alpha1.NodeInfo{ + ObjectMeta: metav1.ObjectMeta{ + Name: "some-node", + }, + }, "field-owner") + + require.Error(t, err, "expected error") + // NewInternalError doesn't wrap the error, so assert that the final error string at least preserves + // the original error message + require.Contains(t, err.Error(), someInternalError.Error()) +} From 8acc55148cb0eb41b9a6b6271ee82413693b9105 Mon Sep 17 00:00:00 2001 From: aggarwal0009 <127549148+aggarwal0009@users.noreply.github.com> Date: Thu, 25 Jan 2024 12:16:15 -0800 Subject: [PATCH 12/31] Add new status to PN CRD (#2535) * Add new status to PN crd * Update PN CRD status * Update PN CRD staus * updating generated manifest for MT podnetwork --------- Co-authored-by: Miguel Gonzalez --- crd/multitenancy/api/v1alpha1/podnetwork.go | 9 +++++---- .../multitenancy.acn.azure.com_podnetworks.yaml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/crd/multitenancy/api/v1alpha1/podnetwork.go b/crd/multitenancy/api/v1alpha1/podnetwork.go index a426a16bd0e..8bb87d0f988 100644 --- a/crd/multitenancy/api/v1alpha1/podnetwork.go +++ b/crd/multitenancy/api/v1alpha1/podnetwork.go @@ -48,13 +48,14 @@ type PodNetworkSpec struct { } // Status indicates the status of PN -// +kubebuilder:validation:Enum=Ready;InUse;SubnetNotDelegated +// +kubebuilder:validation:Enum=Ready;InUse;SubnetNotDelegated;SubnetDelegatedToDifferentService type Status string const ( - Ready Status = "Ready" - InUse Status = "InUse" - SubnetNotDelegated Status = "SubnetNotDelegated" + Ready Status = "Ready" + InUse Status = "InUse" + SubnetNotDelegated Status = "SubnetNotDelegated" + SubnetDelegatedToDifferentService Status = "SubnetDelegatedToDifferentService" ) // PodNetworkStatus defines the observed state of PodNetwork diff --git a/crd/multitenancy/manifests/multitenancy.acn.azure.com_podnetworks.yaml b/crd/multitenancy/manifests/multitenancy.acn.azure.com_podnetworks.yaml index 5c213cecd02..cb73f5b94ac 100644 --- a/crd/multitenancy/manifests/multitenancy.acn.azure.com_podnetworks.yaml +++ b/crd/multitenancy/manifests/multitenancy.acn.azure.com_podnetworks.yaml @@ -80,6 +80,7 @@ spec: - Ready - InUse - SubnetNotDelegated + - SubnetDelegatedToDifferentService type: string type: object type: object From faa2b736341e0ff0366ddfae5f9608cafca29a9e Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Thu, 25 Jan 2024 15:36:13 -0600 Subject: [PATCH 13/31] ci: Improve Validate package (#2485) * ci: improve validate package * ci: add v6 hns validate * chore: address comments Signed-off-by: John Payne <89417863+jpayne3506@users.noreply.github.com> * ci: capture hns json object * chore: lint fix * ci: kubeproxy restart in CreateValidator * fix: validateRestartNetwork for hybrid clusters * ci: cleanup RestartKubeProxyService changes * chore: lint fix * chore: address comments --------- Signed-off-by: John Payne <89417863+jpayne3506@users.noreply.github.com> --- .../datapath/datapath_windows_test.go | 13 ++- test/internal/kubernetes/utils.go | 47 ++++++++++- test/validate/linux_validate.go | 70 ++++++++++------ test/validate/utils.go | 8 +- test/validate/validate.go | 83 ++++++------------- test/validate/windows_validate.go | 73 ++++++++++++---- 6 files changed, 186 insertions(+), 108 deletions(-) diff --git a/test/integration/datapath/datapath_windows_test.go b/test/integration/datapath/datapath_windows_test.go index b0057be9cff..feb7cfc16bc 100644 --- a/test/integration/datapath/datapath_windows_test.go +++ b/test/integration/datapath/datapath_windows_test.go @@ -10,7 +10,6 @@ import ( "github.com/Azure/azure-container-networking/test/internal/datapath" "github.com/Azure/azure-container-networking/test/internal/kubernetes" - "github.com/Azure/azure-container-networking/test/validate" "github.com/stretchr/testify/require" apiv1 "k8s.io/api/core/v1" ) @@ -57,9 +56,15 @@ func setupWindowsEnvironment(t *testing.T) { clientset := kubernetes.MustGetClientset() if *restartKubeproxy { - validator, err := validate.CreateValidator(ctx, clientset, restConfig, *podNamespace, "cniv2", false, "windows") - require.NoError(t, err) - err = validator.RestartKubeProxyService(ctx) + privilegedDaemonSet := kubernetes.MustParseDaemonSet(kubernetes.PrivilegedDaemonSetPath) + daemonsetClient := clientset.AppsV1().DaemonSets(kubernetes.PrivilegedNamespace) + kubernetes.MustCreateDaemonset(ctx, daemonsetClient, privilegedDaemonSet) + + // Ensures that pods have been replaced if test is re-run after failure + if err := kubernetes.WaitForPodDaemonset(ctx, clientset, kubernetes.PrivilegedNamespace, privilegedDaemonSet.Name, kubernetes.PrivilegedLabelSelector); err != nil { + require.NoError(t, err) + } + err := kubernetes.RestartKubeProxyService(ctx, clientset, kubernetes.PrivilegedNamespace, kubernetes.PrivilegedLabelSelector, restConfig) require.NoError(t, err) } diff --git a/test/internal/kubernetes/utils.go b/test/internal/kubernetes/utils.go index 3a066d631db..194e5b5e609 100644 --- a/test/internal/kubernetes/utils.go +++ b/test/internal/kubernetes/utils.go @@ -34,10 +34,13 @@ const ( SubnetNameLabel = "kubernetes.azure.com/podnetwork-subnet" // RetryAttempts is the number of times to retry a test. - RetryAttempts = 90 - RetryDelay = 10 * time.Second - DeleteRetryAttempts = 12 - DeleteRetryDelay = 5 * time.Second + RetryAttempts = 90 + RetryDelay = 10 * time.Second + DeleteRetryAttempts = 12 + DeleteRetryDelay = 5 * time.Second + PrivilegedDaemonSetPath = "../manifests/load/privileged-daemonset-windows.yaml" + PrivilegedLabelSelector = "app=privileged-daemonset" + PrivilegedNamespace = "kube-system" ) var Kubeconfig = flag.String("test-kubeconfig", filepath.Join(homedir.HomeDir(), ".kube", "config"), "(optional) absolute path to the kubeconfig file") @@ -403,6 +406,9 @@ func ExecCmdOnPod(ctx context.Context, clientset *kubernetes.Clientset, namespac if err != nil { return []byte{}, errors.Wrapf(err, "error in executing command %s", cmd) } + if len(stdout.Bytes()) == 0 { + log.Printf("Warning: %v had 0 bytes returned from command - %v", podName, cmd) + } return stdout.Bytes(), nil } @@ -465,3 +471,36 @@ func MustRestartDaemonset(ctx context.Context, clientset *kubernetes.Clientset, _, err = clientset.AppsV1().DaemonSets(namespace).Update(ctx, ds, metav1.UpdateOptions{}) return errors.Wrapf(err, "failed to update ds %s", daemonsetName) } + +// Restarts kubeproxy on windows nodes from an existing privileged daemonset +func RestartKubeProxyService(ctx context.Context, clientset *kubernetes.Clientset, privilegedNamespace, privilegedLabelSelector string, config *rest.Config) error { + restartKubeProxyCmd := []string{"powershell", "Restart-service", "kubeproxy"} + + nodes, err := GetNodeList(ctx, clientset) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + node := nodes.Items[index] + if node.Status.NodeInfo.OperatingSystem != string(corev1.Windows) { + continue + } + // get the privileged pod + pod, err := GetPodsByNode(ctx, clientset, privilegedNamespace, privilegedLabelSelector, node.Name) + if err != nil { + return errors.Wrapf(err, "failed to get privileged pod on node %s", node.Name) + } + + if len(pod.Items) == 0 { + return errors.Errorf("there are no privileged pods on node - %v", node.Name) + } + privilegedPod := pod.Items[0] + // exec into the pod and restart kubeproxy + _, err = ExecCmdOnPod(ctx, clientset, privilegedNamespace, privilegedPod.Name, restartKubeProxyCmd, config) + if err != nil { + return errors.Wrapf(err, "failed to exec into privileged pod %s on node %s", privilegedPod.Name, node.Name) + } + } + return nil +} diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index a54ebed44e5..ed9c919eb27 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -1,11 +1,14 @@ package validate import ( + "context" "encoding/json" "github.com/Azure/azure-container-networking/cns" restserver "github.com/Azure/azure-container-networking/cns/restserver" + acnk8s "github.com/Azure/azure-container-networking/test/internal/kubernetes" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" ) const ( @@ -15,12 +18,12 @@ const ( ) var ( - restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"} - cnsManagedStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} - azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"} - azureVnetIpamStateCmd = []string{"bash", "-c", "cat /var/run/azure-vnet-ipam.json"} - ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} - cnsLocalCacheCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} + restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"} + cnsManagedStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} + azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"} + azureVnetIpamStateCmd = []string{"bash", "-c", "cat /var/run/azure-vnet-ipam.json"} + ciliumStateFileCmd = []string{"bash", "-c", "cilium endpoint list -o json"} + cnsCachedAssignedIPStateCmd = []string{"curl", "localhost:10090/debug/ipaddresses", "-d", "{\"IPConfigStateFilter\":[\"Assigned\"]}"} ) type stateFileIpsFunc func([]byte) (map[string]string, error) @@ -29,18 +32,18 @@ var linuxChecksMap = map[string][]check{ "cilium": { {"cns", cnsManagedStateFileIps, cnsLabelSelector, privilegedNamespace, cnsManagedStateFileCmd}, // cns configmap "ManageEndpointState": true, | Endpoints managed in CNS State File {"cilium", ciliumStateFileIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, - {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, }, "cniv1": { {"azure-vnet", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, {"azure-vnet-ipam", azureVnetIpamStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamStateCmd}, }, "cniv2": { - {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, {"azure-vnet", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, // cns configmap "ManageEndpointState": false, | Endpoints managed in CNI State File }, "dualstack": { - {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsLocalCacheCmd}, + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, {"azure dualstackoverlay", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, }, } @@ -156,21 +159,6 @@ func ciliumStateFileIps(result []byte) (map[string]string, error) { return ciliumPodIps, nil } -func cnsCacheStateFileIps(result []byte) (map[string]string, error) { - var cnsLocalCache CNSLocalCache - - err := json.Unmarshal(result, &cnsLocalCache) - if err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal cns local cache") - } - - cnsPodIps := make(map[string]string) - for index := range cnsLocalCache.IPConfigurationStatus { - cnsPodIps[cnsLocalCache.IPConfigurationStatus[index].IPAddress] = cnsLocalCache.IPConfigurationStatus[index].PodInfo.Name() - } - return cnsPodIps, nil -} - func azureVnetStateIps(result []byte) (map[string]string, error) { var azureVnetResult AzureCniState err := json.Unmarshal(result, &azureVnetResult) @@ -212,3 +200,37 @@ func azureVnetIpamStateIps(result []byte) (map[string]string, error) { } return azureVnetIpamPodIps, nil } + +// Linux only function +func (v *Validator) validateRestartNetwork(ctx context.Context) error { + nodes, err := acnk8s.GetNodeList(ctx, v.clientset) + if err != nil { + return errors.Wrapf(err, "failed to get node list") + } + + for index := range nodes.Items { + node := nodes.Items[index] + if node.Status.NodeInfo.OperatingSystem != string(corev1.Linux) { + continue + } + // get the privileged pod + pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, node.Name) + if err != nil { + return errors.Wrapf(err, "failed to get privileged pod") + } + if len(pod.Items) == 0 { + return errors.Errorf("there are no privileged pods on node - %v", node.Name) + } + privilegedPod := pod.Items[0] + // exec into the pod to get the state file + _, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privilegedPod.Name, restartNetworkCmd, v.config) + if err != nil { + return errors.Wrapf(err, "failed to exec into privileged pod %s on node %s", privilegedPod.Name, node.Name) + } + err = acnk8s.WaitForPodsRunning(ctx, v.clientset, "", "") + if err != nil { + return errors.Wrapf(err, "failed to wait for pods running") + } + } + return nil +} diff --git a/test/validate/utils.go b/test/validate/utils.go index cb3b5daa4cd..b43a9161bc5 100644 --- a/test/validate/utils.go +++ b/test/validate/utils.go @@ -11,14 +11,16 @@ import ( ) func compareIPs(expected map[string]string, actual []string) error { - if len(expected) != len(actual) { - return errors.Errorf("len of expected IPs != len of actual IPs, expected: %+v, actual: %+v", expected, actual) - } + expectedLen := len(expected) for _, ip := range actual { if _, ok := expected[ip]; !ok { return errors.Errorf("actual ip %s is unexpected, expected: %+v, actual: %+v", ip, expected, actual) } + delete(expected, ip) + } + if expectedLen != len(actual) { + return errors.Errorf("len of expected IPs != len of actual IPs, expected: %+v, actual: %+v | Remaining, potentially leaked, IP(s) on state file - %v", expectedLen, len(actual), expected) } return nil diff --git a/test/validate/validate.go b/test/validate/validate.go index d5db2815f4b..5e5dc536f8b 100644 --- a/test/validate/validate.go +++ b/test/validate/validate.go @@ -2,6 +2,7 @@ package validate import ( "context" + "encoding/json" "log" acnk8s "github.com/Azure/azure-container-networking/test/internal/kubernetes" @@ -72,6 +73,10 @@ func CreateValidator(ctx context.Context, clientset *kubernetes.Clientset, confi switch os { case "windows": checks = windowsChecksMap[cni] + err := acnk8s.RestartKubeProxyService(ctx, clientset, privilegedNamespace, privilegedLabelSelector, config) + if err != nil { + return nil, errors.Wrapf(err, "failed to restart kubeproxy") + } case "linux": checks = linuxChecksMap[cni] default: @@ -99,7 +104,7 @@ func (v *Validator) Validate(ctx context.Context) error { if v.os == "linux" { // We are restarting the systmemd network and checking that the connectivity works after the restart. For more details: https://github.com/cilium/cilium/issues/18706 log.Printf("Validating the restart network scenario") - err = v.ValidateRestartNetwork(ctx) + err = v.validateRestartNetwork(ctx) if err != nil { return errors.Wrapf(err, "failed to validate restart network scenario") } @@ -117,33 +122,6 @@ func (v *Validator) ValidateStateFile(ctx context.Context) error { return nil } -func (v *Validator) ValidateRestartNetwork(ctx context.Context) error { - nodes, err := acnk8s.GetNodeList(ctx, v.clientset) - if err != nil { - return errors.Wrapf(err, "failed to get node list") - } - - for index := range nodes.Items { - // get the privileged pod - pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) - if err != nil { - return errors.Wrapf(err, "failed to get privileged pod") - } - - privelegedPod := pod.Items[0] - // exec into the pod to get the state file - _, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartNetworkCmd, v.config) - if err != nil { - return errors.Wrapf(err, "failed to exec into privileged pod - %s", privelegedPod.Name) - } - err = acnk8s.WaitForPodsRunning(ctx, v.clientset, "", "") - if err != nil { - return errors.Wrapf(err, "failed to wait for pods running") - } - } - return nil -} - func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFunc, cmd []string, checkType, namespace, labelSelector string) error { log.Printf("Validating %s state file", checkType) nodes, err := acnk8s.GetNodeListByLabelSelector(ctx, v.clientset, nodeSelectorMap[v.os]) @@ -157,6 +135,9 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu if err != nil { return errors.Wrapf(err, "failed to get privileged pod") } + if len(pod.Items) == 0 { + return errors.Errorf("there are no privileged pods on node - %v", nodes.Items[index].Name) + } podName := pod.Items[0].Name // exec into the pod to get the state file result, err := acnk8s.ExecCmdOnPod(ctx, v.clientset, namespace, podName, cmd, v.config) @@ -165,7 +146,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu } filePodIps, err := stateFileIps(result) if err != nil { - return errors.Wrapf(err, "failed to get pod ips from state file") + return errors.Wrapf(err, "failed to get pod ips from state file on node %v", nodes.Items[index].Name) } if len(filePodIps) == 0 && v.restartCase { log.Printf("No pods found on node %s", nodes.Items[index].Name) @@ -175,7 +156,7 @@ func (v *Validator) validateIPs(ctx context.Context, stateFileIps stateFileIpsFu podIps := getPodIPsWithoutNodeIP(ctx, v.clientset, nodes.Items[index]) if err := compareIPs(filePodIps, podIps); err != nil { - return errors.Wrapf(errors.New("State file validation failed"), "for %s on node %s", checkType, nodes.Items[index].Name) + return errors.Wrapf(err, "State file validation failed for %s on node %s", checkType, nodes.Items[index].Name) } } log.Printf("State file validation for %s passed", checkType) @@ -257,36 +238,24 @@ func (v *Validator) ValidateDualStackControlPlane(ctx context.Context) error { return nil } -func (v *Validator) RestartKubeProxyService(ctx context.Context) error { - nodes, err := acnk8s.GetNodeList(ctx, v.clientset) - if err != nil { - return errors.Wrapf(err, "failed to get node list") - } - - for index := range nodes.Items { - node := nodes.Items[index] - if node.Status.NodeInfo.OperatingSystem != string(corev1.Windows) { - continue - } - // get the privileged pod - pod, err := acnk8s.GetPodsByNode(ctx, v.clientset, privilegedNamespace, privilegedLabelSelector, nodes.Items[index].Name) - if err != nil { - return errors.Wrapf(err, "failed to get privileged pod") - } - - privelegedPod := pod.Items[0] - // exec into the pod and restart kubeproxy - _, err = acnk8s.ExecCmdOnPod(ctx, v.clientset, privilegedNamespace, privelegedPod.Name, restartKubeProxyCmd, v.config) - if err != nil { - return errors.Wrapf(err, "failed to exec into privileged pod - %s", privelegedPod.Name) - } - } - return nil -} - func (v *Validator) Cleanup(ctx context.Context) { // deploy privileged pod privilegedDaemonSet := acnk8s.MustParseDaemonSet(privilegedDaemonSetPathMap[v.os]) daemonsetClient := v.clientset.AppsV1().DaemonSets(privilegedNamespace) acnk8s.MustDeleteDaemonset(ctx, daemonsetClient, privilegedDaemonSet) } + +func cnsCacheStateFileIps(result []byte) (map[string]string, error) { + var cnsLocalCache CNSLocalCache + + err := json.Unmarshal(result, &cnsLocalCache) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cns local cache") + } + + cnsPodIps := make(map[string]string) + for index := range cnsLocalCache.IPConfigurationStatus { + cnsPodIps[cnsLocalCache.IPConfigurationStatus[index].IPAddress] = cnsLocalCache.IPConfigurationStatus[index].PodInfo.Name() + } + return cnsPodIps, nil +} diff --git a/test/validate/windows_validate.go b/test/validate/windows_validate.go index 6cb1182e1a6..437bb0a1adb 100644 --- a/test/validate/windows_validate.go +++ b/test/validate/windows_validate.go @@ -1,6 +1,7 @@ package validate import ( + "bytes" "context" "encoding/json" "log" @@ -13,12 +14,21 @@ import ( "k8s.io/client-go/rest" ) +const ( + cnsWinLabelSelector = "k8s-app=azure-cns-win" +) + var ( - hnsEndPointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} - hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} - azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} - azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} - restartKubeProxyCmd = []string{"powershell", "Restart-service", "kubeproxy"} + hnsEndPointCmd = []string{"powershell", "-c", "Get-HnsEndpoint | ConvertTo-Json"} + hnsNetworkCmd = []string{"powershell", "-c", "Get-HnsNetwork | ConvertTo-Json"} + azureVnetCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet.json"} + azureVnetIpamCmd = []string{"powershell", "-c", "cat ../../k/azure-vnet-ipam.json"} + cnsWinCachedAssignedIPStateCmd = []string{ + "powershell", "Invoke-WebRequest -Uri 127.0.0.1:10090/debug/ipaddresses", + "-Method Post -ContentType application/x-www-form-urlencoded", + "-Body \"{`\"IPConfigStateFilter`\":[`\"Assigned`\"]}\"", + "-UseBasicParsing | Select-Object -Expand Content", + } ) var windowsChecksMap = map[string][]check{ @@ -28,14 +38,16 @@ var windowsChecksMap = map[string][]check{ {"azure-vnet-ipam", azureVnetIpamIps, privilegedLabelSelector, privilegedNamespace, azureVnetIpamCmd}, }, "cniv2": { + {"hns", hnsStateFileIps, privilegedLabelSelector, privilegedNamespace, hnsEndPointCmd}, {"azure-vnet", azureVnetIps, privilegedLabelSelector, privilegedNamespace, azureVnetCmd}, + {"cns cache", cnsCacheStateFileIps, cnsWinLabelSelector, privilegedNamespace, cnsWinCachedAssignedIPStateCmd}, }, } type HNSEndpoint struct { MacAddress string `json:"MacAddress"` IPAddress net.IP `json:"IPAddress"` - IPv6Address net.IP `json:",omitempty"` + IPv6Address net.IP `json:"IPv6Address"` IsRemoteEndpoint bool `json:",omitempty"` } @@ -90,18 +102,45 @@ type AddressRecord struct { } func hnsStateFileIps(result []byte) (map[string]string, error) { - var hnsResult []HNSEndpoint - err := json.Unmarshal(result, &hnsResult) - if err != nil { - return nil, errors.Wrapf(err, "failed to unmarshal hns endpoint list") - } - + jsonType := bytes.TrimLeft(result, " \t\r\n") + isObject := jsonType[0] == '{' + isArray := jsonType[0] == '[' hnsPodIps := make(map[string]string) - for _, v := range hnsResult { - if !v.IsRemoteEndpoint { - hnsPodIps[v.IPAddress.String()] = v.MacAddress + + switch { + case isObject: + var hnsResult HNSEndpoint + err := json.Unmarshal(result, &hnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal hns endpoint list") + } + if !hnsResult.IsRemoteEndpoint { + hnsPodIps[hnsResult.IPAddress.String()] = hnsResult.MacAddress + if hnsResult.IPv6Address.String() != "" { + hnsPodIps[hnsResult.IPv6Address.String()] = hnsResult.MacAddress + } } + case isArray: + var hnsResult []HNSEndpoint + err := json.Unmarshal(result, &hnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal hns endpoint list") + } + + for _, v := range hnsResult { + if !v.IsRemoteEndpoint { + hnsPodIps[v.IPAddress.String()] = v.MacAddress + if v.IPv6Address.String() != "" { + hnsPodIps[v.IPv6Address.String()] = v.MacAddress + } + + } + } + default: + log.Printf("Leading character is - %v", jsonType[0]) + return nil, errors.New("json is malformed and does not have correct leading character") } + return hnsPodIps, nil } @@ -165,7 +204,9 @@ func validateHNSNetworkState(ctx context.Context, nodes *corev1.NodeList, client if err != nil { return errors.Wrap(err, "failed to get privileged pod") } - + if len(pod.Items) == 0 { + return errors.Errorf("there are no privileged pods on node - %v", nodes.Items[index].Name) + } podName := pod.Items[0].Name // exec into the pod to get the state file result, err := acnk8s.ExecCmdOnPod(ctx, clientset, privilegedNamespace, podName, hnsNetworkCmd, restConfig) From 51d7c5cc68d8270ec1fff392b7d1543fb524b826 Mon Sep 17 00:00:00 2001 From: Evan Baker Date: Fri, 26 Jan 2024 13:01:55 -0600 Subject: [PATCH 14/31] feat: v2 swift ipampool (#2422) feat: v2 ipampool with idempotent scaling math --- cns/api.go | 3 +- cns/configuration/configuration.go | 28 +- cns/fakes/cnsfake.go | 9 + cns/ipampool/metrics.go | 108 +++--- cns/ipampool/monitor.go | 2 +- cns/ipampool/v2/adapter.go | 36 ++ cns/ipampool/v2/math_test.go | 116 ++++++ cns/ipampool/v2/monitor.go | 167 ++++++++ cns/ipampool/v2/monitor_test.go | 358 ++++++++++++++++++ cns/kubecontroller/pod/reconciler.go | 100 +++-- cns/restserver/ipam.go | 62 ++- cns/service/main.go | 41 +- go.mod | 2 +- .../manifests/cnsconfig/swiftconfigmap.yaml | 7 +- 14 files changed, 913 insertions(+), 126 deletions(-) create mode 100644 cns/ipampool/v2/adapter.go create mode 100644 cns/ipampool/v2/math_test.go create mode 100644 cns/ipampool/v2/monitor.go create mode 100644 cns/ipampool/v2/monitor_test.go diff --git a/cns/api.go b/cns/api.go index 9252825c259..3894e9aff62 100644 --- a/cns/api.go +++ b/cns/api.go @@ -50,6 +50,7 @@ type HTTPService interface { GetPodIPConfigState() map[string]IPConfigurationStatus MarkIPAsPendingRelease(numberToMark int) (map[string]IPConfigurationStatus, error) AttachIPConfigsHandlerMiddleware(IPConfigsHandlerMiddleware) + MarkNIPsPendingRelease(n int) (map[string]IPConfigurationStatus, error) } // IPConfigsHandlerFunc @@ -281,13 +282,13 @@ type NodeConfiguration struct { NodeSubnet Subnet } +// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct type IPAMPoolMonitor interface { Start(ctx context.Context) error Update(nnc *v1alpha.NodeNetworkConfig) error GetStateSnapshot() IpamPoolMonitorStateSnapshot } -// IpamPoolMonitorStateSnapshot struct to expose state values for IPAMPoolMonitor struct type IpamPoolMonitorStateSnapshot struct { MinimumFreeIps int64 MaximumFreeIps int64 diff --git a/cns/configuration/configuration.go b/cns/configuration/configuration.go index 005ed3f83bf..fd496131312 100644 --- a/cns/configuration/configuration.go +++ b/cns/configuration/configuration.go @@ -26,14 +26,26 @@ const ( ) type CNSConfig struct { + AZRSettings AZRSettings + AsyncPodDeletePath string + CNIConflistFilepath string + CNIConflistScenario string ChannelMode string + EnableAsyncPodDelete bool + EnableCNIConflistGeneration bool + EnableIPAMv2 bool EnablePprof bool EnableSubnetScarcity bool EnableSwiftV2 bool - SWIFTV2Mode SWIFTV2Mode InitializeFromCNI bool + KeyVaultSettings KeyVaultSettings + MSISettings MSISettings + ManageEndpointState bool ManagedSettings ManagedSettings + MellanoxMonitorIntervalSecs int MetricsBindAddress string + ProgramSNATIPTables bool + SWIFTV2Mode SWIFTV2Mode SyncHostNCTimeoutMs int SyncHostNCVersionIntervalMs int TLSCertificatePath string @@ -42,19 +54,8 @@ type CNSConfig struct { TLSSubjectName string TelemetrySettings TelemetrySettings UseHTTPS bool + WatchPods bool `json:"-"` WireserverIP string - KeyVaultSettings KeyVaultSettings - MSISettings MSISettings - ProgramSNATIPTables bool - ManageEndpointState bool - CNIConflistScenario string - EnableCNIConflistGeneration bool - CNIConflistFilepath string - MellanoxMonitorIntervalSecs int - AZRSettings AZRSettings - WatchPods bool - EnableAsyncPodDelete bool - AsyncPodDeletePath string } type TelemetrySettings struct { @@ -219,4 +220,5 @@ func SetCNSConfigDefaults(config *CNSConfig) { if config.AsyncPodDeletePath == "" { config.AsyncPodDeletePath = "/var/run/azure-vnet/deleteIDs" } + config.WatchPods = config.EnableIPAMv2 || config.EnableSwiftV2 } diff --git a/cns/fakes/cnsfake.go b/cns/fakes/cnsfake.go index aabedae2a5d..1158651de43 100644 --- a/cns/fakes/cnsfake.go +++ b/cns/fakes/cnsfake.go @@ -115,6 +115,11 @@ func (ipm *IPStateManager) ReleaseIPConfig(ipconfigID string) (cns.IPConfigurati return ipm.AvailableIPConfigState[ipconfigID], nil } +func (ipm *IPStateManager) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) { + // MarkIPASPendingRelease actually already errors if it is unable to release all N IPs + return ipm.MarkIPAsPendingRelease(n) +} + func (ipm *IPStateManager) MarkIPAsPendingRelease(numberOfIPsToMark int) (map[string]cns.IPConfigurationStatus, error) { ipm.Lock() defer ipm.Unlock() @@ -256,6 +261,10 @@ func (fake *HTTPServiceFake) GetPodIPConfigState() map[string]cns.IPConfiguratio return ipconfigs } +func (fake *HTTPServiceFake) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) { + return fake.IPStateManager.MarkIPAsPendingRelease(n) +} + // TODO: Populate on scale down func (fake *HTTPServiceFake) MarkIPAsPendingRelease(numberToMark int) (map[string]cns.IPConfigurationStatus, error) { return fake.IPStateManager.MarkIPAsPendingRelease(numberToMark) diff --git a/cns/ipampool/metrics.go b/cns/ipampool/metrics.go index 1f98ddc4b2a..75bcdf806da 100644 --- a/cns/ipampool/metrics.go +++ b/cns/ipampool/metrics.go @@ -12,12 +12,12 @@ const ( customerMetricLabel = "customer_metric" customerMetricLabelValue = "customer metric" subnetExhaustionStateLabel = "subnet_exhaustion_state" - subnetIPExhausted = 1 - subnetIPNotExhausted = 0 + SubnetIPExhausted = 1 + SubnetIPNotExhausted = 0 ) var ( - ipamAllocatedIPCount = prometheus.NewGaugeVec( + IpamAllocatedIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_pod_allocated_ips", Help: "IPs currently in use by Pods on this CNS Node.", @@ -25,7 +25,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamAvailableIPCount = prometheus.NewGaugeVec( + IpamAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_available_ips", Help: "IPs available on this CNS Node for use by a Pod.", @@ -33,7 +33,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamBatchSize = prometheus.NewGaugeVec( + IpamBatchSize = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_batch_size", Help: "IPAM IP pool scaling batch size.", @@ -41,7 +41,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamCurrentAvailableIPcount = prometheus.NewGaugeVec( + IpamCurrentAvailableIPcount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_current_available_ips", Help: "Current available IP count.", @@ -49,7 +49,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamExpectedAvailableIPCount = prometheus.NewGaugeVec( + IpamExpectedAvailableIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_expect_available_ips", Help: "Expected future available IP count assuming the Requested IP count is honored.", @@ -57,7 +57,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamMaxIPCount = prometheus.NewGaugeVec( + IpamMaxIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_max_ips", Help: "Maximum Secondary IPs allowed on this Node.", @@ -65,7 +65,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamPendingProgramIPCount = prometheus.NewGaugeVec( + IpamPendingProgramIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_pending_programming_ips", Help: "IPs reserved but not yet available (Pending Programming).", @@ -73,7 +73,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamPendingReleaseIPCount = prometheus.NewGaugeVec( + IpamPendingReleaseIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_pending_release_ips", Help: "IPs reserved but not available anymore (Pending Release).", @@ -81,7 +81,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamPrimaryIPCount = prometheus.NewGaugeVec( + IpamPrimaryIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_primary_ips", Help: "NC Primary IP count (reserved from Pod Subnet for DNS and IMDS SNAT).", @@ -89,7 +89,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamRequestedIPConfigCount = prometheus.NewGaugeVec( + IpamRequestedIPConfigCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_requested_ips", Help: "Secondary Pod Subnet IPs requested by this CNS Node (for Pods).", @@ -97,7 +97,7 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamSecondaryIPCount = prometheus.NewGaugeVec( + IpamSecondaryIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "cx_ipam_secondary_ips", Help: "Node NC Secondary IP count (reserved usable by Pods).", @@ -105,67 +105,67 @@ var ( }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamSubnetExhaustionCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cx_ipam_subnet_exhaustion_state_count_total", - Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion", - }, - []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel}, - ) - ipamSubnetExhaustionState = prometheus.NewGaugeVec( + IpamTotalIPCount = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_subnet_exhaustion_state", - Help: "CNS view of subnet exhaustion state", + Name: "cx_ipam_total_ips", + Help: "Count of total IP pool size allocated to CNS by DNC.", ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) - ipamTotalIPCount = prometheus.NewGaugeVec( + IpamSubnetExhaustionState = prometheus.NewGaugeVec( prometheus.GaugeOpts{ - Name: "cx_ipam_total_ips", - Help: "Total IPs reserved from the Pod Subnet by this Node.", + Name: "cx_ipam_subnet_exhaustion_state", + Help: "IPAM view of subnet exhaustion state", ConstLabels: prometheus.Labels{customerMetricLabel: customerMetricLabelValue}, }, []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel}, ) + IpamSubnetExhaustionCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "cx_ipam_subnet_exhaustion_state_count_total", + Help: "Count of the number of times the ipam pool monitor sees subnet exhaustion", + }, + []string{subnetLabel, subnetCIDRLabel, podnetARMIDLabel, subnetExhaustionStateLabel}, + ) ) func init() { metrics.Registry.MustRegister( - ipamAllocatedIPCount, - ipamAvailableIPCount, - ipamBatchSize, - ipamCurrentAvailableIPcount, - ipamExpectedAvailableIPCount, - ipamMaxIPCount, - ipamPendingProgramIPCount, - ipamPendingReleaseIPCount, - ipamPrimaryIPCount, - ipamRequestedIPConfigCount, - ipamSecondaryIPCount, - ipamSubnetExhaustionCount, - ipamSubnetExhaustionState, - ipamTotalIPCount, + IpamAllocatedIPCount, + IpamAvailableIPCount, + IpamBatchSize, + IpamCurrentAvailableIPcount, + IpamExpectedAvailableIPCount, + IpamMaxIPCount, + IpamPendingProgramIPCount, + IpamPendingReleaseIPCount, + IpamPrimaryIPCount, + IpamSecondaryIPCount, + IpamRequestedIPConfigCount, + IpamTotalIPCount, + IpamSubnetExhaustionState, + IpamSubnetExhaustionCount, ) } func observeIPPoolState(state ipPoolState, meta metaState) { labels := []string{meta.subnet, meta.subnetCIDR, meta.subnetARMID} - ipamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods)) - ipamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available)) - ipamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch)) - ipamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs)) - ipamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs)) - ipamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max)) - ipamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming)) - ipamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease)) - ipamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses))) - ipamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs)) - ipamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs)) - ipamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses)))) + IpamAllocatedIPCount.WithLabelValues(labels...).Set(float64(state.allocatedToPods)) + IpamAvailableIPCount.WithLabelValues(labels...).Set(float64(state.available)) + IpamBatchSize.WithLabelValues(labels...).Set(float64(meta.batch)) + IpamCurrentAvailableIPcount.WithLabelValues(labels...).Set(float64(state.currentAvailableIPs)) + IpamExpectedAvailableIPCount.WithLabelValues(labels...).Set(float64(state.expectedAvailableIPs)) + IpamMaxIPCount.WithLabelValues(labels...).Set(float64(meta.max)) + IpamPendingProgramIPCount.WithLabelValues(labels...).Set(float64(state.pendingProgramming)) + IpamPendingReleaseIPCount.WithLabelValues(labels...).Set(float64(state.pendingRelease)) + IpamPrimaryIPCount.WithLabelValues(labels...).Set(float64(len(meta.primaryIPAddresses))) + IpamRequestedIPConfigCount.WithLabelValues(labels...).Set(float64(state.requestedIPs)) + IpamSecondaryIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs)) + IpamTotalIPCount.WithLabelValues(labels...).Set(float64(state.secondaryIPs + int64(len(meta.primaryIPAddresses)))) if meta.exhausted { - ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPExhausted)) + IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPExhausted)) } else { - ipamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(subnetIPNotExhausted)) + IpamSubnetExhaustionState.WithLabelValues(labels...).Set(float64(SubnetIPNotExhausted)) } } diff --git a/cns/ipampool/monitor.go b/cns/ipampool/monitor.go index 18caf00e033..babf8639649 100644 --- a/cns/ipampool/monitor.go +++ b/cns/ipampool/monitor.go @@ -105,7 +105,7 @@ func (pm *Monitor) Start(ctx context.Context) error { case css := <-pm.cssSource: // received an updated ClusterSubnetState pm.metastate.exhausted = css.Status.Exhausted logger.Printf("subnet exhausted status = %t", pm.metastate.exhausted) - ipamSubnetExhaustionCount.With(prometheus.Labels{ + IpamSubnetExhaustionCount.With(prometheus.Labels{ subnetLabel: pm.metastate.subnet, subnetCIDRLabel: pm.metastate.subnetCIDR, podnetARMIDLabel: pm.metastate.subnetARMID, subnetExhaustionStateLabel: strconv.FormatBool(pm.metastate.exhausted), }).Inc() diff --git a/cns/ipampool/v2/adapter.go b/cns/ipampool/v2/adapter.go new file mode 100644 index 00000000000..3cc02d0e00e --- /dev/null +++ b/cns/ipampool/v2/adapter.go @@ -0,0 +1,36 @@ +package v2 + +import ( + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + v1 "k8s.io/api/core/v1" +) + +var _ cns.IPAMPoolMonitor = (*adapter)(nil) + +type adapter struct { + nncSink chan<- v1alpha.NodeNetworkConfig + *Monitor +} + +func (m *Monitor) AsV1(nncSink chan<- v1alpha.NodeNetworkConfig) cns.IPAMPoolMonitor { + return &adapter{ + nncSink: nncSink, + Monitor: m, + } +} + +func (m *adapter) Update(nnc *v1alpha.NodeNetworkConfig) error { + m.nncSink <- *nnc + return nil +} + +func (m *adapter) GetStateSnapshot() cns.IpamPoolMonitorStateSnapshot { + return cns.IpamPoolMonitorStateSnapshot{} +} + +func PodIPDemandListener(ch chan<- int) func([]v1.Pod) { + return func(pods []v1.Pod) { + ch <- len(pods) + } +} diff --git a/cns/ipampool/v2/math_test.go b/cns/ipampool/v2/math_test.go new file mode 100644 index 00000000000..e0c2ba9af16 --- /dev/null +++ b/cns/ipampool/v2/math_test.go @@ -0,0 +1,116 @@ +package v2 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCalculateTargetIPCount(t *testing.T) { + tests := []struct { + name string + demand int64 + batch int64 + buffer float64 + want int64 + }{ + { + name: "base case", + demand: 0, + batch: 16, + buffer: .5, + want: 16, + }, + { + name: "1/2 demand", + demand: 8, + batch: 16, + buffer: .5, + want: 16, + }, + { + name: "1x demand", + demand: 16, + batch: 16, + buffer: .5, + want: 32, + }, + { + name: "2x demand", + demand: 32, + batch: 16, + buffer: .5, + want: 48, + }, + { + name: "3x demand", + demand: 48, + batch: 16, + buffer: .5, + want: 64, + }, + { + name: "batch of one", + demand: 10, + batch: 1, + buffer: .5, + want: 11, + }, + { + name: "zero buffer", + demand: 10, + batch: 16, + buffer: 0, + want: 16, + }, + { + name: "zero buffer batch of one", + demand: 13, + batch: 1, + buffer: 0, + want: 13, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, calculateTargetIPCount(tt.demand, tt.batch, tt.buffer)) + }) + } +} + +func TestCalculateTargetIPCountOrMax(t *testing.T) { + tests := []struct { + name string + demand int64 + batch int64 + buffer float64 + max int64 + want int64 + }{ + { + name: "base case", + demand: 0, + batch: 16, + buffer: .5, + max: 100, + want: 16, + }, + { + name: "clamp to max", + demand: 500, + batch: 16, + buffer: .5, + max: 250, + want: 250, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + assert.Equal(t, tt.want, calculateTargetIPCountOrMax(tt.demand, tt.batch, tt.max, tt.buffer)) + }) + } +} diff --git a/cns/ipampool/v2/monitor.go b/cns/ipampool/v2/monitor.go new file mode 100644 index 00000000000..9f2081d3e76 --- /dev/null +++ b/cns/ipampool/v2/monitor.go @@ -0,0 +1,167 @@ +package v2 + +import ( + "context" + "math" + "sync" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/crd/clustersubnetstate/api/v1alpha1" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +const ( + // DefaultMaxIPs default maximum allocatable IPs on a k8s Node. + DefaultMaxIPs = 250 + // fieldManager is the field manager used when patching the NodeNetworkConfig. + fieldManager = "azure-cns" +) + +type nodeNetworkConfigSpecUpdater interface { + PatchSpec(context.Context, *v1alpha.NodeNetworkConfigSpec, string) (*v1alpha.NodeNetworkConfig, error) +} + +type ipStateStore interface { + GetPendingReleaseIPConfigs() []cns.IPConfigurationStatus + MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) +} + +type scaler struct { + batch int64 + buffer float64 + exhausted bool + max int64 +} + +type Monitor struct { + z *zap.Logger + scaler scaler + nnccli nodeNetworkConfigSpecUpdater + store ipStateStore + demand int64 + request int64 + demandSource <-chan int + cssSource <-chan v1alpha1.ClusterSubnetState + nncSource <-chan v1alpha.NodeNetworkConfig + started chan interface{} + once sync.Once +} + +func NewMonitor(z *zap.Logger, store ipStateStore, nnccli nodeNetworkConfigSpecUpdater, demandSource <-chan int, nncSource <-chan v1alpha.NodeNetworkConfig, cssSource <-chan v1alpha1.ClusterSubnetState) *Monitor { //nolint:lll // it's fine + return &Monitor{ + z: z.With(zap.String("component", "ipam-pool-monitor")), + store: store, + nnccli: nnccli, + demandSource: demandSource, + cssSource: cssSource, + nncSource: nncSource, + started: make(chan interface{}), + } +} + +// Start begins the Monitor's pool reconcile loop. +// On first run, it will block until a NodeNetworkConfig is received (through a call to Update()). +// Subsequently, it will run run once per RefreshDelay and attempt to re-reconcile the pool. +func (pm *Monitor) Start(ctx context.Context) error { + pm.z.Debug("starting") + for { + // proceed when things happen: + select { + case <-ctx.Done(): // calling context has closed, we'll exit. + return errors.Wrap(ctx.Err(), "pool monitor context closed") + case demand := <-pm.demandSource: // updated demand for IPs, recalculate request + pm.demand = int64(demand) + pm.z.Info("demand update", zap.Int64("demand", pm.demand)) + case css := <-pm.cssSource: // received an updated ClusterSubnetState, recalculate request + pm.scaler.exhausted = css.Status.Exhausted + pm.z.Info("exhaustion update", zap.Bool("exhausted", pm.scaler.exhausted)) + case nnc := <-pm.nncSource: // received a new NodeNetworkConfig, extract the data from it and recalculate request + pm.scaler.max = int64(math.Min(float64(nnc.Status.Scaler.MaxIPCount), DefaultMaxIPs)) + pm.scaler.batch = int64(math.Min(math.Max(float64(nnc.Status.Scaler.BatchSize), 1), float64(pm.scaler.max))) + pm.scaler.buffer = math.Abs(float64(nnc.Status.Scaler.RequestThresholdPercent)) / 100 //nolint:gomnd // it's a percentage + pm.once.Do(func() { + pm.request = nnc.Spec.RequestedIPCount + close(pm.started) // close the init channel the first time we fully receive a NodeNetworkConfig. + pm.z.Debug("started", zap.Int64("initial request", pm.request)) + }) + pm.z.Info("scaler update", zap.Int64("batch", pm.scaler.batch), zap.Float64("buffer", pm.scaler.buffer), zap.Int64("max", pm.scaler.max), zap.Int64("request", pm.request)) + } + select { + case <-pm.started: // this blocks until we have initialized + default: + // if we haven't started yet, we need to wait for the first NNC to be received. + continue // jumps to the next iteration of the outer for-loop + } + // if control has flowed through the select(s) to this point, we can now reconcile. + if err := pm.reconcile(ctx); err != nil { + pm.z.Error("reconcile failed", zap.Error(err)) + } + } +} + +func (pm *Monitor) reconcile(ctx context.Context) error { + // if the subnet is exhausted, locally overwrite the batch/minfree/maxfree in the meta copy for this iteration + // (until the controlplane owns this and modifies the scaler values for us directly instead of writing "exhausted") + // TODO(rbtr) + s := pm.scaler + if s.exhausted { + s.batch = 1 + s.buffer = 1 + } + + // calculate the target state from the current pool state and scaler + target := calculateTargetIPCountOrMax(pm.demand, s.batch, s.max, s.buffer) + pm.z.Info("calculated new request", zap.Int64("demand", pm.demand), zap.Int64("batch", s.batch), zap.Int64("max", s.max), zap.Float64("buffer", s.buffer), zap.Int64("target", target)) + delta := target - pm.request + if delta == 0 { + return nil + } + pm.z.Info("scaling pool", zap.Int64("delta", delta)) + // try to release -delta IPs. this is no-op if delta is negative. + if _, err := pm.store.MarkNIPsPendingRelease(int(-delta)); err != nil { + return errors.Wrapf(err, "failed to mark sufficient IPs as PendingRelease, wanted %d", pm.request-target) + } + spec := pm.buildNNCSpec(target) + if _, err := pm.nnccli.PatchSpec(ctx, &spec, fieldManager); err != nil { + return errors.Wrap(err, "failed to UpdateSpec with NNC client") + } + pm.request = target + pm.z.Info("scaled pool", zap.Int64("request", pm.request)) + return nil +} + +// buildNNCSpec translates CNS's map of IPs to be released and requested IP count into an NNC Spec. +func (pm *Monitor) buildNNCSpec(request int64) v1alpha.NodeNetworkConfigSpec { + // Get All Pending IPs from CNS and populate it again. + pendingReleaseIPs := pm.store.GetPendingReleaseIPConfigs() + spec := v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: request, + IPsNotInUse: make([]string, len(pendingReleaseIPs)), + } + for i := range pendingReleaseIPs { + spec.IPsNotInUse[i] = pendingReleaseIPs[i].ID + } + return spec +} + +// calculateTargetIPCountOrMax calculates the target IP count request +// using the scaling function and clamps the result at the max IPs. +func calculateTargetIPCountOrMax(demand, batch, max int64, buffer float64) int64 { + targetRequest := calculateTargetIPCount(demand, batch, buffer) + if targetRequest > max { + // clamp request at the max IPs + targetRequest = max + } + return targetRequest +} + +// calculateTargetIPCount calculates an IP count request based on the +// current demand, batch size, and buffer. +// ref: https://github.com/Azure/azure-container-networking/blob/master/docs/feature/ipammath/0-background.md +// the idempotent scaling function is: +// Target = Batch \times \lceil buffer + \frac{Demand}{Batch} \rceil +func calculateTargetIPCount(demand, batch int64, buffer float64) int64 { + return batch * int64(math.Ceil(buffer+float64(demand)/float64(batch))) +} diff --git a/cns/ipampool/v2/monitor_test.go b/cns/ipampool/v2/monitor_test.go new file mode 100644 index 00000000000..8d10b5b9881 --- /dev/null +++ b/cns/ipampool/v2/monitor_test.go @@ -0,0 +1,358 @@ +package v2 + +import ( + "context" + "math/rand" + "net/netip" + "testing" + + "github.com/Azure/azure-container-networking/cns" + "github.com/Azure/azure-container-networking/cns/types" + "github.com/Azure/azure-container-networking/crd/nodenetworkconfig/api/v1alpha" + "github.com/google/uuid" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "golang.org/x/exp/maps" +) + +type ipStateStoreMock struct { + pendingReleaseIPConfigs map[string]cns.IPConfigurationStatus + err error +} + +func (m *ipStateStoreMock) GetPendingReleaseIPConfigs() []cns.IPConfigurationStatus { + return maps.Values(m.pendingReleaseIPConfigs) +} + +func (m *ipStateStoreMock) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) { + if m.err != nil { + return nil, m.err + } + newPendingRelease := pendingReleaseGenerator(n) + maps.Copy(newPendingRelease, m.pendingReleaseIPConfigs) + m.pendingReleaseIPConfigs = newPendingRelease + return m.pendingReleaseIPConfigs, nil +} + +// pendingReleaseGenerator generates a variable number of random pendingRelease IPConfigs. +func pendingReleaseGenerator(n int) map[string]cns.IPConfigurationStatus { + m := make(map[string]cns.IPConfigurationStatus, n) + ip := netip.MustParseAddr("10.0.0.0") + for i := 0; i < n; i++ { + id := uuid.New().String() + ip = ip.Next() + status := cns.IPConfigurationStatus{ + ID: id, + IPAddress: ip.String(), + } + status.SetState(types.PendingRelease) + m[id] = status + } + return m +} + +func TestPendingReleaseIPConfigsGenerator(t *testing.T) { + t.Parallel() + n := rand.Intn(100) //nolint:gosec // test + m := pendingReleaseGenerator(n) + assert.Len(t, m, n, "pendingReleaseGenerator made the wrong quantity") + for k, v := range m { + _, err := uuid.Parse(v.ID) + require.NoError(t, err, "pendingReleaseGenerator made a bad UUID") + assert.Equal(t, k, v.ID, "pendingReleaseGenerator stored using the wrong key ") + _, err = netip.ParseAddr(v.IPAddress) + require.NoError(t, err, "pendingReleaseGenerator made a bad IP") + assert.Equal(t, types.PendingRelease, v.GetState(), "pendingReleaseGenerator set the wrong State") + } +} + +func TestBuildNNCSpec(t *testing.T) { + tests := []struct { + name string + pendingReleaseIPConfigs map[string]cns.IPConfigurationStatus + request int64 + }{ + { + name: "without no pending release", + request: 16, + }, + { + name: "with pending release", + pendingReleaseIPConfigs: pendingReleaseGenerator(16), + request: 16, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + pm := &Monitor{ + store: &ipStateStoreMock{ + pendingReleaseIPConfigs: tt.pendingReleaseIPConfigs, + }, + } + spec := pm.buildNNCSpec(tt.request) + assert.Equal(t, tt.request, spec.RequestedIPCount) + assert.Equal(t, len(tt.pendingReleaseIPConfigs), len(spec.IPsNotInUse)) + assert.ElementsMatch(t, maps.Keys(tt.pendingReleaseIPConfigs), spec.IPsNotInUse) + }) + } +} + +type nncClientMock struct { + req v1alpha.NodeNetworkConfigSpec + err error +} + +func (m *nncClientMock) PatchSpec(_ context.Context, spec *v1alpha.NodeNetworkConfigSpec, _ string) (*v1alpha.NodeNetworkConfig, error) { + if m.err != nil { + return nil, m.err + } + m.req = *spec + return nil, nil +} + +func TestReconcile(t *testing.T) { + tests := []struct { + name string + demand int64 + request int64 + scaler scaler + nnccli nncClientMock + store ipStateStoreMock + wantRequest int64 + wantPendingRelease int + wantErr bool + }{ + // no-op case + { + name: "no delta", + demand: 5, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{ + req: v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: 16, + }, + }, + store: ipStateStoreMock{}, + wantRequest: 16, + }, + // fail to mark IPs pending release + { + name: "fail to release", + demand: 6, + request: 32, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{ + req: v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: 32, + }, + }, + store: ipStateStoreMock{ + err: errors.Errorf("failed to mark IPs pending release"), + }, + wantRequest: 32, + wantErr: true, + }, + // fail to Patch NNC Spec + { + name: "fail to patch", + demand: 20, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{ + req: v1alpha.NodeNetworkConfigSpec{ + RequestedIPCount: 16, + }, + err: errors.Errorf("failed to patch NNC Spec"), + }, + store: ipStateStoreMock{}, + wantRequest: 16, + wantErr: true, + }, + // normal scale ups with no pending release + { + name: "single scale up", + demand: 15, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 32, + }, + { + name: "big scale up", + demand: 75, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 96, + }, + { + name: "capped scale up", + demand: 300, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 250, + }, + // normal scale down with no previously pending release + { + name: "single scale down", + demand: 5, + request: 32, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 16, + wantPendingRelease: 16, + }, + { + name: "big scale down", + demand: 5, + request: 128, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 16, + wantPendingRelease: 112, + }, + { + name: "capped scale down", + demand: 0, + request: 32, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 16, + wantPendingRelease: 16, + }, + // realign to batch if request is skewed + { + name: "scale up unskew", + demand: 15, + request: 3, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 32, + }, + { + name: "scale down unskew", + demand: 5, + request: 37, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{}, + wantRequest: 16, + wantPendingRelease: 21, + }, + // normal scale up with previous pending release + { + name: "single scale up with pending release", + demand: 20, + request: 16, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{ + pendingReleaseIPConfigs: pendingReleaseGenerator(16), + }, + wantRequest: 32, + wantPendingRelease: 16, + }, + // normal scale down with previous pending release + { + name: "single scale down with pending release", + demand: 5, + request: 32, + scaler: scaler{ + batch: 16, + buffer: .5, + max: 250, + }, + nnccli: nncClientMock{}, + store: ipStateStoreMock{ + pendingReleaseIPConfigs: pendingReleaseGenerator(16), + }, + wantRequest: 16, + wantPendingRelease: 32, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tt := tt + t.Parallel() + pm := &Monitor{ + z: zap.NewNop(), + demand: tt.demand, + request: tt.request, + scaler: tt.scaler, + nnccli: &tt.nnccli, + store: &tt.store, + } + err := pm.reconcile(context.Background()) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + assert.Equal(t, tt.wantRequest, pm.request) + assert.Equal(t, tt.wantRequest, tt.nnccli.req.RequestedIPCount) + assert.Len(t, tt.nnccli.req.IPsNotInUse, tt.wantPendingRelease) + assert.Equal(t, tt.wantRequest, pm.request) + }) + } +} diff --git a/cns/kubecontroller/pod/reconciler.go b/cns/kubecontroller/pod/reconciler.go index c984574327f..9767ddad222 100644 --- a/cns/kubecontroller/pod/reconciler.go +++ b/cns/kubecontroller/pod/reconciler.go @@ -2,10 +2,11 @@ package pod import ( "context" + "strconv" "github.com/pkg/errors" + "go.uber.org/zap" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/fields" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/event" @@ -13,68 +14,87 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" ) -type podcli interface { +type cli interface { List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error } -type podListener interface { - Update([]v1.Pod) +// watcher watches Pods on the current Node and notifies listeners of changes. +type watcher struct { + z *zap.Logger + cli cli + reconcileFuncs []reconcile.Func } -type PodWatcher struct { - cli podcli - listOpt client.ListOption - ReconcileFuncs []reconcile.Func +func New(z *zap.Logger) *watcher { //nolint:revive // force usage of new by keeping the struct private + return &watcher{ + z: z.With(zap.String("component", "pod-watcher")), + } } -func New(nodename string) *PodWatcher { //nolint:revive // private struct to force constructor - return &PodWatcher{ - listOpt: &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": nodename})}, - } +// With adds reconcile.Funcs to the Watcher. +func (p *watcher) With(fs ...reconcile.Func) *watcher { + p.reconcileFuncs = append(p.reconcileFuncs, fs...) + return p } -func (p *PodWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { - for _, f := range p.ReconcileFuncs { - if _, err := f(ctx, req); err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to reconcile") +func (p *watcher) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { + for _, f := range p.reconcileFuncs { + if res, err := f(ctx, req); !res.IsZero() || err != nil { + return res, errors.Wrap(err, "failed to reconcile") } } - return reconcile.Result{}, nil + return ctrl.Result{}, nil } -type PodFilter func([]v1.Pod) []v1.Pod - -var PodNetworkFilter PodFilter = func(pods []v1.Pod) []v1.Pod { - var filtered []v1.Pod - for _, pod := range pods { - if !pod.Spec.HostNetwork { - filtered = append(filtered, pod) - } - } - return filtered +type limiter interface { + Allow() bool } -func (p *PodWatcher) PodNotifierFunc(f PodFilter, listeners ...podListener) reconcile.Func { +// NotifierFunc returns a reconcile.Func that lists Pods to get the latest +// state and notifies listeners of the resulting Pods. +// listOpts are passed to the client.List call to filter the Pod list. +// limiter is an optional rate limiter which may be used to limit the +// rate at which listeners are notified of list changes. This guarantees +// that all Pod events will eventually be processed, but allows the listeners +// to react to less (but more complete) changes. If we rate limit events, we +// end up sending a version of the Pod list that is newer, without missing +// any events. +// listeners are called with the new Pod list. +func (p *watcher) NewNotifierFunc(listOpts *client.ListOptions, limiter limiter, listeners ...func([]v1.Pod)) reconcile.Func { + p.z.Debug("adding notified for listeners", zap.Int("listeners", len(listeners))) return func(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + if !limiter.Allow() { + // rate limit exceeded, requeue + p.z.Debug("rate limit exceeded") + return ctrl.Result{Requeue: true}, nil + } podList := &v1.PodList{} - if err := p.cli.List(ctx, podList, p.listOpt); err != nil { - return reconcile.Result{}, errors.Wrap(err, "failed to list pods") + if err := p.cli.List(ctx, podList, listOpts); err != nil { + return ctrl.Result{}, errors.Wrap(err, "failed to list pods") } pods := podList.Items - if f != nil { - pods = f(pods) - } for _, l := range listeners { - l.Update(pods) + l(pods) } - return reconcile.Result{}, nil + return ctrl.Result{}, nil } } +var hostNetworkIndexer = client.IndexerFunc(func(o client.Object) []string { + pod, ok := o.(*v1.Pod) + if !ok { + return nil + } + return []string{strconv.FormatBool(pod.Spec.HostNetwork)} +}) + // SetupWithManager Sets up the reconciler with a new manager, filtering using NodeNetworkConfigFilter on nodeName. -func (p *PodWatcher) SetupWithManager(mgr ctrl.Manager) error { +func (p *watcher) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { p.cli = mgr.GetClient() - err := ctrl.NewControllerManagedBy(mgr). + if err := mgr.GetFieldIndexer().IndexField(ctx, &v1.Pod{}, "spec.hostNetwork", hostNetworkIndexer); err != nil { + return errors.Wrap(err, "failed to set up hostNetwork indexer") + } + if err := ctrl.NewControllerManagedBy(mgr). For(&v1.Pod{}). WithEventFilter(predicate.Funcs{ // we only want create/delete events UpdateFunc: func(event.UpdateEvent) bool { @@ -84,6 +104,8 @@ func (p *PodWatcher) SetupWithManager(mgr ctrl.Manager) error { return false }, }). - Complete(p) - return errors.Wrap(err, "failed to set up pod watcher with manager") + Complete(p); err != nil { + return errors.Wrap(err, "failed to set up pod watcher with manager") + } + return nil } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index c9f4d059cfc..d9cf742848d 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -18,6 +18,7 @@ import ( "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/store" "github.com/pkg/errors" + "golang.org/x/exp/maps" ) var ( @@ -437,6 +438,65 @@ func (service *HTTPRestService) MarkIPAsPendingRelease(totalIpsToRelease int) (m return pendingReleasedIps, nil } +// MarkIPAsPendingRelease will attempt to set [n] number of ips to PendingRelease state. +// It will start with any IPs in PendingProgramming state and then move on to any IPs in Allocated state +// until it has reached the target release quantity. +// If it is unable to set the expected number of IPs to PendingRelease, it will revert the changed IPs +// and return an error. +// MarkNIPsPendingRelease is no-op if [n] is not a positive integer. +func (service *HTTPRestService) MarkNIPsPendingRelease(n int) (map[string]cns.IPConfigurationStatus, error) { + service.Lock() + defer service.Unlock() + // try to release from PendingProgramming + pendingProgrammingIPs := make(map[string]cns.IPConfigurationStatus) + for uuid, ipConfig := range service.PodIPConfigState { //nolint:gocritic // intentional value copy + if n <= 0 { + break + } + if ipConfig.GetState() == types.PendingProgramming { + updatedIPConfig, err := service.updateIPConfigState(uuid, types.PendingRelease, ipConfig.PodInfo) + if err != nil { + return nil, err + } + + pendingProgrammingIPs[uuid] = updatedIPConfig + n-- + } + } + + // try to release from Available + availableIPs := make(map[string]cns.IPConfigurationStatus) + for uuid, ipConfig := range service.PodIPConfigState { //nolint:gocritic // intentional value copy + if n <= 0 { + break + } + if ipConfig.GetState() == types.Available { + updatedIPConfig, err := service.updateIPConfigState(uuid, types.PendingRelease, ipConfig.PodInfo) + if err != nil { + return nil, err + } + + availableIPs[uuid] = updatedIPConfig + n-- + } + } + + // if we can release the requested quantity, return the IPs + if n <= 0 { + maps.Copy(pendingProgrammingIPs, availableIPs) + return pendingProgrammingIPs, nil + } + + // else revert changes + for uuid, ipConfig := range pendingProgrammingIPs { //nolint:gocritic // intentional value copy + _, _ = service.updateIPConfigState(uuid, types.PendingProgramming, ipConfig.PodInfo) + } + for uuid, ipConfig := range availableIPs { //nolint:gocritic // intentional value copy + _, _ = service.updateIPConfigState(uuid, types.Available, ipConfig.PodInfo) + } + return nil, errors.New("unable to release requested number of IPs") +} + // TODO: Add a change so that we should only update the current state if it is different than the new state func (service *HTTPRestService) updateIPConfigState(ipID string, updatedState types.IPState, podInfo cns.PodInfo) (cns.IPConfigurationStatus, error) { if ipConfig, found := service.PodIPConfigState[ipID]; found { @@ -447,7 +507,7 @@ func (service *HTTPRestService) updateIPConfigState(ipID string, updatedState ty return ipConfig, nil } - //nolint:goerr113 + //nolint:goerr113 //legacy return cns.IPConfigurationStatus{}, fmt.Errorf("[updateIPConfigState] Failed to update state %s for the IPConfig. ID %s not found PodIPConfigState", updatedState, ipID) } diff --git a/cns/service/main.go b/cns/service/main.go index fec9a25194f..405ff9f16d7 100644 --- a/cns/service/main.go +++ b/cns/service/main.go @@ -33,6 +33,7 @@ import ( "github.com/Azure/azure-container-networking/cns/hnsclient" "github.com/Azure/azure-container-networking/cns/imds" "github.com/Azure/azure-container-networking/cns/ipampool" + ipampoolv2 "github.com/Azure/azure-container-networking/cns/ipampool/v2" cssctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/clustersubnetstate" mtpncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/multitenantpodnetworkconfig" nncctrl "github.com/Azure/azure-container-networking/cns/kubecontroller/nodenetworkconfig" @@ -63,6 +64,7 @@ import ( "github.com/pkg/errors" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "golang.org/x/time/rate" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -115,6 +117,7 @@ const ( var ( rootCtx context.Context rootErrCh chan error + z *zap.Logger ) // Version is populated by make during build. @@ -617,7 +620,10 @@ func main() { // configure zap logger zconfig := zap.NewProductionConfig() zconfig.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder - z, _ := zconfig.Build() + if z, err = zconfig.Build(); err != nil { + fmt.Printf("failed to create logger: %v", err) + os.Exit(1) + } // start the healthz/readyz/metrics server readyCh := make(chan interface{}) @@ -1336,9 +1342,6 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn return errors.Wrap(err, "failed to create manager") } - // Build the IPAM Pool monitor - clusterSubnetStateChan := make(chan cssv1alpha1.ClusterSubnetState) - // this cachedscopedclient is built using the Manager's cached client, which is // NOT SAFE TO USE UNTIL THE MANAGER IS STARTED! // This is okay because it is only used to build the IPAMPoolMonitor, which does not @@ -1347,18 +1350,24 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // reconciler has pushed the Monitor a NodeNetworkConfig. cachedscopedcli := nncctrl.NewScopedClient(nodenetworkconfig.NewClient(manager.GetClient()), types.NamespacedName{Namespace: "kube-system", Name: nodeName}) - poolOpts := ipampool.Options{ - RefreshDelay: poolIPAMRefreshRateInMilliseconds * time.Millisecond, + // Build the IPAM Pool monitor + var poolMonitor cns.IPAMPoolMonitor + cssCh := make(chan cssv1alpha1.ClusterSubnetState) + ipDemandCh := make(chan int) + if cnsconfig.EnableIPAMv2 { + nncCh := make(chan v1alpha.NodeNetworkConfig) + poolMonitor = ipampoolv2.NewMonitor(z, httpRestServiceImplementation, cachedscopedcli, ipDemandCh, nncCh, cssCh).AsV1(nncCh) + } else { + poolOpts := ipampool.Options{ + RefreshDelay: poolIPAMRefreshRateInMilliseconds * time.Millisecond, + } + poolMonitor = ipampool.NewMonitor(httpRestServiceImplementation, cachedscopedcli, cssCh, &poolOpts) } - poolMonitor := ipampool.NewMonitor(httpRestServiceImplementation, cachedscopedcli, clusterSubnetStateChan, &poolOpts) - httpRestServiceImplementation.IPAMPoolMonitor = poolMonitor // Start building the NNC Reconciler // get CNS Node IP to compare NC Node IP with this Node IP to ensure NCs were created for this node nodeIP := configuration.NodeIP() - - // NodeNetworkConfig reconciler nncReconciler := nncctrl.NewReconciler(httpRestServiceImplementation, poolMonitor, nodeIP) // pass Node to the Reconciler for Controller xref if err := nncReconciler.SetupWithManager(manager, node); err != nil { //nolint:govet // intentional shadow @@ -1367,7 +1376,7 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn if cnsconfig.EnableSubnetScarcity { // ClusterSubnetState reconciler - cssReconciler := cssctrl.New(clusterSubnetStateChan) + cssReconciler := cssctrl.New(cssCh) if err := cssReconciler.SetupWithManager(manager); err != nil { return errors.Wrapf(err, "failed to setup css reconciler with manager") } @@ -1375,8 +1384,14 @@ func InitializeCRDState(ctx context.Context, httpRestService cns.HTTPService, cn // TODO: add pod listeners based on Swift V1 vs MT/V2 configuration if cnsconfig.WatchPods { - pw := podctrl.New(nodeName) - if err := pw.SetupWithManager(manager); err != nil { + pw := podctrl.New(z) + if cnsconfig.EnableIPAMv2 { + hostNetworkListOpt := &client.ListOptions{FieldSelector: fields.SelectorFromSet(fields.Set{"spec.hostNetwork": "false"})} // filter only podsubnet pods + // don't relist pods more than every 500ms + limit := rate.NewLimiter(rate.Every(500*time.Millisecond), 1) //nolint:gomnd // clearly 500ms + pw.With(pw.NewNotifierFunc(hostNetworkListOpt, limit, ipampoolv2.PodIPDemandListener(ipDemandCh))) + } + if err := pw.SetupWithManager(ctx, manager); err != nil { return errors.Wrapf(err, "failed to setup pod watcher with manager") } } diff --git a/go.mod b/go.mod index 5f93e41cf49..ee9e4eccd3f 100644 --- a/go.mod +++ b/go.mod @@ -113,7 +113,7 @@ require ( golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/time v0.5.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/test/integration/manifests/cnsconfig/swiftconfigmap.yaml b/test/integration/manifests/cnsconfig/swiftconfigmap.yaml index 76bc6642c79..254118e391e 100644 --- a/test/integration/manifests/cnsconfig/swiftconfigmap.yaml +++ b/test/integration/manifests/cnsconfig/swiftconfigmap.yaml @@ -21,10 +21,11 @@ data: "NodeID": "", "NodeSyncIntervalInSeconds": 30 }, + "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs", "ChannelMode": "CRD", + "EnableAsyncPodDelete": true, + "EnableIPAMv2": true, "InitializeFromCNI": true, "ManageEndpointState": false, - "ProgramSNATIPTables" : false, - "EnableAsyncPodDelete": true, - "AsyncPodDeletePath": "/var/run/azure-vnet/deleteIDs" + "ProgramSNATIPTables" : false } From d2f142fd901af3ad41c0c6226ab51ef69b6aa022 Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Fri, 26 Jan 2024 14:01:11 -0600 Subject: [PATCH 15/31] ci: Replace make test-intergration within ACN PR pipeline (#2463) * ci: change CNS integration * ci: output pod status before/after CNS --- .../aks-swift/e2e-step-template.yaml | 31 ++--------------- .../azure-cni-overlay-e2e-step-template.yaml | 5 ++- .../cilium-overlay-e2e-step-template.yaml | 34 ++----------------- .../cilium/cilium-e2e-step-template.yaml | 33 ++---------------- .../dualstackoverlay-e2e-step-template.yaml | 4 ++- 5 files changed, 15 insertions(+), 92 deletions(-) diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml index 61b39e23bb1..ddb4c219254 100644 --- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml +++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml @@ -52,29 +52,13 @@ steps: pwd kubectl cluster-info kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) INSTALL_CNS=true INSTALL_AZURE_VNET=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aksswifte2e" displayName: "Run AKS Swift E2E" - script: | - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: aks-swift-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output" - condition: always() - - - script: | + kubectl get po -owide -A echo "Run Service Conformance E2E" export PATH=${PATH}:/usr/local/bin/gsutil KUBECONFIG=~/.kube/config kubetest2 noop \ @@ -98,7 +82,7 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - + - script: | cd hack/scripts chmod +x async-delete-test.sh @@ -108,12 +92,3 @@ steps: fi name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml index 3a6f5290619..cd942cdb89b 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e-step-template.yaml @@ -34,7 +34,8 @@ steps: - ${{ if eq(parameters.os, 'linux') }}: - script: | echo "Start Integration Tests on Overlay Cluster" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true retryCountOnTaskFailure: 2 name: "integrationTest" displayName: "Run CNS Integration Tests on AKS Overlay" @@ -47,6 +48,7 @@ steps: addSpnToEnvironment: true inlineScript: | set -e + kubectl get po -owide -A clusterName=${{ parameters.clusterName }} echo "Restarting nodes" for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do @@ -101,6 +103,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} echo "Windows node are successfully added to v4 Overlay Cluster" kubectl cluster-info diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml index 71a7bfc8bd3..13e3387aedf 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e-step-template.yaml @@ -81,33 +81,14 @@ steps: else CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" - script: | - echo "Status of the nodes and pods after the test" - kubectl get nodes -o wide - kubectl get pods -A -o wide - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: test-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" - condition: always() - - - script: | - kubectl get pods -A + kubectl get po -owide -A echo "Waiting < 2 minutes for cilium to be ready" # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m @@ -203,12 +184,3 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml index c09b215bc81..f0b529bcb61 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e-step-template.yaml @@ -72,33 +72,13 @@ steps: - script: | echo "Start Azilium E2E Tests" kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_AZILIUM=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZILIUM=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E" - script: | - echo "Status of the nodes and pods after the test" - kubectl get nodes -o wide - kubectl get pods -A -o wide - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: test-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" - condition: always() - - - script: | - kubectl get pods -A + kubectl get po -owide -A echo "Waiting < 2 minutes for cilium to be ready" # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m @@ -147,12 +127,3 @@ steps: retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml index aef22612bcf..3a519f83cea 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e-step-template.yaml @@ -36,14 +36,15 @@ steps: - script: | kubectl cluster-info kubectl get node - sudo -E env "PATH=$PATH" make test-integration CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true retryCountOnTaskFailure: 3 name: "integrationTest" displayName: "Run CNS Integration Tests on AKS DualStack Overlay" - script: | set -e + kubectl get po -owide -A cd test/integration/datapath echo "Dualstack Overlay Linux datapath IPv6 test" go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true @@ -108,6 +109,7 @@ steps: scriptType: "bash" addSpnToEnvironment: true inlineScript: | + set -e make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} echo "Windows nodes have been successfully added to DualStack Overlay Cluster" kubectl cluster-info From 39f436eabba40347f3cea50d0fbed87cb6254f35 Mon Sep 17 00:00:00 2001 From: Camryn Lee <31013536+camrynl@users.noreply.github.com> Date: Fri, 26 Jan 2024 13:21:32 -0800 Subject: [PATCH 16/31] ci: update pod-cidrs in ip-masq-agent (#2549) add more pod cidrs to ip-mas-agent cm --- test/integration/manifests/ip-masq-agent/config-custom.yaml | 2 ++ test/integration/manifests/ip-masq-agent/config-reconcile.yaml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/test/integration/manifests/ip-masq-agent/config-custom.yaml b/test/integration/manifests/ip-masq-agent/config-custom.yaml index b8502aa71ff..4bdc6cc5ee1 100644 --- a/test/integration/manifests/ip-masq-agent/config-custom.yaml +++ b/test/integration/manifests/ip-masq-agent/config-custom.yaml @@ -11,5 +11,7 @@ data: ip-masq-agent: |- nonMasqueradeCIDRs: - 192.168.0.0/16 + - 100.64.0.0/10 + - 10.244.0.0/16 masqLinkLocal: false masqLinkLocalIPv6: true diff --git a/test/integration/manifests/ip-masq-agent/config-reconcile.yaml b/test/integration/manifests/ip-masq-agent/config-reconcile.yaml index 1358ad0f1da..0f715267d82 100644 --- a/test/integration/manifests/ip-masq-agent/config-reconcile.yaml +++ b/test/integration/manifests/ip-masq-agent/config-reconcile.yaml @@ -9,4 +9,6 @@ data: ip-masq-agent-reconciled: |- nonMasqueradeCIDRs: - 192.168.0.0/16 + - 100.64.0.0/10 + - 10.244.0.0/16 masqLinkLocal: true From 36d83ce1ceebc14b749c067fb46b42f4dcc820a1 Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Fri, 26 Jan 2024 20:43:08 -0600 Subject: [PATCH 17/31] ci: nsenter daemonset (#2553) --- .../manifests/load/privileged-daemonset.yaml | 15 +++++++++++++-- test/validate/linux_validate.go | 2 +- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/test/integration/manifests/load/privileged-daemonset.yaml b/test/integration/manifests/load/privileged-daemonset.yaml index 7f4d370b66f..715f031ef52 100644 --- a/test/integration/manifests/load/privileged-daemonset.yaml +++ b/test/integration/manifests/load/privileged-daemonset.yaml @@ -21,8 +21,19 @@ spec: hostPID: true containers: - name: privileged-container - image: mcr.microsoft.com/dotnet/runtime-deps:6.0 - command: ["/bin/sleep", "3650d"] + image: mcr.microsoft.com/mirror/docker/library/alpine:3.16 + command: + - nsenter + - -t + - "1" + - -m + - -u + - -i + - -n + - -p + - bash + - -c + - sleep 3650d securityContext: privileged: true runAsUser: 0 diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index ed9c919eb27..2e2dcc807d1 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -18,7 +18,7 @@ const ( ) var ( - restartNetworkCmd = []string{"bash", "-c", "chroot /host /bin/bash -c systemctl restart systemd-networkd"} + restartNetworkCmd = []string{"bash", "-c", "systemctl restart systemd-networkd"} cnsManagedStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-cns/azure-endpoints.json"} azureVnetStateFileCmd = []string{"bash", "-c", "cat /var/run/azure-vnet.json"} azureVnetIpamStateCmd = []string{"bash", "-c", "cat /var/run/azure-vnet-ipam.json"} From 8b5a956db8181de0099a75d71f8dc346fa44ca7a Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Fri, 26 Jan 2024 22:13:07 -0600 Subject: [PATCH 18/31] ci: Replace make test-integration [2/2] (#2554) ci: capture missed e2e --- .../cilium-overlay-e2e-step-template.yaml | 35 +------------------ 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml index 0e59910c8b5..0367ed2ad8d 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-step-template.yaml @@ -73,31 +73,11 @@ steps: else CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} INSTALL_CNS=true INSTALL_OVERLAY=true + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true retryCountOnTaskFailure: 3 name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" - - script: | - echo "Status of the nodes and pods after the test" - kubectl get nodes -o wide - kubectl get pods -A -o wide - echo "Logs will be available as a build artifact" - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo mkdir $ARTIFACT_DIR - sudo cp test/integration/logs/* $ARTIFACT_DIR - name: "GetLogs" - displayName: "Get logs" - condition: always() - - - task: PublishBuildArtifacts@1 - inputs: - artifactName: test-output - pathtoPublish: "$(Build.ArtifactStagingDirectory)/test-output" - condition: always() - - script: | kubectl get pods -A echo "Waiting < 2 minutes for cilium to be ready" @@ -176,19 +156,6 @@ steps: name: "CiliumIdentities" displayName: "Verify Cilium Identities Deletion" - - script: | - echo "validate pod IP assignment before CNS restart" - kubectl get pod -owide -A - make test-validate-state - echo "restart CNS" - kubectl rollout restart ds azure-cns -n kube-system - kubectl rollout status ds azure-cns -n kube-system - kubectl get pod -owide -A - echo "validate pods after CNS restart" - make test-validate-state - name: "restartCNS" - displayName: "Restart CNS and validate pods" - - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh From d2bb281cf0691a377c8e65ab17e40acf8c790863 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:44:52 -0600 Subject: [PATCH 19/31] deps: bump mvdan.cc/gofumpt from 0.5.0 to 0.6.0 in /build/tools (#2556) Bumps [mvdan.cc/gofumpt](https://github.com/mvdan/gofumpt) from 0.5.0 to 0.6.0. - [Release notes](https://github.com/mvdan/gofumpt/releases) - [Changelog](https://github.com/mvdan/gofumpt/blob/master/CHANGELOG.md) - [Commits](https://github.com/mvdan/gofumpt/compare/v0.5.0...v0.6.0) --- updated-dependencies: - dependency-name: mvdan.cc/gofumpt dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build/tools/go.mod | 10 +++++----- build/tools/go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/build/tools/go.mod b/build/tools/go.mod index 46365747a0a..f9a8a06d452 100644 --- a/build/tools/go.mod +++ b/build/tools/go.mod @@ -10,7 +10,7 @@ require ( github.com/jstemmer/go-junit-report v1.0.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.32.0 - mvdan.cc/gofumpt v0.5.0 + mvdan.cc/gofumpt v0.6.0 sigs.k8s.io/controller-tools v0.13.0 ) @@ -190,11 +190,11 @@ require ( golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect golang.org/x/exp/typeparams v0.0.0-20231214170342-aacd6d4b4611 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.15.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.17.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/build/tools/go.sum b/build/tools/go.sum index c1f62534369..55bc04e7007 100644 --- a/build/tools/go.sum +++ b/build/tools/go.sum @@ -476,8 +476,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryancurrah/gomodguard v1.3.0 h1:q15RT/pd6UggBXVBuLps8BXRvl5GPBcwVA7BJHMLuTw= github.com/ryancurrah/gomodguard v1.3.0/go.mod h1:ggBxb3luypPEzqVtq33ee7YSN35V28XeGnid8dnni50= @@ -699,8 +699,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -720,8 +720,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -775,8 +775,8 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -863,8 +863,8 @@ golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -991,8 +991,8 @@ k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= k8s.io/utils v0.0.0-20231127182322-b307cd553661/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -mvdan.cc/gofumpt v0.5.0 h1:0EQ+Z56k8tXjj/6TQD25BFNKQXpCvT0rnansIc7Ug5E= -mvdan.cc/gofumpt v0.5.0/go.mod h1:HBeVDtMKRZpXyxFciAirzdKklDlGu8aAy1wEbH5Y9js= +mvdan.cc/gofumpt v0.6.0 h1:G3QvahNDmpD+Aek/bNOLrFR2XC6ZAdo62dZu65gmwGo= +mvdan.cc/gofumpt v0.6.0/go.mod h1:4L0wf+kgIPZtcCWXynNS2e6bhmj73umwnuXSZarixzA= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I= mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= From 01bebcf014e563280c309ecbdb10aaeef1cbe6c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 Jan 2024 15:56:40 -0600 Subject: [PATCH 20/31] deps: bump github.com/Azure/azure-container-networking from 1.5.19 to 1.5.20 in /azure-ipam (#2557) deps: bump github.com/Azure/azure-container-networking in /azure-ipam Bumps [github.com/Azure/azure-container-networking](https://github.com/Azure/azure-container-networking) from 1.5.19 to 1.5.20. - [Release notes](https://github.com/Azure/azure-container-networking/releases) - [Commits](https://github.com/Azure/azure-container-networking/compare/v1.5.19...v1.5.20) --- updated-dependencies: - dependency-name: github.com/Azure/azure-container-networking dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- azure-ipam/go.mod | 21 ++++++++++----------- azure-ipam/go.sum | 44 +++++++++++++++++++++----------------------- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/azure-ipam/go.mod b/azure-ipam/go.mod index e697aa8ad50..5d2c4040e11 100644 --- a/azure-ipam/go.mod +++ b/azure-ipam/go.mod @@ -3,7 +3,7 @@ module github.com/Azure/azure-container-networking/azure-ipam go 1.21 require ( - github.com/Azure/azure-container-networking v1.5.19 + github.com/Azure/azure-container-networking v1.5.20 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.4.0 github.com/pkg/errors v0.9.1 @@ -15,11 +15,11 @@ require ( require ( code.cloudfoundry.org/clock v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect - github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect @@ -43,23 +43,22 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/uuid v1.5.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/microsoft/ApplicationInsights-Go v0.4.4 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect - github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.18.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/common v0.46.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -67,8 +66,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.18.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/oauth2 v0.15.0 // indirect + golang.org/x/net v0.20.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -76,7 +75,7 @@ require ( golang.org/x/tools v0.16.1 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect - google.golang.org/grpc v1.60.1 // indirect + google.golang.org/grpc v1.61.0 // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -84,7 +83,7 @@ require ( k8s.io/api v0.29.0 // indirect k8s.io/apimachinery v0.29.0 // indirect k8s.io/client-go v0.29.0 // indirect - k8s.io/klog/v2 v2.120.0 // indirect + k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20231214164306-ab13479f8bf8 // indirect k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect sigs.k8s.io/controller-runtime v0.16.3 // indirect diff --git a/azure-ipam/go.sum b/azure-ipam/go.sum index 0167d9020cd..eafa082bfaf 100644 --- a/azure-ipam/go.sum +++ b/azure-ipam/go.sum @@ -2,20 +2,20 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= code.cloudfoundry.org/clock v1.1.0 h1:XLzC6W3Ah/Y7ht1rmZ6+QfPdt1iGWEAAtIZXgiaj57c= code.cloudfoundry.org/clock v1.1.0/go.mod h1:yA3fxddT9RINQL2XHS7PS+OXxKCGhfrZmlNUCIM6AKo= -github.com/Azure/azure-container-networking v1.5.19 h1:vdSUU0EjyUu5ePdJyiBT2bOUH24hFI6sB6eWocvfXr4= -github.com/Azure/azure-container-networking v1.5.19/go.mod h1:T0wq4BcGMX+S0ue3grruyej/C7GbhjN2B2ciHxqPNeg= +github.com/Azure/azure-container-networking v1.5.20 h1:wThjVxYSfSjIaoFMxMRxdvP6oNaO0cTUkU0LDnXXX0Q= +github.com/Azure/azure-container-networking v1.5.20/go.mod h1:Z9E/Z+eM0Pr8oxCzqJlhWFLyYML5VBGb+s4TNTKZRqQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0 h1:BMAjVKJM0U/CYF27gA0ZMmXGkOcvfFtD0oHVZ1TIPRI= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.4.0/go.mod h1:1fXstnBMas5kzG+S3q8UoJcmyU6nUeunJcMDHcRYHhs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1/go.mod h1:9V2j0jn9jDEkCkv8w/bKTNppX/d0FVA1ud77xCIP4KA= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0 h1:hVeq+yCyUi+MsoO/CU95yqCIcdzra5ovzk8Q2BBpV2M= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.0/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= @@ -129,8 +129,8 @@ github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= -github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -153,8 +153,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/microsoft/ApplicationInsights-Go v0.4.4 h1:G4+H9WNs6ygSCe6sUyxRc2U81TI5Es90b2t/MwX5KqY= github.com/microsoft/ApplicationInsights-Go v0.4.4/go.mod h1:fKRUseBqkw6bDiXTs3ESTiU/4YTIHsQS4W3fP2ieF4U= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -185,8 +183,8 @@ github.com/onsi/gomega v1.30.0 h1:hvMK7xYz4D3HapigLTeGdId/NcfQx1VHMJc60ew99+8= github.com/onsi/gomega v1.30.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= -github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -197,8 +195,8 @@ github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlk github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.46.0 h1:doXzt5ybi1HBKpsZOL0sSkaNHJJqkyfEWZGGqqScV0Y= +github.com/prometheus/common v0.46.0/go.mod h1:Tp0qkxpb9Jsg54QMe+EAmqXkSV7Evdy1BTn+g2pa/hQ= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -261,11 +259,11 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= -golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -288,10 +286,10 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= @@ -338,8 +336,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -382,8 +380,8 @@ k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= -k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231214164306-ab13479f8bf8 h1:yHNkNuLjht7iq95pO9QmbjOWCguvn8mDe3lT78nqPkw= k8s.io/kube-openapi v0.0.0-20231214164306-ab13479f8bf8/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20231127182322-b307cd553661 h1:FepOBzJ0GXm8t0su67ln2wAZjbQ6RxQGZDnzuLcrUTI= From 5778117938b135279033252df24d914a5803b96e Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Tue, 30 Jan 2024 14:03:56 -0800 Subject: [PATCH 21/31] ci: E2E Framework [Scenarios] [5/6] (#2527) * add scenarios * port-forward and stitch hubble scenario * run jobs in background * import stop to steps * add prometheus server test * validate drop metric * reorg scenarios * skip e2e in ut * test logging * pod name empty * wait cleanup --- .pipelines/networkobservability/e2e.yaml | 29 ++++ .pipelines/pipeline.yaml | 1 + Makefile | 6 +- .../framework/azure/create-cilium-cluster.go | 4 +- test/e2e/framework/azure/create-cluster.go | 4 +- test/e2e/framework/azure/create-rg.go | 2 +- test/e2e/framework/azure/create-vnet.go | 4 +- test/e2e/framework/azure/delete-cluster.go | 2 +- test/e2e/framework/azure/delete-rg.go | 10 +- test/e2e/framework/azure/enable-ama.go | 2 +- test/e2e/framework/azure/get-kubeconfig.go | 2 +- .../kubernetes/create-agnhost-statefulset.go | 7 +- .../kubernetes/create-kapinger-deployment.go | 2 +- .../kubernetes/create-network-policy.go | 6 +- test/e2e/framework/kubernetes/exec-pod.go | 2 +- .../{port-forward.txt => port-forward.go} | 63 ++++---- .../types/examples/background_test.go | 98 ++++++++++++ test/e2e/framework/types/job.go | 147 +++++++++++++----- test/e2e/framework/types/runner.go | 2 + test/e2e/framework/types/step.go | 27 +++- test/e2e/framework/types/step_sleep.go | 6 +- test/e2e/framework/types/step_stop.go | 30 ++++ .../scenarios/hubble/azuremonitor/scenario.go | 35 +++++ .../azuremonitor/validate-ama-targets.go | 67 ++++++++ test/e2e/scenarios/hubble/drop/scenario.go | 94 +++++++++++ .../hubble/drop/validate-drop-metric.go | 130 ++++++++++++++++ test/e2e/scenarios/hubble/flow/scenario.go | 16 ++ .../scenarios/hubble/flow/validate-metrics.go | 117 ++++++++++++++ .../scenarios/hubble/hubble_scenarios_test.go | 63 ++++++++ test/integration/portforward.go | 21 ++- 30 files changed, 891 insertions(+), 108 deletions(-) create mode 100644 .pipelines/networkobservability/e2e.yaml rename test/e2e/framework/kubernetes/{port-forward.txt => port-forward.go} (78%) create mode 100644 test/e2e/framework/types/examples/background_test.go create mode 100644 test/e2e/framework/types/step_stop.go create mode 100644 test/e2e/scenarios/hubble/azuremonitor/scenario.go create mode 100644 test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go create mode 100644 test/e2e/scenarios/hubble/drop/scenario.go create mode 100644 test/e2e/scenarios/hubble/drop/validate-drop-metric.go create mode 100644 test/e2e/scenarios/hubble/flow/scenario.go create mode 100644 test/e2e/scenarios/hubble/flow/validate-metrics.go create mode 100644 test/e2e/scenarios/hubble/hubble_scenarios_test.go diff --git a/.pipelines/networkobservability/e2e.yaml b/.pipelines/networkobservability/e2e.yaml new file mode 100644 index 00000000000..67406a8c50e --- /dev/null +++ b/.pipelines/networkobservability/e2e.yaml @@ -0,0 +1,29 @@ +pr: none + +stages: + - stage: Run_E2E + displayName: Run E2E + jobs: + - job: Run_E2E + displayName: Run Network Observability E2E + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + + steps: + - task: AzureCLI@2 + displayName: "Verify E2E Manifests" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + go test -v ./test/e2e/manifests/*.go + + - task: AzureCLI@2 + displayName: "Run Network Observability E2E" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + go test -v ./test/e2e/scenarios/hubble/*.go -timeout 30m diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 8603f749bbd..460ebd0384f 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -596,6 +596,7 @@ stages: - aks_swift_e2e - cilium_e2e - cilium_overlay_e2e + - cilium_h_overlay_e2e - aks_ubuntu_22_linux_e2e - aks_windows_22_e2e - dualstackoverlay_e2e diff --git a/Makefile b/Makefile index bdfe4c99474..dc85d2b2a20 100644 --- a/Makefile +++ b/Makefile @@ -775,13 +775,13 @@ CNI_TYPE ?= cilium test-all: ## run all unit tests. @$(eval COVER_FILTER=`go list --tags ignore_uncovered,ignore_autogenerated $(COVER_PKG)/... | tr '\n' ','`) @echo Test coverpkg: $(COVER_FILTER) - go test -mod=readonly -buildvcs=false -tags "unit" -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/... + go test -mod=readonly -buildvcs=false -tags "unit" --skip 'TestE2E*' -coverpkg=$(COVER_FILTER) -race -covermode atomic -coverprofile=coverage.out $(COVER_PKG)/... test-integration: ## run all integration tests. AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ CNI_VERSION=$(CNI_VERSION) \ CNS_VERSION=$(CNS_VERSION) \ - go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration ./test/integration... + go test -mod=readonly -buildvcs=false -timeout 1h -coverpkg=./... -race -covermode atomic -coverprofile=coverage.out -tags=integration --skip 'TestE2E*' ./test/integration... test-load: ## run all load tests AZURE_IPAM_VERSION=$(AZURE_IPAM_VERSION) \ @@ -790,7 +790,7 @@ test-load: ## run all load tests go test -timeout 30m -race -tags=load ./test/integration/load... -v test-validate-state: - cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load -run ^TestValidateState + cd test/integration/load && go test -mod=readonly -count=1 -timeout 30m -tags load --skip 'TestE2E*' -run ^TestValidateState cd ../../.. test-cyclonus: ## run the cyclonus test for npm. diff --git a/test/e2e/framework/azure/create-cilium-cluster.go b/test/e2e/framework/azure/create-cilium-cluster.go index 671a25945bb..a31b47081df 100644 --- a/test/e2e/framework/azure/create-cilium-cluster.go +++ b/test/e2e/framework/azure/create-cilium-cluster.go @@ -68,7 +68,7 @@ func (c *CreateBYOCiliumCluster) Prevalidate() error { return nil } -func (c *CreateBYOCiliumCluster) Postvalidate() error { +func (c *CreateBYOCiliumCluster) Stop() error { return nil } @@ -112,6 +112,8 @@ func (c *CreateBYOCiliumCluster) Run() error { return fmt.Errorf("failed to create az client: %w", err) } + log.Printf("when the cluster is ready, use the below command to access and debug") + log.Printf("az aks get-credentials --resource-group %s --name %s --subscription %s", c.ResourceGroupName, c.ClusterName, c.SubscriptionID) log.Printf("creating cluster \"%s\" in resource group \"%s\"...", c.ClusterName, c.ResourceGroupName) poller, err := clientFactory.NewManagedClustersClient().BeginCreateOrUpdate(ctx, c.ResourceGroupName, c.ClusterName, ciliumCluster, nil) diff --git a/test/e2e/framework/azure/create-cluster.go b/test/e2e/framework/azure/create-cluster.go index 54250bbabc6..f21a02ec991 100644 --- a/test/e2e/framework/azure/create-cluster.go +++ b/test/e2e/framework/azure/create-cluster.go @@ -13,7 +13,7 @@ import ( const ( MaxNumberOfNodes = 3 MaxPodsPerNode = 250 - AgentSKU = "Standard_D4s_v3" + AgentSKU = "Standard_DS4_v2" ) var defaultClusterCreateTimeout = 30 * time.Minute @@ -110,6 +110,6 @@ func (c *CreateCluster) Prevalidate() error { return nil } -func (c *CreateCluster) Postvalidate() error { +func (c *CreateCluster) Stop() error { return nil } diff --git a/test/e2e/framework/azure/create-rg.go b/test/e2e/framework/azure/create-rg.go index efa1d88ea55..190d67ec133 100644 --- a/test/e2e/framework/azure/create-rg.go +++ b/test/e2e/framework/azure/create-rg.go @@ -43,6 +43,6 @@ func (c *CreateResourceGroup) Prevalidate() error { return nil } -func (c *CreateResourceGroup) Postvalidate() error { +func (c *CreateResourceGroup) Stop() error { return nil } diff --git a/test/e2e/framework/azure/create-vnet.go b/test/e2e/framework/azure/create-vnet.go index 0149a14c8d0..40f414bc23c 100644 --- a/test/e2e/framework/azure/create-vnet.go +++ b/test/e2e/framework/azure/create-vnet.go @@ -59,7 +59,7 @@ func (c *CreateVNet) Prevalidate() error { return nil } -func (c *CreateVNet) Postvalidate() error { +func (c *CreateVNet) Stop() error { return nil } @@ -105,6 +105,6 @@ func (c *CreateSubnet) Prevalidate() error { return nil } -func (c *CreateSubnet) Postvalidate() error { +func (c *CreateSubnet) Stop() error { return nil } diff --git a/test/e2e/framework/azure/delete-cluster.go b/test/e2e/framework/azure/delete-cluster.go index 674a2fbb620..ebb7b11043b 100644 --- a/test/e2e/framework/azure/delete-cluster.go +++ b/test/e2e/framework/azure/delete-cluster.go @@ -43,6 +43,6 @@ func (d *DeleteCluster) Prevalidate() error { return nil } -func (d *DeleteCluster) Postvalidate() error { +func (d *DeleteCluster) Stop() error { return nil } diff --git a/test/e2e/framework/azure/delete-rg.go b/test/e2e/framework/azure/delete-rg.go index 0d65df6d5fe..9691f0fe4e6 100644 --- a/test/e2e/framework/azure/delete-rg.go +++ b/test/e2e/framework/azure/delete-rg.go @@ -27,18 +27,12 @@ func (d *DeleteResourceGroup) Run() error { if err != nil { return fmt.Errorf("failed to create resource group client: %w", err) } - forceDeleteType := "Microsoft.Compute/virtualMachines,Microsoft.Compute/virtualMachineScaleSets" - poller, err := clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) + _, err = clientFactory.NewResourceGroupsClient().BeginDelete(ctx, d.ResourceGroupName, &armresources.ResourceGroupsClientBeginDeleteOptions{ForceDeletionTypes: to.Ptr(forceDeleteType)}) if err != nil { return fmt.Errorf("failed to finish the delete resource group request: %w", err) } - _, err = poller.PollUntilDone(ctx, nil) - if err != nil { - return fmt.Errorf("failed to pull the result for delete resource group: %w", err) - } - log.Printf("resource group \"%s\" deleted successfully", d.ResourceGroupName) return nil } @@ -47,6 +41,6 @@ func (d *DeleteResourceGroup) Prevalidate() error { return nil } -func (d *DeleteResourceGroup) Postvalidate() error { +func (d *DeleteResourceGroup) Stop() error { return nil } diff --git a/test/e2e/framework/azure/enable-ama.go b/test/e2e/framework/azure/enable-ama.go index e6bdaf4e618..0df02e9e8db 100644 --- a/test/e2e/framework/azure/enable-ama.go +++ b/test/e2e/framework/azure/enable-ama.go @@ -112,6 +112,6 @@ func (c *CreateAzureMonitor) Prevalidate() error { return nil } -func (c *CreateAzureMonitor) Postvalidate() error { +func (c *CreateAzureMonitor) Stop() error { return nil } diff --git a/test/e2e/framework/azure/get-kubeconfig.go b/test/e2e/framework/azure/get-kubeconfig.go index 08cb8414036..f25cddd9b00 100644 --- a/test/e2e/framework/azure/get-kubeconfig.go +++ b/test/e2e/framework/azure/get-kubeconfig.go @@ -48,6 +48,6 @@ func (c *GetAKSKubeConfig) Prevalidate() error { return nil } -func (c *GetAKSKubeConfig) Postvalidate() error { +func (c *GetAKSKubeConfig) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go index 6be316c1447..b35649c9d7f 100644 --- a/test/e2e/framework/kubernetes/create-agnhost-statefulset.go +++ b/test/e2e/framework/kubernetes/create-agnhost-statefulset.go @@ -19,11 +19,6 @@ var ErrLabelMissingFromPod = fmt.Errorf("label missing from pod") const ( AgnhostHTTPPort = 80 AgnhostReplicas = 1 - - defaultTimeoutSeconds = 300 - defaultRetryDelay = 5 * time.Second - defaultRetryAttempts = 60 - defaultHTTPClientTimeout = 2 * time.Second ) type CreateAgnhostStatefulSet struct { @@ -71,7 +66,7 @@ func (c *CreateAgnhostStatefulSet) Prevalidate() error { return nil } -func (c *CreateAgnhostStatefulSet) Postvalidate() error { +func (c *CreateAgnhostStatefulSet) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-kapinger-deployment.go b/test/e2e/framework/kubernetes/create-kapinger-deployment.go index b65732d2ee8..4cab47068a8 100644 --- a/test/e2e/framework/kubernetes/create-kapinger-deployment.go +++ b/test/e2e/framework/kubernetes/create-kapinger-deployment.go @@ -70,7 +70,7 @@ func (c *CreateKapingerDeployment) Prevalidate() error { return nil } -func (c *CreateKapingerDeployment) Postvalidate() error { +func (c *CreateKapingerDeployment) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/create-network-policy.go b/test/e2e/framework/kubernetes/create-network-policy.go index fa1171a257e..2b67210c3ee 100644 --- a/test/e2e/framework/kubernetes/create-network-policy.go +++ b/test/e2e/framework/kubernetes/create-network-policy.go @@ -72,7 +72,7 @@ func (c *CreateDenyAllNetworkPolicy) Prevalidate() error { return nil } -func (c *CreateDenyAllNetworkPolicy) Postvalidate() error { +func (c *CreateDenyAllNetworkPolicy) Stop() error { return nil } @@ -108,7 +108,3 @@ func (d *DeleteDenyAllNetworkPolicy) Run() error { func (d *DeleteDenyAllNetworkPolicy) Prevalidate() error { return nil } - -func (d *DeleteDenyAllNetworkPolicy) Postvalidate() error { - return nil -} diff --git a/test/e2e/framework/kubernetes/exec-pod.go b/test/e2e/framework/kubernetes/exec-pod.go index aaf71c49459..25fa6be12c5 100644 --- a/test/e2e/framework/kubernetes/exec-pod.go +++ b/test/e2e/framework/kubernetes/exec-pod.go @@ -39,7 +39,7 @@ func (e *ExecInPod) Prevalidate() error { return nil } -func (e *ExecInPod) Postvalidate() error { +func (e *ExecInPod) Stop() error { return nil } diff --git a/test/e2e/framework/kubernetes/port-forward.txt b/test/e2e/framework/kubernetes/port-forward.go similarity index 78% rename from test/e2e/framework/kubernetes/port-forward.txt rename to test/e2e/framework/kubernetes/port-forward.go index 7dcabe496da..05a5f2afe7f 100644 --- a/test/e2e/framework/kubernetes/port-forward.txt +++ b/test/e2e/framework/kubernetes/port-forward.go @@ -39,8 +39,7 @@ type PortForward struct { OptionalLabelAffinity string // local properties - pf *k8s.PortForwarder - portForwardHandle k8s.PortForwardStreamHandle + pf *k8s.PortForwarder } func (p *PortForward) Run() error { @@ -61,15 +60,11 @@ func (p *PortForward) Run() error { return fmt.Errorf("could not create clientset: %w", err) } - p.pf, err = k8s.NewPortForwarder(config) - if err != nil { - return fmt.Errorf("could not create port forwarder: %w", err) - } - // if we have an optional label affinity, find a pod with that label, on the same node as a pod with the label selector targetPodName := "" if p.OptionalLabelAffinity != "" { // get all pods with label + log.Printf("attempting to find pod with label \"%s\", on a node with a pod with label \"%s\"\n", p.LabelSelector, p.OptionalLabelAffinity) targetPodName, err = p.findPodsWithAffinity(pctx, clientset) if err != nil { return fmt.Errorf("could not find pod with affinity: %w", err) @@ -77,37 +72,43 @@ func (p *PortForward) Run() error { } portForwardFn := func() error { - log.Printf("attempting port forward to a pod with label \"%s\", in namespace \"%s\"...\n", p.LabelSelector, p.Namespace) - - var handle k8s.PortForwardStreamHandle // if we have a pod name (likely from affinity above), use it, otherwise use label selector + opts := k8s.PortForwardingOpts{ + Namespace: p.Namespace, + PodName: targetPodName, + LocalPort: lport, + DestPort: rport, + } + if targetPodName != "" { - handle, err = p.pf.ForwardWithPodName(pctx, p.Namespace, targetPodName, lport, rport) - if err != nil { - return fmt.Errorf("could not start port forward: %w", err) - } - } else { - handle, err = p.pf.ForwardWithLabelSelector(pctx, p.Namespace, p.LabelSelector, lport, rport) - if err != nil { - return fmt.Errorf("could not start port forward: %w", err) - } + opts.PodName = targetPodName + } + + log.Printf("attempting port forward to pod name \"%s\" with label \"%s\", in namespace \"%s\"...\n", targetPodName, p.LabelSelector, p.Namespace) + + p.pf, err = k8s.NewPortForwarder(config, &logger{}, opts) + if err != nil { + return fmt.Errorf("could not create port forwarder: %w", err) + } + err = p.pf.Forward(pctx) + if err != nil { + return fmt.Errorf("could not start port forward: %w", err) } // verify port forward succeeded client := http.Client{ Timeout: defaultHTTPClientTimeout, } - resp, err := client.Get(handle.URL()) //nolint + resp, err := client.Get(p.pf.Address()) //nolint if err != nil { - log.Printf("port forward validation HTTP request to %s failed: %v\n", handle.URL(), err) - handle.Stop() - return fmt.Errorf("port forward validation HTTP request to %s failed: %w", handle.URL(), err) + log.Printf("port forward validation HTTP request to %s failed: %v\n", p.pf.Address(), err) + p.pf.Stop() + return fmt.Errorf("port forward validation HTTP request to %s failed: %w", p.pf.Address(), err) } defer resp.Body.Close() - log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", handle.URL(), resp.Status) - p.portForwardHandle = handle + log.Printf("port forward validation HTTP request to \"%s\" succeeded, response: %s\n", p.pf.Address(), resp.Status) return nil } @@ -115,7 +116,7 @@ func (p *PortForward) Run() error { if err = defaultRetrier.Do(portForwardCtx, portForwardFn); err != nil { return fmt.Errorf("could not start port forward within %ds: %w", defaultTimeoutSeconds, err) } - log.Printf("successfully port forwarded to \"%s\"\n", p.portForwardHandle.URL()) + log.Printf("successfully port forwarded to \"%s\"\n", p.pf.Address()) return nil } @@ -157,7 +158,13 @@ func (p *PortForward) Prevalidate() error { return nil } -func (p *PortForward) Postvalidate() error { - p.portForwardHandle.Stop() +func (p *PortForward) Stop() error { + p.pf.Stop() return nil } + +type logger struct{} + +func (l *logger) Logf(format string, args ...interface{}) { + log.Printf(format, args...) +} diff --git a/test/e2e/framework/types/examples/background_test.go b/test/e2e/framework/types/examples/background_test.go new file mode 100644 index 00000000000..b1c63e07c03 --- /dev/null +++ b/test/e2e/framework/types/examples/background_test.go @@ -0,0 +1,98 @@ +package types + +import ( + "fmt" + "log" + "sync" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +func TestFramework(t *testing.T) { + job := types.NewJob("Validate that drop metrics are present in the prometheus endpoint") + runner := types.NewRunner(t, job) + defer runner.Run() + + job.AddStep(&TestBackground{ + CounterName: "Example Counter", + }, &types.StepOptions{ + ExpectError: false, + RunInBackgroundWithID: "TestStep", + }) + + job.AddStep(&types.Sleep{ + Duration: 1 * time.Second, + }, nil) + + job.AddStep(&types.Stop{ + BackgroundID: "TestStep", + }, nil) +} + +type TestBackground struct { + CounterName string + c *counter +} + +func (t *TestBackground) Run() error { + t.c = newCounter() + err := t.c.Start() + if err != nil { + return fmt.Errorf("failed to start counter: %w", err) + } + log.Println("running counter: " + t.CounterName) + return nil +} + +func (t *TestBackground) Stop() error { + log.Println("stopping counter: " + t.CounterName) + err := t.c.Stop() + if err != nil { + return fmt.Errorf("failed to stop counter: %w", err) + } + log.Println("count:", t.c.count) + return nil +} + +func (t *TestBackground) Prevalidate() error { + return nil +} + +type counter struct { + ticker *time.Ticker + count int + ch chan struct{} + wg sync.WaitGroup +} + +func newCounter() *counter { + return &counter{ + ch: make(chan struct{}), + } +} + +func (c *counter) Start() error { + c.ticker = time.NewTicker(1 * time.Millisecond) + c.wg.Add(1) + go func() { + for { + select { + case <-c.ticker.C: + c.count++ + case <-c.ch: + c.wg.Done() + return + } + } + }() + + return nil +} + +func (c *counter) Stop() error { + close(c.ch) + c.wg.Wait() + return nil +} diff --git a/test/e2e/framework/types/job.go b/test/e2e/framework/types/job.go index eef5bb0b789..edd10fc75b3 100644 --- a/test/e2e/framework/types/job.go +++ b/test/e2e/framework/types/job.go @@ -12,19 +12,31 @@ var ( ErrNilError = fmt.Errorf("expected error to be nil") ErrMissingParameter = fmt.Errorf("missing parameter") ErrParameterAlreadySet = fmt.Errorf("parameter already set") + ErrOrphanSteps = fmt.Errorf("background steps with no corresponding stop") + ErrCannotStopStep = fmt.Errorf("cannot stop step") + ErrMissingBackroundID = fmt.Errorf("missing background id") ) +// A Job is a logical grouping of steps, options and values type Job struct { - Values *JobValues - Description string - Steps []*StepWrapper + Values *JobValues + Description string + Steps []*StepWrapper + BackgroundSteps map[string]*StepWrapper } +// A StepWrapper is a coupling of a step and it's options type StepWrapper struct { Step Step Opts *StepOptions } +// A Scenario is a logical grouping of steps, used to describe a scenario such as "test drop metrics" +// which will require port forwarding, exec'ing, scraping, etc. +type Scenario struct { + Steps []*StepWrapper +} + func responseDivider(jobname string) { totalWidth := 100 start := 20 @@ -45,21 +57,23 @@ func NewJob(description string) *Job { Values: &JobValues{ kv: make(map[string]string), }, - Description: description, + BackgroundSteps: make(map[string]*StepWrapper), + Description: description, } } -func (j *Job) AddScenario(steps ...StepWrapper) { - for _, step := range steps { +func (j *Job) AddScenario(scenario *Scenario) { + for _, step := range scenario.Steps { j.AddStep(step.Step, step.Opts) } } func (j *Job) AddStep(step Step, opts *StepOptions) { - j.Steps = append(j.Steps, &StepWrapper{ + stepw := &StepWrapper{ Step: step, Opts: opts, - }) + } + j.Steps = append(j.Steps, stepw) } func (j *Job) Run() error { @@ -67,6 +81,7 @@ func (j *Job) Run() error { return ErrEmptyDescription } + // validate all steps in the job, making sure parameters are set/validated etc. err := j.Validate() if err != nil { return err // nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself @@ -81,7 +96,6 @@ func (j *Job) Run() error { for _, wrapper := range j.Steps { responseDivider(reflect.TypeOf(wrapper.Step).Elem().Name()) - log.Printf("INFO: step options provided: %+v\n", wrapper.Opts) err := wrapper.Step.Run() if wrapper.Opts.ExpectError && err == nil { return fmt.Errorf("expected error from step %s but got nil: %w", reflect.TypeOf(wrapper.Step).Elem().Name(), ErrNilError) @@ -90,21 +104,66 @@ func (j *Job) Run() error { } } - for _, wrapper := range j.Steps { - err := wrapper.Step.Postvalidate() - if err != nil { - return err //nolint:wrapcheck // don't wrap error, wouldn't provide any more context than the error itself - } - } return nil } func (j *Job) Validate() error { + // ensure that there are no background steps left after running + for _, wrapper := range j.Steps { err := j.validateStep(wrapper) if err != nil { return err } + + } + + err := j.validateBackgroundSteps() + if err != nil { + return err + } + + return nil +} + +func (j *Job) validateBackgroundSteps() error { + stoppedBackgroundSteps := make(map[string]bool) + + for _, stepw := range j.Steps { + switch s := stepw.Step.(type) { + case *Stop: + if s.BackgroundID == "" { + return fmt.Errorf("cannot stop step with empty background id; %w", ErrMissingBackroundID) + } + + if j.BackgroundSteps[s.BackgroundID] == nil { + return fmt.Errorf("cannot stop step %s, as it won't be started by this time; %w", s.BackgroundID, ErrCannotStopStep) + } + if stopped := stoppedBackgroundSteps[s.BackgroundID]; stopped { + return fmt.Errorf("cannot stop step %s, as it has already been stopped; %w", s.BackgroundID, ErrCannotStopStep) + } + + // track for later on if the stop step is called + stoppedBackgroundSteps[s.BackgroundID] = true + + // set the stop step within the step + s.Step = j.BackgroundSteps[s.BackgroundID].Step + + default: + if stepw.Opts.RunInBackgroundWithID != "" { + if _, exists := j.BackgroundSteps[stepw.Opts.RunInBackgroundWithID]; exists { + log.Fatalf("step with id %s already exists", stepw.Opts.RunInBackgroundWithID) + } + j.BackgroundSteps[stepw.Opts.RunInBackgroundWithID] = stepw + stoppedBackgroundSteps[stepw.Opts.RunInBackgroundWithID] = false + } + } + } + + for stepName, stopped := range stoppedBackgroundSteps { + if !stopped { + return fmt.Errorf("step %s was not stopped; %w", stepName, ErrOrphanSteps) + } } return nil @@ -119,41 +178,53 @@ func (j *Job) validateStep(stepw *StepWrapper) error { stepw.Opts = &DefaultOpts } - for i, f := range reflect.VisibleFields(val.Type()) { + switch stepw.Step.(type) { + case *Stop: + // don't validate stop steps + return nil - // skip saving unexported fields - if !f.IsExported() { - continue - } + case *Sleep: + // don't validate sleep steps + return nil - k := reflect.Indirect(val.Field(i)).Kind() + default: + for i, f := range reflect.VisibleFields(val.Type()) { - if k == reflect.String { - parameter := val.Type().Field(i).Name - value := val.Field(i).Interface().(string) - storedValue := j.Values.Get(parameter) + // skip saving unexported fields + if !f.IsExported() { + continue + } - if storedValue == "" { - if value != "" { - if stepw.Opts.SaveParametersToJob { - fmt.Printf("%s setting parameter %s in job context to %s\n", stepName, parameter, value) + k := reflect.Indirect(val.Field(i)).Kind() + + if k == reflect.String { + parameter := val.Type().Field(i).Name + value := val.Field(i).Interface().(string) + storedValue := j.Values.Get(parameter) + + if storedValue == "" { + + switch { + case stepw.Opts.SkipSavingParamatersToJob: + continue + case value != "": + fmt.Printf("\"%s\" setting parameter \"%s\" in job context to \"%s\"\n", stepName, parameter, value) j.Values.Set(parameter, value) + default: + return fmt.Errorf("missing parameter \"%s\" for step \"%s\": %w", parameter, stepName, ErrMissingParameter) } continue } - return fmt.Errorf("missing parameter %s for step %s: %w", parameter, stepName, ErrMissingParameter) - } + if value != "" { + return fmt.Errorf("parameter %s for step %s is already set from previous step: %w", parameter, stepName, ErrParameterAlreadySet) + } - if value != "" { - return fmt.Errorf("parameter %s for step %s is already set from previous step: %w", parameter, stepName, ErrParameterAlreadySet) + // don't use log format since this is technically preexecution and easier to read + fmt.Println(stepName, "using previously stored value for parameter", parameter, "set as", j.Values.Get(parameter)) + val.Field(i).SetString(storedValue) } - - // don't use log format since this is technically preexecution and easier to read - fmt.Println(stepName, "using previously stored value for parameter", parameter, "set as", j.Values.Get(parameter)) - val.Field(i).SetString(storedValue) } } - return nil } diff --git a/test/e2e/framework/types/runner.go b/test/e2e/framework/types/runner.go index 76cc3984fc1..835fe05f32e 100644 --- a/test/e2e/framework/types/runner.go +++ b/test/e2e/framework/types/runner.go @@ -6,6 +6,8 @@ import ( "github.com/stretchr/testify/require" ) +// A wrapper around a job, so that internal job components don't require things like *testing.T +// and can be reused elsewhere type Runner struct { t *testing.T Job *Job diff --git a/test/e2e/framework/types/step.go b/test/e2e/framework/types/step.go index 096de789700..dd14f2bcd1e 100644 --- a/test/e2e/framework/types/step.go +++ b/test/e2e/framework/types/step.go @@ -1,14 +1,28 @@ package types var DefaultOpts = StepOptions{ - ExpectError: false, - SaveParametersToJob: true, + // when wanting to expect an error, set to true + ExpectError: false, + + // when wanting to avoid saving the parameters to the job, + // such as a repetetive task where step is used multiple times sequentially, + // but parameters are different each time + SkipSavingParamatersToJob: false, } type Step interface { + // Useful when wanting to do parameter checking, for example + // if a parameter length is known to be required less than 80 characters, + // do this here so we don't find out later on when we run the step + // when possible, try to avoid making external calls, this should be fast and simple Prevalidate() error + + // Primary step where test logic is executed + // Returning an error will cause the test to fail Run() error - Postvalidate() error + + // Require for background steps + Stop() error } type StepOptions struct { @@ -18,5 +32,10 @@ type StepOptions struct { // a step, but you don't want to save the parameters // ex: Sleep for 15 seconds, then Sleep for 10 seconds, // you don't want to save the parameters - SaveParametersToJob bool + SkipSavingParamatersToJob bool + + // Will save this step to the job's steps + // and then later on when Stop is called with job name, + // it will call Stop() on the step + RunInBackgroundWithID string } diff --git a/test/e2e/framework/types/step_sleep.go b/test/e2e/framework/types/step_sleep.go index b65f7bfeafd..2c89d1deb9e 100644 --- a/test/e2e/framework/types/step_sleep.go +++ b/test/e2e/framework/types/step_sleep.go @@ -10,15 +10,15 @@ type Sleep struct { } func (c *Sleep) Run() error { - log.Printf("sleeping for %s...\n", c.Duration) + log.Printf("sleeping for %s...\n", c.Duration.String()) time.Sleep(c.Duration) return nil } -func (c *Sleep) Prevalidate() error { +func (c *Sleep) Stop() error { return nil } -func (c *Sleep) Postvalidate() error { +func (c *Sleep) Prevalidate() error { return nil } diff --git a/test/e2e/framework/types/step_stop.go b/test/e2e/framework/types/step_stop.go new file mode 100644 index 00000000000..226fd8b2f5e --- /dev/null +++ b/test/e2e/framework/types/step_stop.go @@ -0,0 +1,30 @@ +package types + +import ( + "fmt" + "log" + "reflect" +) + +type Stop struct { + BackgroundID string + Step Step +} + +func (c *Stop) Run() error { + stepName := reflect.TypeOf(c.Step).Elem().Name() + log.Println("stopping step:", stepName) + err := c.Step.Stop() + if err != nil { + return fmt.Errorf("failed to stop step: %s with err %w", stepName, err) + } + return nil +} + +func (c *Stop) Stop() error { + return nil +} + +func (c *Stop) Prevalidate() error { + return nil +} diff --git a/test/e2e/scenarios/hubble/azuremonitor/scenario.go b/test/e2e/scenarios/hubble/azuremonitor/scenario.go new file mode 100644 index 00000000000..889a3ed217b --- /dev/null +++ b/test/e2e/scenarios/hubble/azuremonitor/scenario.go @@ -0,0 +1,35 @@ +package azuremonitor + +import ( + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +// todo: once AMA is rolled out +func ValidateAMATargets() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "validate-ama-targets", + }, + }, + { + Step: &VerifyPrometheusMetrics{ + Address: "http://localhost:9090", + }, + }, + { + Step: &types.Stop{ + BackgroundID: "validate-ama-targets", + }, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go b/test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go new file mode 100644 index 00000000000..42af2785980 --- /dev/null +++ b/test/e2e/scenarios/hubble/azuremonitor/validate-ama-targets.go @@ -0,0 +1,67 @@ +package azuremonitor + +import ( + "context" + "fmt" + "log" + + "github.com/prometheus/client_golang/api" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" +) + +const promHubbleJob = "hubble-pods" + +var ( + ErrHubbleTargetNotUp = fmt.Errorf("hubble target not up") + ErrNoActiveTargets = fmt.Errorf("no active targets found") +) + +type VerifyPrometheusMetrics struct { + Address string +} + +func (v *VerifyPrometheusMetrics) Run() error { + client, err := api.NewClient(api.Config{ + Address: v.Address, + }) + if err != nil { + return fmt.Errorf("failed to create prometheus client: %w", err) + } + + promapi := promv1.NewAPI(client) + ctx := context.Background() + targets, err := promapi.Targets(ctx) + if err != nil { + return fmt.Errorf("failed to get targets: %w", err) + } + + if len(targets.Active) == 0 { + return fmt.Errorf("no active targets found: %w", ErrNoActiveTargets) + } + + validTarget := &promv1.ActiveTarget{ + ScrapePool: promHubbleJob, + Health: "up", + } + + for i := range targets.Active { + target := &targets.Active[i] + if target.ScrapePool == validTarget.ScrapePool { + if target.Health != validTarget.Health { + return ErrHubbleTargetNotUp + } + break + } + } + + log.Printf("Verified Hubble Prometheus targets are up") + return nil +} + +func (v *VerifyPrometheusMetrics) Prevalidate() error { + return nil +} + +func (v *VerifyPrometheusMetrics) Stop() error { + return nil +} diff --git a/test/e2e/scenarios/hubble/drop/scenario.go b/test/e2e/scenarios/hubble/drop/scenario.go new file mode 100644 index 00000000000..3efd78af04e --- /dev/null +++ b/test/e2e/scenarios/hubble/drop/scenario.go @@ -0,0 +1,94 @@ +package drop + +import ( + k8s "github.com/Azure/azure-container-networking/test/e2e/framework/kubernetes" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" +) + +const ( + // Hubble drop reasons + UnsupportedL3Protocol = "UNSUPPORTED_L3_PROTOCOL" + PolicyDenied = "POLICY_DENIED" + + // L4 protocols + TCP = "TCP" + UDP = "UDP" +) + +func ValidateDropMetric() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &k8s.CreateKapingerDeployment{ + KapingerNamespace: "kube-system", + KapingerReplicas: "1", + }, + }, + { + Step: &k8s.CreateDenyAllNetworkPolicy{ + NetworkPolicyNamespace: "kube-system", + DenyAllLabelSelector: "app=agnhost-a", + }, + }, + { + Step: &k8s.CreateAgnhostStatefulSet{ + AgnhostName: "agnhost-a", + AgnhostNamespace: "kube-system", + }, + }, + { + Step: &k8s.ExecInPod{ + PodName: "agnhost-a-0", + PodNamespace: "kube-system", + Command: "curl -s -m 5 bing.com", + }, + Opts: &types.StepOptions{ + ExpectError: true, + SkipSavingParamatersToJob: true, + }, + }, + { + Step: &types.Sleep{ + Duration: sleepDelay, + }, + }, + // run curl again + { + Step: &k8s.ExecInPod{ + PodName: "agnhost-a-0", + PodNamespace: "kube-system", + Command: "curl -s -m 5 bing.com", + }, + Opts: &types.StepOptions{ + ExpectError: true, + SkipSavingParamatersToJob: true, + }, + }, + { + Step: &k8s.PortForward{ + Namespace: "kube-system", + LabelSelector: "k8s-app=cilium", + LocalPort: "9965", + RemotePort: "9965", + OptionalLabelAffinity: "app=agnhost-a", // port forward to a pod on a node that also has this pod with this label, assuming same namespace + }, + Opts: &types.StepOptions{ + RunInBackgroundWithID: "hubble-drop-port-forward", + }, + }, + { + Step: &ValidateHubbleDropMetric{ + PortForwardedHubblePort: "9965", + Source: "agnhost-a", + Reason: PolicyDenied, + Protocol: UDP, + }, + }, + { + Step: &types.Stop{ + BackgroundID: "hubble-drop-port-forward", + }, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/drop/validate-drop-metric.go b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go new file mode 100644 index 00000000000..e6d7aa66488 --- /dev/null +++ b/test/e2e/scenarios/hubble/drop/validate-drop-metric.go @@ -0,0 +1,130 @@ +package drop + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "reflect" + "time" + + "github.com/Azure/azure-container-networking/test/internal/retry" + promclient "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" +) + +var ErrNoMetricFound = fmt.Errorf("no metric found") + +const ( + destinationKey = "destination" + sourceKey = "source" + protcolKey = "protocol" + reason = "reason" + + sleepDelay = 5 * time.Second + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second + defaultRetryAttempts = 60 +) + +type ValidateHubbleDropMetric struct { + PortForwardedHubblePort string // presumably port-forwarded to a cilium pod + Source string + Protocol string + Reason string +} + +func (v *ValidateHubbleDropMetric) Run() error { + defaultRetrier := retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} + + promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.PortForwardedHubblePort) + ctx := context.Background() + pctx, cancel := context.WithCancel(ctx) + defer cancel() + + validMetric := map[string]string{ + destinationKey: "", + sourceKey: v.Source, + protcolKey: v.Protocol, + reason: v.Reason, + } + + metrics := map[string]*promclient.MetricFamily{} + scrapeMetricsFn := func() error { + log.Printf("checking for drop metrics on %s", promAddress) + var err error + metrics, err = getPrometheusDropMetrics(promAddress) + if err != nil { + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeout, err) + } + + err = verifyLabelsPresent(metrics, validMetric) + if err != nil { + return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) + } + + return nil + } + + err := defaultRetrier.Do(pctx, scrapeMetricsFn) + if err != nil { + return fmt.Errorf("failed to get prometheus metrics: %w", err) + } + + log.Printf("found metric matching %+v\n", validMetric) + return nil +} + +func verifyLabelsPresent(data map[string]*promclient.MetricFamily, validMetric map[string]string) error { + for _, metric := range data { + if metric.GetName() == "hubble_drop_total" { + for _, metric := range metric.GetMetric() { + + // get all labels and values on the metric + metricLabels := map[string]string{} + for _, label := range metric.GetLabel() { + metricLabels[label.GetName()] = label.GetValue() + } + if reflect.DeepEqual(metricLabels, validMetric) { + return nil + } + } + } + } + + return fmt.Errorf("failed to find metric matching %+v: %w", validMetric, ErrNoMetricFound) +} + +func getPrometheusDropMetrics(url string) (map[string]*promclient.MetricFamily, error) { + client := http.Client{} + resp, err := client.Get(url) //nolint + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic + } + + metrics, err := parseReaderPrometheusMetrics(resp.Body) + if err != nil { + return nil, err + } + + return metrics, nil +} + +func parseReaderPrometheusMetrics(input io.Reader) (map[string]*promclient.MetricFamily, error) { + var parser expfmt.TextParser + return parser.TextToMetricFamilies(input) //nolint +} + +func (v *ValidateHubbleDropMetric) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleDropMetric) Stop() error { + return nil +} diff --git a/test/e2e/scenarios/hubble/flow/scenario.go b/test/e2e/scenarios/hubble/flow/scenario.go new file mode 100644 index 00000000000..b89a056ced5 --- /dev/null +++ b/test/e2e/scenarios/hubble/flow/scenario.go @@ -0,0 +1,16 @@ +package flow + +import "github.com/Azure/azure-container-networking/test/e2e/framework/types" + +// todo: once AMA is rolled out +func ValidateAMATargets() *types.Scenario { + return &types.Scenario{ + Steps: []*types.StepWrapper{ + { + Step: &ValidateHubbleFlowMetric{ + LocalPort: "9090", + }, + }, + }, + } +} diff --git a/test/e2e/scenarios/hubble/flow/validate-metrics.go b/test/e2e/scenarios/hubble/flow/validate-metrics.go new file mode 100644 index 00000000000..4c28c84bbe6 --- /dev/null +++ b/test/e2e/scenarios/hubble/flow/validate-metrics.go @@ -0,0 +1,117 @@ +package flow + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "strings" + "time" + + "github.com/Azure/azure-container-networking/test/internal/retry" +) + +const ( + defaultRetryAttempts = 20 + defaultTimeout = 300 * time.Second + defaultRetryDelay = 5 * time.Second +) + +var requiredMetrics = []string{ + "hubble_flows_processed_total", + "hubble_tcp_flags_total", +} + +type ValidateHubbleFlowMetric struct { + LocalPort string +} + +func (v *ValidateHubbleFlowMetric) Run() error { + defaultRetrier := retry.Retrier{Attempts: defaultRetryAttempts, Delay: defaultRetryDelay} + promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.LocalPort) + log.Printf("require all metrics to be present: %+v\n", requiredMetrics) + ctx := context.Background() + var metrics map[string]struct{} + scrapeMetricsFn := func() error { + log.Printf("attempting scrape metrics on %s", promAddress) + + var err error + metrics, err = getPrometheusMetrics(promAddress) + if err != nil { + return fmt.Errorf("failed to get prometheus metrics: %w", err) + } + return nil + } + + portForwardCtx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + if err := defaultRetrier.Do(portForwardCtx, scrapeMetricsFn); err != nil { + return fmt.Errorf("could not start port forward within %ds: %w ", defaultTimeout, err) + } + + for _, reqMetric := range requiredMetrics { + if _, exists := metrics[reqMetric]; !exists { + return fmt.Errorf("scraping %s, did not find metric %s: ", promAddress, reqMetric) //nolint:goerr113,gocritic + } + log.Printf("found metric %s\n", reqMetric) + } + + log.Printf("all metrics validated: %+v\n", requiredMetrics) + return nil +} + +func (v *ValidateHubbleFlowMetric) Prevalidate() error { + return nil +} + +func (v *ValidateHubbleFlowMetric) Stop() error { + return nil +} + +func getPrometheusMetrics(url string) (map[string]struct{}, error) { + client := http.Client{} + resp, err := client.Get(url) //nolint + if err != nil { + return nil, fmt.Errorf("HTTP request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("HTTP request failed with status: %v", resp.Status) //nolint:goerr113,gocritic + } + + metricsData, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("reading HTTP response body failed: %w", err) + } + + metrics := parseMetrics(string(metricsData)) + return metrics, nil +} + +func parseMetrics(metricsData string) map[string]struct{} { + // Create a map to store the strings before the first '{'. + metrics := make(map[string]struct{}) + + // sample metrics + // hubble_tcp_flags_total{destination="",family="IPv4",flag="RST",source="kube-system/metrics-server"} 980 + // hubble_tcp_flags_total{destination="",family="IPv4",flag="SYN",source="kube-system/ama-metrics"} 1777 + // we only want the metric name for the time being + // label order/parseing can happen later + lines := strings.Split(metricsData, "\n") + // Iterate through each line. + for _, line := range lines { + // Find the index of the first '{' character. + index := strings.Index(line, "{") + if index >= 0 { + // Extract the string before the first '{'. + str := strings.TrimSpace(line[:index]) + // Store the string in the map. + metrics[str] = struct{}{} + } + } + + return metrics +} diff --git a/test/e2e/scenarios/hubble/hubble_scenarios_test.go b/test/e2e/scenarios/hubble/hubble_scenarios_test.go new file mode 100644 index 00000000000..cea8e0f8dec --- /dev/null +++ b/test/e2e/scenarios/hubble/hubble_scenarios_test.go @@ -0,0 +1,63 @@ +package hubble + +import ( + "os" + "os/user" + "strconv" + "testing" + "time" + + "github.com/Azure/azure-container-networking/test/e2e/framework/azure" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" + "github.com/Azure/azure-container-networking/test/e2e/scenarios/hubble/drop" +) + +const ( + // netObsRGtag is used to tag resources created by this test suite + netObsRGtag = "-e2e-netobs-" +) + +// Test against a BYO cluster with Cilium and Hubble enabled, +// create a pod with a deny all network policy and validate +// that the drop metrics are present in the prometheus endpoint +func TestE2EDropHubbleMetrics(t *testing.T) { + job := types.NewJob("Validate that drop metrics are present in the prometheus endpoint") + runner := types.NewRunner(t, job) + defer runner.Run() + + curuser, _ := user.Current() + + testName := curuser.Username + netObsRGtag + strconv.FormatInt(time.Now().Unix(), 10) + sub := os.Getenv("AZURE_SUBSCRIPTION_ID") + + job.AddStep(&azure.CreateResourceGroup{ + SubscriptionID: sub, + ResourceGroupName: testName, + Location: "eastus", + }, nil) + + job.AddStep(&azure.CreateVNet{ + VnetName: "testvnet", + VnetAddressSpace: "10.0.0.0/9", + }, nil) + + job.AddStep(&azure.CreateSubnet{ + SubnetName: "testsubnet", + SubnetAddressSpace: "10.0.0.0/12", + }, nil) + + job.AddStep(&azure.CreateBYOCiliumCluster{ + ClusterName: testName, + PodCidr: "10.128.0.0/9", + DNSServiceIP: "192.168.0.10", + ServiceCidr: "192.168.0.0/28", + }, nil) + + job.AddStep(&azure.GetAKSKubeConfig{ + KubeConfigFilePath: "./test.pem", + }, nil) + + job.AddScenario(drop.ValidateDropMetric()) + + job.AddStep(&azure.DeleteResourceGroup{}, nil) +} diff --git a/test/integration/portforward.go b/test/integration/portforward.go index 5170b91dc79..4765b9a715e 100644 --- a/test/integration/portforward.go +++ b/test/integration/portforward.go @@ -39,6 +39,7 @@ type PortForwarder struct { type PortForwardingOpts struct { Namespace string LabelSelector string + PodName string LocalPort int DestPort int } @@ -71,7 +72,25 @@ func NewPortForwarder(restConfig *rest.Config, logger logger, opts PortForwardin // An error is returned if a port forwarding session could not be started. If no error is returned, the // Address method can be used to communicate with the pod, and the Stop and KeepAlive methods can be used // to manage the lifetime of the port forwarding session. + func (p *PortForwarder) Forward(ctx context.Context) error { + var podName string + if p.opts.PodName == "" { + pods, err := p.clientset.CoreV1().Pods(p.opts.Namespace).List(ctx, metav1.ListOptions{LabelSelector: p.opts.LabelSelector, FieldSelector: "status.phase=Running"}) + if err != nil { + return fmt.Errorf("could not list pods in %q with label %q: %w", p.opts.Namespace, p.opts.LabelSelector, err) + } + + if len(pods.Items) < 1 { + return fmt.Errorf("no pods found in %q with label %q", p.opts.Namespace, p.opts.LabelSelector) //nolint:goerr113 //no specific handling expected + } + + randomIndex := rand.Intn(len(pods.Items)) //nolint:gosec //this is going to be revised in the future anyways, avoid random pods + podName = pods.Items[randomIndex].Name + } else { + podName = p.opts.PodName + } + pods, err := p.clientset.CoreV1().Pods(p.opts.Namespace).List(ctx, metav1.ListOptions{LabelSelector: p.opts.LabelSelector, FieldSelector: "status.phase=Running"}) if err != nil { return fmt.Errorf("could not list pods in %q with label %q: %w", p.opts.Namespace, p.opts.LabelSelector, err) @@ -81,8 +100,6 @@ func (p *PortForwarder) Forward(ctx context.Context) error { return fmt.Errorf("no pods found in %q with label %q", p.opts.Namespace, p.opts.LabelSelector) //nolint:goerr113 //no specific handling expected } - randomIndex := rand.Intn(len(pods.Items)) //nolint:gosec //this is going to be revised in the future anyways, avoid random pods - podName := pods.Items[randomIndex].Name portForwardURL := p.clientset.CoreV1().RESTClient().Post(). Resource("pods"). Namespace(p.opts.Namespace). From e735d6e43522044594d742ed77e119c558045b53 Mon Sep 17 00:00:00 2001 From: Mathew Merrick Date: Mon, 5 Feb 2024 09:17:48 -0800 Subject: [PATCH 22/31] ci: E2E Framework [Dev CLI] [6/6] (#2530) add deploy cli --- .gitignore | 3 ++ test/e2e/cmd/README.md | 2 + test/e2e/cmd/clusters.go | 91 ++++++++++++++++++++++++++++++++++++++++ test/e2e/cmd/main.go | 63 ++++++++++++++++++++++++++++ 4 files changed, 159 insertions(+) create mode 100644 test/e2e/cmd/README.md create mode 100644 test/e2e/cmd/clusters.go create mode 100644 test/e2e/cmd/main.go diff --git a/.gitignore b/.gitignore index e55c1f21fd6..f97e0461990 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,6 @@ test/scale/generated/* # test env file *.env + +# omit e2e bin +test/e2e/bin/* diff --git a/test/e2e/cmd/README.md b/test/e2e/cmd/README.md new file mode 100644 index 00000000000..a4b73641274 --- /dev/null +++ b/test/e2e/cmd/README.md @@ -0,0 +1,2 @@ +# This package is not for use in tests or authoring tests +## It merely provides a way to use the test infra in a way that is not coupled to the test framework, i.e. manually deploying a cluster on the fly diff --git a/test/e2e/cmd/clusters.go b/test/e2e/cmd/clusters.go new file mode 100644 index 00000000000..831c4ac1308 --- /dev/null +++ b/test/e2e/cmd/clusters.go @@ -0,0 +1,91 @@ +package main + +import ( + "fmt" + "os/exec" + "os/user" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-container-networking/test/e2e/framework/azure" + "github.com/Azure/azure-container-networking/test/e2e/framework/types" + "github.com/spf13/cobra" +) + +func newClusterCmd() *cobra.Command { + clusterCmd := &cobra.Command{ + Use: "cluster", + Short: "deploys a cluster", + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, + }, + } + + clusterCmd.AddCommand(newBYOCiliumCmd()) + + return clusterCmd +} + +func newBYOCiliumCmd() *cobra.Command { + byocilium := &cobra.Command{ + Use: "byocilium", + Short: "deploys a BYO Cilium Cluster", + RunE: func(cmd *cobra.Command, args []string) error { + job := types.NewJob("deploy BYO Cilium Cluster") + + sub, err := GetCurrentAzCLISubscriptionID() + if err != nil { + return fmt.Errorf("failed to get subscription id: %w", err) + } + + curuser, _ := user.Current() + clusterName := curuser.Username + "-byocilium-" + strconv.FormatInt(time.Now().Unix(), 10) + + job.AddStep(&azure.CreateResourceGroup{ + SubscriptionID: sub, + ResourceGroupName: clusterName, + Location: "westus2", + }, nil) + + job.AddStep(&azure.CreateVNet{ + VnetName: "testvnet", + VnetAddressSpace: "10.0.0.0/9", + }, nil) + + job.AddStep(&azure.CreateSubnet{ + SubnetName: "testsubnet", + SubnetAddressSpace: "10.0.0.0/12", + }, nil) + + job.AddStep(&azure.CreateBYOCiliumCluster{ + ClusterName: clusterName, + PodCidr: "10.128.0.0/9", + DNSServiceIP: "192.168.0.10", + ServiceCidr: "192.168.0.0/28", + }, nil) + + if err := job.Run(); err != nil { + return err // nolint // wrapping this error is noise, Cobra will handle + } + + fmt.Printf("\nto get the kubeconfig for this cluster, run:\n\n\taz aks get-credentials --resource-group %s --name %s\n\n", clusterName, clusterName) + + return nil + }, + } + + return byocilium +} + +func GetCurrentAzCLISubscriptionID() (string, error) { + // this function requires Azure CLI to be installed, as even the Azure SDK for Go makes a call to it when using the Azure CLI credential type: + // https://github.com/Azure/azure-sdk-for-go/blob/0cda95c7a7e55361d9602a7c8a141eec584f75cc/sdk/azidentity/azure_cli_credential.go#L116 + + cmd := exec.Command("az", "account", "show", "--query=id", "-o", "tsv") + output, err := cmd.Output() + if err != nil { + return "", err //nolint // wrapping this error is noise, caller will handle + } + return strings.TrimSpace(string(output)), nil +} diff --git a/test/e2e/cmd/main.go b/test/e2e/cmd/main.go new file mode 100644 index 00000000000..6a963cfcc24 --- /dev/null +++ b/test/e2e/cmd/main.go @@ -0,0 +1,63 @@ +// this package is not used for e2e tests, but rather +// leverage the e2e framework to deploy components for +// quick access + +package main + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +func main() { + rootCmd := NewRootCmd() + + cobra.OnInitialize(func() { + viper.AutomaticEnv() + initCommandFlags(rootCmd.Commands()) + }) + + cobra.CheckErr(rootCmd.Execute()) +} + +func NewRootCmd() *cobra.Command { + rootCmd := &cobra.Command{ + Use: "acndev", + Short: "Manual CLI for deploying specific ACN components, leveraging the e2e framework", + CompletionOptions: cobra.CompletionOptions{ + DisableDefaultCmd: true, + }, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + viper.AutomaticEnv() // read in environment variables that match + return nil + }, + } + + clusterCmd := newClusterCmd() + + rootCmd.AddCommand(clusterCmd) + + return rootCmd +} + +func initCommandFlags(commands []*cobra.Command) { + for _, cmd := range commands { + // bind vars from env or conf to pflags + err := viper.BindPFlags(cmd.Flags()) + cobra.CheckErr(err) + + c := cmd + c.Flags().VisitAll(func(flag *pflag.Flag) { + if viper.IsSet(flag.Name) && viper.GetString(flag.Name) != "" { + err := c.Flags().Set(flag.Name, viper.GetString(flag.Name)) + cobra.CheckErr(err) + } + }) + + // call recursively on subcommands + if cmd.HasSubCommands() { + initCommandFlags(cmd.Commands()) + } + } +} From 0ac2938f2aa3e3e6264cc8f334c60d0ea7ffd763 Mon Sep 17 00:00:00 2001 From: rjdenney <105380463+rjdenney@users.noreply.github.com> Date: Mon, 5 Feb 2024 15:31:05 -0500 Subject: [PATCH 23/31] ci:Adding cilium dualstack overlay pipeline steps (#2418) * adding cilium dualstack overlay pipeline steps * changing step order * remove dropgz step * add CNS instead of azurevnet * add v6 CNS state file test * remove nightly pipeline * removing nightly and adding v6 to cilium linux validate * fix func name * removing unneeded steps * addressing comments * remove 120 timeout * cluster type to dualstack-byocni-nokubeproxy-up * setting to INSTALL_OVERLAY * using make to get azure-ipam nad cns * fixing yaml in cil dual steps * update step * skipping bad step for cilium dualstack * update variable name * CNI type * remove file path * fixing CNI versions * removing log outputs * change step * adding cd in directory before running validate * hostport to false * further update * add cilium to dualstack test name --- .../cni/k8s-e2e/k8s-e2e-job-template.yaml | 13 +- .pipelines/pipeline.yaml | 17 +++ ...ium-dualstackoverlay-e2e-job-template.yaml | 82 ++++++++++++ ...um-dualstackoverlay-e2e-step-template.yaml | 124 ++++++++++++++++++ .../cilium/cilium-config-dualstack.yaml | 89 +++++++++++++ test/validate/linux_validate.go | 53 +++++++- 6 files changed, 372 insertions(+), 6 deletions(-) create mode 100644 .pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml create mode 100644 .pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml create mode 100644 test/integration/manifests/cilium/cilium-config-dualstack.yaml diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml index 2ebc576077d..014adfde0ac 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml @@ -119,7 +119,7 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if eq(parameters.dualstack, true) }}: + - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}: - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: DualStack Test @@ -130,3 +130,14 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 + - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}: + - template: ../k8s-e2e/k8s-e2e-step-template.yaml + parameters: + testName: DualStack Test|Cilium + name: DualStack + clusterName: ${{ parameters.clusterName }} + ginkgoFocus: '\[Feature:IPv6DualStack\]' + ginkgoSkip: 'SCTP|session affinity|should function for service endpoints using hostNetwork' # Cilium dualstack has a known issue with this test https://github.com/cilium/cilium/issues/25135 + os: ${{ parameters.os }} + processes: 8 + attempts: 3 diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 460ebd0384f..d58d867fa20 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -420,6 +420,18 @@ stages: k8sVersion: "" dependsOn: "test" + # Cilium Dualstack Overlay E2E tests + - template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml + parameters: + name: "cilium_dualstackoverlay_e2e" + displayName: Cilium on AKS DualStack Overlay + os: linux + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cildsovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "test" + # Cilium Overlay with hubble E2E tests - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e-job-template.yaml parameters: @@ -517,6 +529,7 @@ stages: - aks_swift_vnetscale_e2e - aks_windows_22_e2e - dualstackoverlay_e2e + - cilium_dualstackoverlay_e2e variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] jobs: @@ -562,6 +575,9 @@ stages: dualstackoverlay_e2e: name: dualstackoverlay_e2e clusterName: "dsovere2e" + cilium_dualstackoverlay_e2e: + name: cilium_dualstackoverlay_e2e + clusterName: "cildsovere2e" steps: - template: templates/delete-cluster.yaml parameters: @@ -600,6 +616,7 @@ stages: - aks_ubuntu_22_linux_e2e - aks_windows_22_e2e - dualstackoverlay_e2e + - cilium_dualstackoverlay_e2e jobs: - job: delete_remote_artifacts displayName: Delete remote artifacts diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml new file mode 100644 index 00000000000..bf8cb35b2a8 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-job-template.yaml @@ -0,0 +1,82 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) # Dualstack has a specific region requirement + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + jobs: + - job: ${{ parameters.name }} + displayName: Cilium Dualstack Overlay Test Suite - (${{ parameters.name }}) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + demands: + - agent.os -equals Linux + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - template: cilium-dualstackoverlay-e2e-step-template.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + dependsOn: ${{ parameters.name }} + dualstack: true + dns: true + portforward: true + service: true + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml new file mode 100644 index 00000000000..e91282f6d41 --- /dev/null +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e-step-template.yaml @@ -0,0 +1,124 @@ +parameters: + name: "" + clusterName: "" + +steps: + + - bash: | + go version + go env + mkdir -p '$(GOBIN)' + mkdir -p '$(GOPATH)/pkg' + mkdir -p '$(modulePath)' + echo '##vso[task.prependpath]$(GOBIN)' + echo '##vso[task.prependpath]$(GOROOT)/bin' + name: "GoEnv" + displayName: "Set up the Go environment" + + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@1 + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/cilium-config-dualstack.yaml + echo "install Cilium ${CILIUM_VERSION_TAG}" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + # Use different file directories for nightly and current cilium version + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + kubectl get po -owide -A + name: "installCilium" + displayName: "Install Cilium on AKS Dualstack Overlay" + + - script: | + echo "install cilium CLI" + if [[ ${CILIUM_VERSION_TAG} =~ ^1.1[1-3].[0-9]{1,2} ]]; then + echo "Cilium Agent Version ${BASH_REMATCH[0]}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable-v0.14.txt) + else + echo "Cilium Agent Version ${CILIUM_VERSION_TAG}" + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) + fi + CLI_ARCH=amd64 + if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum + sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin + rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + cilium status + cilium version + name: "installCiliumCLI" + displayName: "Install Cilium CLI" + + - script: | + echo "Start Azilium E2E Tests on Overlay Cluster" + sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true + retryCountOnTaskFailure: 3 + name: "aziliumTest" + displayName: "Run Azilium E2E on AKS Overlay" + + - script: | + kubectl get pods -A + echo "Waiting < 2 minutes for cilium to be ready" + # Ensure Cilium is ready Xm\Xs + cilium status --wait --wait-duration 2m + retryCountOnTaskFailure: 3 + name: "CiliumStatus" + displayName: "Cilium Status" + + - script: | + echo "Run Cilium Connectivity Tests" + cilium status + cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' + retryCountOnTaskFailure: 3 + name: "ciliumConnectivityTests" + displayName: "Run Cilium Connectivity Tests" + + - script: | + set -e + kubectl get po -owide -A + cd test/integration/datapath + echo "Dualstack Overlay Linux datapath IPv6 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true + echo "Dualstack Overlay Linux datapath IPv4 test" + go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration + retryCountOnTaskFailure: 3 + name: "DualStack_Overlay_Linux_Tests" + displayName: "DualStack Overlay Linux Tests" + + - script: | + echo "validate pod IP assignment and check systemd-networkd restart" + kubectl get pod -owide -A + # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. + # Saves 17 minutes + kubectl delete deploy -n cilium-test echo-external-node + cd test/integration/load + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns cilium-test + kubectl get pod -owide -A + CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ + name: "validatePods" + displayName: "Validate Pods" + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + name: "WireserverMetadataConnectivityTests" + displayName: "Run Wireserver and Metadata Connectivity Tests" diff --git a/test/integration/manifests/cilium/cilium-config-dualstack.yaml b/test/integration/manifests/cilium/cilium-config-dualstack.yaml new file mode 100644 index 00000000000..7ca869e6662 --- /dev/null +++ b/test/integration/manifests/cilium/cilium-config-dualstack.yaml @@ -0,0 +1,89 @@ +apiVersion: v1 +data: + agent-not-ready-taint-key: node.cilium.io/agent-not-ready + arping-refresh-period: 30s + auto-direct-node-routes: "false" + bpf-lb-external-clusterip: "false" + bpf-lb-map-max: "65536" + bpf-lb-mode: snat + bpf-map-dynamic-size-ratio: "0.0025" + bpf-policy-map-max: "16384" + bpf-root: /sys/fs/bpf + cgroup-root: /run/cilium/cgroupv2 + cilium-endpoint-gc-interval: 5m0s + cluster-id: "0" + cluster-name: default + debug: "false" + disable-cnp-status-updates: "true" + disable-endpoint-crd: "false" + enable-auto-protect-node-port-range: "true" + enable-bgp-control-plane: "false" + enable-bpf-clock-probe: "true" + enable-endpoint-health-checking: "false" + enable-endpoint-routes: "true" + enable-health-check-nodeport: "true" + enable-health-checking: "true" + enable-host-legacy-routing: "true" + enable-hubble: "false" + enable-ipv4: "true" + enable-ipv4-masquerade: "false" + enable-ipv6: "true" + enable-ipv6-masquerade: "false" + enable-k8s-terminating-endpoint: "true" + enable-l2-neigh-discovery: "true" + enable-l7-proxy: "false" + enable-local-node-route: "false" + enable-local-redirect-policy: "false" + enable-metrics: "true" + enable-policy: default + enable-remote-node-identity: "true" + enable-session-affinity: "true" + enable-svc-source-range-check: "true" + enable-vtep: "false" + enable-well-known-identities: "false" + enable-xt-socket-fallback: "true" + identity-allocation-mode: crd + install-iptables-rules: "true" + install-no-conntrack-iptables-rules: "false" + ipam: delegated-plugin + kube-proxy-replacement: strict + kube-proxy-replacement-healthz-bind-address: "0.0.0.0:10256" + local-router-ipv4: 169.254.23.0 + local-router-ipv6: "fe80::" + metrics: +cilium_bpf_map_pressure + monitor-aggregation: medium + monitor-aggregation-flags: all + monitor-aggregation-interval: 5s + node-port-bind-protection: "true" + nodes-gc-interval: 5m0s + operator-api-serve-addr: 127.0.0.1:9234 + operator-prometheus-serve-addr: :9963 + preallocate-bpf-maps: "false" + procfs: /host/proc + prometheus-serve-addr: :9962 + remove-cilium-node-taints: "true" + set-cilium-is-up-condition: "true" + sidecar-istio-proxy-image: cilium/istio_proxy + synchronize-k8s-nodes: "true" + tofqdns-dns-reject-response-code: refused + tofqdns-enable-dns-compression: "true" + tofqdns-endpoint-max-ip-per-hostname: "50" + tofqdns-idle-connection-grace-period: 0s + tofqdns-max-deferred-connection-deletes: "10000" + tofqdns-min-ttl: "3600" + tofqdns-proxy-response-max-delay: 100ms + tunnel: disabled + unmanaged-pod-watcher-interval: "15" + vtep-cidr: "" + vtep-endpoint: "" + vtep-mac: "" + vtep-mask: "" +kind: ConfigMap +metadata: + annotations: + meta.helm.sh/release-name: cilium + meta.helm.sh/release-namespace: kube-system + labels: + app.kubernetes.io/managed-by: Helm + name: cilium-config + namespace: kube-system diff --git a/test/validate/linux_validate.go b/test/validate/linux_validate.go index 2e2dcc807d1..4c12cb7833b 100644 --- a/test/validate/linux_validate.go +++ b/test/validate/linux_validate.go @@ -46,6 +46,11 @@ var linuxChecksMap = map[string][]check{ {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, {"azure dualstackoverlay", azureVnetStateIps, privilegedLabelSelector, privilegedNamespace, azureVnetStateFileCmd}, }, + "cilium_dualstack": { + {"cns dualstack", cnsManagedStateFileDualStackIps, cnsLabelSelector, privilegedNamespace, cnsManagedStateFileCmd}, // cns configmap "ManageEndpointState": true, | Endpoints managed in CNS State File + {"cilium", ciliumStateFileDualStackIps, ciliumLabelSelector, privilegedNamespace, ciliumStateFileCmd}, + {"cns cache", cnsCacheStateFileIps, cnsLabelSelector, privilegedNamespace, cnsCachedAssignedIPStateCmd}, + }, } type CnsManagedState struct { @@ -70,7 +75,8 @@ type NetworkingAddressing struct { } type Address struct { - Addr string `json:"ipv4"` + IPv4 string `json:"ipv4"` + IPv6 string `json:"ipv6"` } // parse azure-vnet.json @@ -133,8 +139,26 @@ func cnsManagedStateFileIps(result []byte) (map[string]string, error) { for _, v := range cnsResult.Endpoints { for ifName, ip := range v.IfnameToIPMap { if ifName == "eth0" { - ip := ip.IPv4[0].IP.String() - cnsPodIps[ip] = v.PodName + cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName + } + } + } + return cnsPodIps, nil +} + +func cnsManagedStateFileDualStackIps(result []byte) (map[string]string, error) { + var cnsResult CnsManagedState + err := json.Unmarshal(result, &cnsResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cns endpoint list") + } + + cnsPodIps := make(map[string]string) + for _, v := range cnsResult.Endpoints { + for ifName, ip := range v.IfnameToIPMap { + if ifName == "eth0" { + cnsPodIps[ip.IPv4[0].IP.String()] = v.PodName + cnsPodIps[ip.IPv6[0].IP.String()] = v.PodName } } } @@ -151,8 +175,27 @@ func ciliumStateFileIps(result []byte) (map[string]string, error) { ciliumPodIps := make(map[string]string) for _, v := range ciliumResult { for _, addr := range v.Status.Networking.Addresses { - if addr.Addr != "" { - ciliumPodIps[addr.Addr] = v.Status.Networking.InterfaceName + if addr.IPv4 != "" { + ciliumPodIps[addr.IPv4] = v.Status.Networking.InterfaceName + } + } + } + return ciliumPodIps, nil +} + +func ciliumStateFileDualStackIps(result []byte) (map[string]string, error) { + var ciliumResult []CiliumEndpointStatus + err := json.Unmarshal(result, &ciliumResult) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal cilium endpoint list") + } + + ciliumPodIps := make(map[string]string) + for _, v := range ciliumResult { + for _, addr := range v.Status.Networking.Addresses { + if addr.IPv4 != "" && addr.IPv6 != "" { + ciliumPodIps[addr.IPv4] = v.Status.Networking.InterfaceName + ciliumPodIps[addr.IPv6] = v.Status.Networking.InterfaceName } } } From 5171d7ebb2425b9cab0ce72008e9c7bd8bd795db Mon Sep 17 00:00:00 2001 From: John Payne <89417863+jpayne3506@users.noreply.github.com> Date: Mon, 5 Feb 2024 13:04:55 -0800 Subject: [PATCH 24/31] fix: az pipeline output within Merqe Queue Check (#2560) fix: az pipeline bug --- .github/workflows/pipeline.yaml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/pipeline.yaml b/.github/workflows/pipeline.yaml index b325ebdcf3b..ec093e09654 100644 --- a/.github/workflows/pipeline.yaml +++ b/.github/workflows/pipeline.yaml @@ -38,22 +38,22 @@ jobs: echo ${{ secrets.AZURE_DEVOPS_EXT_PAT }} | az devops login --org ${{ secrets.AZURE_PIPELINE_ORG }} echo "Sanity check recently triggered Merge Queue Pipeline runs" - az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --reason individualCI --top 10 --output json | jq -r .[].sourceBranch - status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status` + az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --reason individualCI --top 10 --query-order QueueTimeDesc --output json | jq -r .[].sourceBranch + status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status` echo "Triggered CI Status - $status" echo "Branch Ref - $GITHUB_REF" echo "Checking for AZP triggered CI for 60s" end=$((SECONDS+60)) # Stop checking if not queued within a minute while [ $SECONDS -lt $end ]; do + echo "Waiting for 5 seconds for AZP to trigger run and show inProgress or notStarted" + sleep 5s + status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status` + echo "Current CI Status - $status" if [ $status = 'inProgress' ] || [ $status = 'notStarted' ]; then echo "AZP triggered pipeline queued successfully" exit 0 fi - echo "Waiting for 15 seconds for AZP to trigger run and show inProgress or notStarted" - sleep 15s - status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status` - echo "Current CI Status - $status" done echo "AZP did not trigger CI" @@ -66,7 +66,7 @@ jobs: while [ $SECONDS -lt $end ]; do echo "Waiting for 5 seconds for pipeline to show inProgress or notStarted on AZP" sleep 5s - status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --output json | jq -r .[].status` + status=`az pipelines runs list --pipeline-ids ${{ secrets.AZURE_PIPELINE_ID }} --org ${{ secrets.AZURE_PIPELINE_ORG }} --project ${{ secrets.AZURE_PIPELINE_PROJECT }} --top 1 --branch $GITHUB_REF --query-order QueueTimeDesc --output json | jq -r .[].status` echo "Current CI Status - $status" if [ $status = 'inProgress' ] || [ $status = 'notStarted' ]; then echo "Manually triggered pipeline queued successfully" From ea7e7336376e5b44357d873b68ba95328939d91d Mon Sep 17 00:00:00 2001 From: rayaisaiah Date: Wed, 7 Feb 2024 09:16:20 -0800 Subject: [PATCH 25/31] chore: [NPM] Remove TLS Certifications in NPM (#2561) Remove certs created in npm images --- .gitignore | 6 ------ npm/linux.Dockerfile | 3 --- npm/scripts/generate_certs.sh | 38 ----------------------------------- npm/scripts/san.cnf | 21 ------------------- 4 files changed, 68 deletions(-) delete mode 100755 npm/scripts/generate_certs.sh delete mode 100644 npm/scripts/san.cnf diff --git a/.gitignore b/.gitignore index f97e0461990..b28e45c9298 100644 --- a/.gitignore +++ b/.gitignore @@ -25,12 +25,6 @@ controller-gen build/tools/bin npm/debug/http -# certificates -*/**/certs/ -*.crt -*.pem -*.srl - go.work* # scale-test diff --git a/npm/linux.Dockerfile b/npm/linux.Dockerfile index 48036e50b36..f9f0cdf2c37 100644 --- a/npm/linux.Dockerfile +++ b/npm/linux.Dockerfile @@ -8,9 +8,6 @@ RUN CGO_ENABLED=0 go build -v -o /usr/local/bin/azure-npm -ldflags "-X main.vers FROM mcr.microsoft.com/mirror/docker/library/ubuntu:20.04 COPY --from=builder /usr/local/bin/azure-npm /usr/bin/azure-npm -COPY --from=builder /usr/local/src/npm/scripts /usr/local/npm RUN apt-get update && apt-get install -y iptables ipset ca-certificates && apt-get autoremove -y && apt-get clean RUN chmod +x /usr/bin/azure-npm -WORKDIR /usr/local/npm -RUN ./generate_certs.sh ENTRYPOINT ["/usr/bin/azure-npm", "start"] diff --git a/npm/scripts/generate_certs.sh b/npm/scripts/generate_certs.sh deleted file mode 100755 index 22df96102c7..00000000000 --- a/npm/scripts/generate_certs.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -CERTS_STAGING_DIR=. -SAN_CNF_FILE=san.cnf -CERTIFICATE_VALIDITY_DAYS=3650 -CERT_SUBJ="/C=US/ST=Washington/L=Redmond/O=Microsoft/OU=Azure/CN=azure-npm.kube-system.svc.cluster.local" - -# Check if openssl is installed -if ! command -v openssl &> /dev/null -then - echo "openssl could not be found" - exit -fi - -# Check if SAN_CNF_FILE exists -if [ ! -f "$SAN_CNF_FILE" ] -then - echo "SAN_CNF_FILE does not exist" - exit -fi - -if [ ! -d "$CERTS_STAGING_DIR" ] -then - echo "Creating $CERTS_STAGING_DIR" - mkdir -p $CERTS_STAGING_DIR -fi - -# Generate the ca certificate and key -openssl req -x509 -newkey rsa:4096 -days $CERTIFICATE_VALIDITY_DAYS -nodes -keyout $CERTS_STAGING_DIR/ca.key -out $CERTS_STAGING_DIR/ca.crt -subj $CERT_SUBJ - -# Create a certificate signing request for the server -openssl req -newkey rsa:4096 -nodes -keyout $CERTS_STAGING_DIR/tls.key -out $CERTS_STAGING_DIR/server-req.pem -config $SAN_CNF_FILE -extensions v3_req -subj $CERT_SUBJ - -# Sign the server certificate with the CA -openssl x509 -req -in $CERTS_STAGING_DIR/server-req.pem -CA $CERTS_STAGING_DIR/ca.crt -CAkey $CERTS_STAGING_DIR/ca.key -CAcreateserial -out $CERTS_STAGING_DIR/tls.crt --days $CERTIFICATE_VALIDITY_DAYS --extfile $SAN_CNF_FILE --extensions v3_req - -# Remove the secret CA key and signing request -rm -rf $CERTS_STAGING_DIR/ca.key $CERTS_STAGING_DIR/server-req.pem diff --git a/npm/scripts/san.cnf b/npm/scripts/san.cnf deleted file mode 100644 index c6b10bac918..00000000000 --- a/npm/scripts/san.cnf +++ /dev/null @@ -1,21 +0,0 @@ -[ req ] -default_bits = 2048 -distinguished_name = req_distinguished_name -req_extensions = v3_req - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -stateOrProvinceName = State or Province Name (full name) -localityName = Locality Name (eg, city) -organizationName = Organization Name (eg, company) -commonName = Common Name (e.g. server FQDN or YOUR name) - -[ v3_req ] -keyUsage = digitalSignature, nonRepudiation, keyEncipherment -subjectAltName = @alt_names - -[alt_names] -DNS.1 = azure-npm.kube-system.svc.cluster.local -DNS.2 = azure-npm.kube-system -DNS.3 = azure-npm -DNS.4 = 0.0.0.0 From 1d36cfc2de4ab910e7eeb2b994a7ea163efcec68 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:31:58 -0600 Subject: [PATCH 26/31] deps: bump github.com/Azure/azure-sdk-for-go/sdk/azcore from 1.9.1 to 1.9.2 (#2563) deps: bump github.com/Azure/azure-sdk-for-go/sdk/azcore Bumps [github.com/Azure/azure-sdk-for-go/sdk/azcore](https://github.com/Azure/azure-sdk-for-go) from 1.9.1 to 1.9.2. - [Release notes](https://github.com/Azure/azure-sdk-for-go/releases) - [Changelog](https://github.com/Azure/azure-sdk-for-go/blob/main/documentation/release.md) - [Commits](https://github.com/Azure/azure-sdk-for-go/compare/sdk/azcore/v1.9.1...sdk/azcore/v1.9.2) --- updated-dependencies: - dependency-name: github.com/Azure/azure-sdk-for-go/sdk/azcore dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index ee9e4eccd3f..549a9a8eaee 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/Azure/azure-container-networking go 1.21 require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 github.com/Masterminds/semver v1.5.0 @@ -54,7 +54,7 @@ require ( require ( code.cloudfoundry.org/clock v1.0.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 03025387cfe..b542c6305b7 100644 --- a/go.sum +++ b/go.sum @@ -2,12 +2,12 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMT code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2o= code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 h1:lGlwhPtrX6EVml1hO0ivjkUxsSyl4dsiw9qcA1k/3IQ= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1/go.mod h1:RKUqNu35KJYcVG/fqTRqmuXJZYNhYkBrnC/hX7yGbTA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2 h1:c4k2FIYIh4xtwqrQwV0Ct1v5+ehlNXj5NI/MWVsiTkQ= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.2/go.mod h1:5FDJtLEO/GxwNgUxbwrY3LP0pEoThTQJtk2oysdXHxM= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1/go.mod h1:h8hyGFDsU5HMivxiS2iYFZsgDbU9OnnJ163x5UGVKYo= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 h1:6oNBlSdi1QqM1PNW7FPA6xOGA5UNsXnkaYZz9vdPGhA= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1/go.mod h1:s4kgfzA0covAXNicZHDMN58jExvcng2mC/DepXiF1EI= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 h1:LqbJ/WzJUwBf8UiaSzgX7aMclParm9/5Vgp+TY51uBQ= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2/go.mod h1:yInRyqWXAuaPrgI7p70+lDDgh3mlBohis29jGMISnmc= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0 h1:xnO4sFyG8UH2fElBkcqLTOZsAajvKfnSlgBBW8dXYjw= github.com/Azure/azure-sdk-for-go/sdk/keyvault/azsecrets v0.12.0/go.mod h1:XD3DIOOVgBCO03OleB1fHjgktVRFxlT++KwKgIOewdM= github.com/Azure/azure-sdk-for-go/sdk/keyvault/internal v0.7.1 h1:FbH3BbSb4bvGluTesZZ+ttN/MDsnMmQP36OSnDuSXqw= @@ -26,7 +26,6 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v5 v5.0.0/go.mod h1:Wfx7a5UHfOLG6O4NZ7Q0BPZUYwvlNCBR/OlIBpP3dlA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE= - github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 h1:DzHpqpoJVaCgOUdVHxE8QB52S6NiVdDQvGlny1qvPqA= github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -174,6 +173,8 @@ github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee h1:PAXLXk1heNZ5y github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -216,6 +217,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= From f97eb963554a8883f2d0e19f326f4a99f8b64a60 Mon Sep 17 00:00:00 2001 From: Matthew Long <61910737+thatmattlong@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:32:36 -0800 Subject: [PATCH 27/31] fix: build imds compute url correctly and give a default retryAttempts (#2566) * fix: build imds compute url correctly * fix: remove httputil debug stuff --- cns/imds/client.go | 19 +++++++++++-------- cns/imds/client_test.go | 6 ++++++ 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/cns/imds/client.go b/cns/imds/client.go index 5ab43e76ba0..6e210e4705a 100644 --- a/cns/imds/client.go +++ b/cns/imds/client.go @@ -44,12 +44,14 @@ func RetryAttempts(attempts uint) ClientOption { } const ( - vmUniqueIDProperty = "vmId" - imdsComputePath = "/metadata/instance/compute?api-version=2021-01-01&format=json" - metadataHeaderKey = "Metadata" - metadataHeaderValue = "true" - defaultRetryAttempts = 10 - defaultIMDSEndpoint = "http://169.254.169.254" + vmUniqueIDProperty = "vmId" + imdsComputePath = "/metadata/instance/compute" + imdsComputeAPIVersion = "api-version=2021-01-01" + imdsFormatJSON = "format=json" + metadataHeaderKey = "Metadata" + metadataHeaderValue = "true" + defaultRetryAttempts = 3 + defaultIMDSEndpoint = "http://169.254.169.254" ) var ( @@ -60,7 +62,8 @@ var ( // NewClient creates a new imds client func NewClient(opts ...ClientOption) *Client { config := clientConfig{ - endpoint: defaultIMDSEndpoint, + endpoint: defaultIMDSEndpoint, + retryAttempts: defaultRetryAttempts, } for _, o := range opts { @@ -104,6 +107,7 @@ func (c *Client) getInstanceComputeMetadata(ctx context.Context) (map[string]any if err != nil { return nil, errors.Wrap(err, "unable to build path to IMDS compute metadata") } + imdsComputeURL = imdsComputeURL + "?" + imdsComputeAPIVersion + "&" + imdsFormatJSON req, err := http.NewRequestWithContext(ctx, http.MethodGet, imdsComputeURL, http.NoBody) if err != nil { @@ -112,7 +116,6 @@ func (c *Client) getInstanceComputeMetadata(ctx context.Context) (map[string]any // IMDS requires the "Metadata: true" header req.Header.Add(metadataHeaderKey, metadataHeaderValue) - resp, err := c.cli.Do(req) if err != nil { return nil, errors.Wrap(err, "error querying IMDS") diff --git a/cns/imds/client_test.go b/cns/imds/client_test.go index c0c03b40a3f..fe8324666fa 100644 --- a/cns/imds/client_test.go +++ b/cns/imds/client_test.go @@ -23,6 +23,12 @@ func TestGetVMUniqueID(t *testing.T) { // request header "Metadata: true" must be present metadataHeader := r.Header.Get("Metadata") assert.Equal(t, "true", metadataHeader) + + // query params should include apiversion and json format + apiVersion := r.URL.Query().Get("api-version") + assert.Equal(t, "2021-01-01", apiVersion) + format := r.URL.Query().Get("format") + assert.Equal(t, "json", format) w.WriteHeader(http.StatusOK) _, writeErr := w.Write(computeMetadata) require.NoError(t, writeErr, "error writing response") From a1bf7bfb715c860e4d9686c0b13dbbbf9eb07ea1 Mon Sep 17 00:00:00 2001 From: QxBytes <39818795+QxBytes@users.noreply.github.com> Date: Thu, 8 Feb 2024 08:37:30 -0800 Subject: [PATCH 28/31] fix: block pod to wireserver port 80 traffic on windows multitenancy (#2515) Add endpoint policy to block wireserver traffic on windows --- cni/azure-windows-multitenancy.conflist | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cni/azure-windows-multitenancy.conflist b/cni/azure-windows-multitenancy.conflist index 98689b80ecc..3114265932f 100644 --- a/cni/azure-windows-multitenancy.conflist +++ b/cni/azure-windows-multitenancy.conflist @@ -42,6 +42,19 @@ "DestinationPrefix": "10.0.0.0/8", "NeedEncap": true } + }, + { + "Name": "EndpointPolicy", + "Value": { + "Type": "ACL", + "Protocols": "6", + "Action": "Block", + "Direction": "Out", + "RemoteAddresses": "168.63.129.16", + "RemotePorts": "80", + "Priority": 200, + "RuleType": "Switch" + } } ], "windowsSettings": { From ccfea64bb56195577f447d9186b7e3159b5be4a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:49:17 +0000 Subject: [PATCH 29/31] deps: bump golang.org/x/sys from 0.16.0 to 0.17.0 (#2567) Bumps [golang.org/x/sys](https://github.com/golang/sys) from 0.16.0 to 0.17.0. - [Commits](https://github.com/golang/sys/compare/v0.16.0...v0.17.0) --- updated-dependencies: - dependency-name: golang.org/x/sys dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 549a9a8eaee..2236538f344 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.26.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/sys v0.16.0 + golang.org/x/sys v0.17.0 google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect google.golang.org/grpc v1.61.0 google.golang.org/protobuf v1.32.0 diff --git a/go.sum b/go.sum index b542c6305b7..555fbb782db 100644 --- a/go.sum +++ b/go.sum @@ -381,8 +381,8 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= From 01d26ff0bbe440f11a5b984a729e0580fc247bea Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Feb 2024 16:49:59 +0000 Subject: [PATCH 30/31] deps: bump github.com/golangci/golangci-lint from 1.55.2 to 1.56.0 in /build/tools (#2569) deps: bump github.com/golangci/golangci-lint in /build/tools Bumps [github.com/golangci/golangci-lint](https://github.com/golangci/golangci-lint) from 1.55.2 to 1.56.0. - [Release notes](https://github.com/golangci/golangci-lint/releases) - [Changelog](https://github.com/golangci/golangci-lint/blob/master/CHANGELOG.md) - [Commits](https://github.com/golangci/golangci-lint/compare/v1.55.2...v1.56.0) --- updated-dependencies: - dependency-name: github.com/golangci/golangci-lint dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- build/tools/go.mod | 40 ++++++++++----------- build/tools/go.sum | 89 +++++++++++++++++++++++----------------------- 2 files changed, 64 insertions(+), 65 deletions(-) diff --git a/build/tools/go.mod b/build/tools/go.mod index f9a8a06d452..39c3409c00c 100644 --- a/build/tools/go.mod +++ b/build/tools/go.mod @@ -6,7 +6,7 @@ require ( github.com/AlekSi/gocov-xml v1.1.0 github.com/axw/gocov v1.1.0 github.com/golang/mock v1.6.0 - github.com/golangci/golangci-lint v1.55.2 + github.com/golangci/golangci-lint v1.56.0 github.com/jstemmer/go-junit-report v1.0.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.32.0 @@ -21,13 +21,13 @@ require ( github.com/Abirdcfly/dupword v0.0.13 // indirect github.com/Antonboom/errname v0.1.12 // indirect github.com/Antonboom/nilnil v0.1.7 // indirect - github.com/Antonboom/testifylint v1.0.2 // indirect + github.com/Antonboom/testifylint v1.1.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 // indirect - github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 // indirect + github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/OpenPeeDeeP/depguard/v2 v2.2.0 // indirect - github.com/alecthomas/go-check-sumtype v0.1.3 // indirect + github.com/alecthomas/go-check-sumtype v0.1.4 // indirect github.com/alexkohler/nakedret/v2 v2.0.2 // indirect github.com/alexkohler/prealloc v1.0.0 // indirect github.com/alingse/asasalint v0.0.11 // indirect @@ -36,18 +36,18 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bkielbasa/cyclop v1.2.1 // indirect github.com/blizzy78/varnamelen v0.8.0 // indirect - github.com/bombsimon/wsl/v3 v3.4.0 // indirect + github.com/bombsimon/wsl/v4 v4.2.0 // indirect github.com/breml/bidichk v0.2.7 // indirect github.com/breml/errchkjson v0.3.6 // indirect - github.com/butuzov/ireturn v0.2.2 // indirect + github.com/butuzov/ireturn v0.3.0 // indirect github.com/butuzov/mirror v1.1.0 // indirect - github.com/catenacyber/perfsprint v0.5.0 // indirect + github.com/catenacyber/perfsprint v0.6.0 // indirect github.com/ccojocar/zxcvbn-go v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/charithe/durationcheck v0.0.10 // indirect github.com/chavacava/garif v0.1.0 // indirect github.com/curioswitch/go-reassign v0.2.0 // indirect - github.com/daixiang0/gci v0.12.0 // indirect + github.com/daixiang0/gci v0.12.1 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/denis-tingaikin/go-header v0.4.3 // indirect github.com/esimonov/ifshort v1.0.4 // indirect @@ -57,8 +57,8 @@ require ( github.com/firefart/nonamedreturns v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fzipp/gocyclo v0.6.0 // indirect - github.com/ghostiam/protogetter v0.3.3 // indirect - github.com/go-critic/go-critic v0.10.0 // indirect + github.com/ghostiam/protogetter v0.3.4 // indirect + github.com/go-critic/go-critic v0.11.0 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-toolsmith/astcast v1.1.0 // indirect github.com/go-toolsmith/astcopy v1.1.0 // indirect @@ -67,6 +67,7 @@ require ( github.com/go-toolsmith/astp v1.1.0 // indirect github.com/go-toolsmith/strparse v1.1.0 // indirect github.com/go-toolsmith/typep v1.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect github.com/go-xmlfmt/xmlfmt v1.1.2 // indirect github.com/gobuffalo/flect v1.0.2 // indirect github.com/gobwas/glob v0.2.3 // indirect @@ -88,8 +89,6 @@ require ( github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect @@ -97,6 +96,7 @@ require ( github.com/jgautheron/goconst v1.7.0 // indirect github.com/jingyugao/rowserrcheck v1.1.1 // indirect github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af // indirect + github.com/jjti/go-spancheck v0.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/julz/importas v0.1.0 // indirect github.com/kisielk/errcheck v1.6.3 // indirect @@ -109,7 +109,7 @@ require ( github.com/ldez/tagliatelle v0.5.0 // indirect github.com/leonklingele/grouper v1.1.1 // indirect github.com/lufeee/execinquery v1.2.1 // indirect - github.com/macabu/inamedparam v0.1.2 // indirect + github.com/macabu/inamedparam v0.1.3 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/maratori/testableexamples v1.0.0 // indirect github.com/maratori/testpackage v1.1.1 // indirect @@ -119,7 +119,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mbilski/exhaustivestruct v1.2.0 // indirect - github.com/mgechev/revive v1.3.4 // indirect + github.com/mgechev/revive v1.3.6 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -128,11 +128,11 @@ require ( github.com/nakabonne/nestif v0.3.1 // indirect github.com/nishanths/exhaustive v0.12.0 // indirect github.com/nishanths/predeclared v0.2.2 // indirect - github.com/nunnatsa/ginkgolinter v0.15.1 // indirect + github.com/nunnatsa/ginkgolinter v0.15.2 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.1.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/polyfloyd/go-errorlint v1.4.7 // indirect + github.com/polyfloyd/go-errorlint v1.4.8 // indirect github.com/prometheus/client_golang v1.17.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect @@ -183,12 +183,12 @@ require ( github.com/yeya24/promlinter v0.2.0 // indirect github.com/ykadowak/zerologlint v0.1.5 // indirect gitlab.com/bosi/decorder v0.4.1 // indirect + go-simpler.org/musttag v0.8.0 // indirect go-simpler.org/sloglint v0.4.0 // indirect - go.tmz.dev/musttag v0.7.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 // indirect - golang.org/x/exp/typeparams v0.0.0-20231214170342-aacd6d4b4611 // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.20.0 // indirect golang.org/x/sync v0.6.0 // indirect @@ -207,7 +207,7 @@ require ( k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed // indirect mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect - mvdan.cc/unparam v0.0.0-20230917202934-3ee2d22f45fb // indirect + mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/build/tools/go.sum b/build/tools/go.sum index 55bc04e7007..6efa5e252d3 100644 --- a/build/tools/go.sum +++ b/build/tools/go.sum @@ -45,24 +45,24 @@ github.com/Antonboom/errname v0.1.12 h1:oh9ak2zUtsLp5oaEd/erjB4GPu9w19NyoIskZClD github.com/Antonboom/errname v0.1.12/go.mod h1:bK7todrzvlaZoQagP1orKzWXv59X/x0W0Io2XT1Ssro= github.com/Antonboom/nilnil v0.1.7 h1:ofgL+BA7vlA1K2wNQOsHzLJ2Pw5B5DpWRLdDAVvvTow= github.com/Antonboom/nilnil v0.1.7/go.mod h1:TP+ScQWVEq0eSIxqU8CbdT5DFWoHp0MbP+KMUO1BKYQ= -github.com/Antonboom/testifylint v1.0.2 h1:WkSc4c6AcYAPrSqj/3MYrewhszk+mnwd07acH1NL9Vw= -github.com/Antonboom/testifylint v1.0.2/go.mod h1:tGEV9t6Th7DHXFVjd8oyLOBbIxXzs4CMEIAkbQ2RuC8= +github.com/Antonboom/testifylint v1.1.0 h1:HrgwOEqVQc5eAsWEDA6JvK7ZSzfdTBjWt0PAYHweu+o= +github.com/Antonboom/testifylint v1.1.0/go.mod h1:m62Du5rtu7uwrWsypuLPTVeKbTB3NZgPWrxfffu2r/8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24 h1:sHglBQTwgx+rWPdisA5ynNEsoARbiCBOyGcJM4/OzsM= github.com/Djarvur/go-err113 v0.0.0-20210108212216-aea10b59be24/go.mod h1:4UJr5HIiMZrwgkSPdsjy2uOQExX/WEILpIrO9UPGuXs= -github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0 h1:3ZBs7LAezy8gh0uECsA6CGU43FF3zsx5f4eah5FxTMA= -github.com/GaijinEntertainment/go-exhaustruct/v3 v3.1.0/go.mod h1:rZLTje5A9kFBe0pzhpe2TdhRniBF++PRHQuRpR8esVc= +github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0 h1:sATXp1x6/axKxz2Gjxv8MALP0bXaNRfQinEwyfMcx8c= +github.com/GaijinEntertainment/go-exhaustruct/v3 v3.2.0/go.mod h1:Nl76DrGNJTA1KJ0LePKBw/vznBX1EHbAZX8mwjR82nI= github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/OpenPeeDeeP/depguard/v2 v2.2.0 h1:vDfG60vDtIuf0MEOhmLlLLSzqaRM8EMcgJPdp74zmpA= github.com/OpenPeeDeeP/depguard/v2 v2.2.0/go.mod h1:CIzddKRvLBC4Au5aYP/i3nyaWQ+ClszLIuVocRiCYFQ= github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= github.com/alecthomas/assert/v2 v2.2.2/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ= -github.com/alecthomas/go-check-sumtype v0.1.3 h1:M+tqMxB68hcgccRXBMVCPI4UJ+QUfdSx0xdbypKCqA8= -github.com/alecthomas/go-check-sumtype v0.1.3/go.mod h1:WyYPfhfkdhyrdaligV6svFopZV8Lqdzn5pyVBaV6jhQ= +github.com/alecthomas/go-check-sumtype v0.1.4 h1:WCvlB3l5Vq5dZQTFmodqL2g68uHiSwwlWcT5a2FGK0c= +github.com/alecthomas/go-check-sumtype v0.1.4/go.mod h1:WyYPfhfkdhyrdaligV6svFopZV8Lqdzn5pyVBaV6jhQ= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -90,18 +90,18 @@ github.com/bkielbasa/cyclop v1.2.1 h1:AeF71HZDob1P2/pRm1so9cd1alZnrpyc4q2uP2l0gJ github.com/bkielbasa/cyclop v1.2.1/go.mod h1:K/dT/M0FPAiYjBgQGau7tz+3TMh4FWAEqlMhzFWCrgM= github.com/blizzy78/varnamelen v0.8.0 h1:oqSblyuQvFsW1hbBHh1zfwrKe3kcSj0rnXkKzsQ089M= github.com/blizzy78/varnamelen v0.8.0/go.mod h1:V9TzQZ4fLJ1DSrjVDfl89H7aMnTvKkApdHeyESmyR7k= -github.com/bombsimon/wsl/v3 v3.4.0 h1:RkSxjT3tmlptwfgEgTgU+KYKLI35p/tviNXNXiL2aNU= -github.com/bombsimon/wsl/v3 v3.4.0/go.mod h1:KkIB+TXkqy6MvK9BDZVbZxKNYsE1/oLRJbIFtf14qqo= +github.com/bombsimon/wsl/v4 v4.2.0 h1:dKK3o/Hk2aIt6t72CWg02ham2P5lnH9MBSW6cTU9xxU= +github.com/bombsimon/wsl/v4 v4.2.0/go.mod h1:1zaTbf/7ywOQtMdoUdTF2X1fbbBLiBUkajyuFAanT28= github.com/breml/bidichk v0.2.7 h1:dAkKQPLl/Qrk7hnP6P+E0xOodrq8Us7+U0o4UBOAlQY= github.com/breml/bidichk v0.2.7/go.mod h1:YodjipAGI9fGcYM7II6wFvGhdMYsC5pHDlGzqvEW3tQ= github.com/breml/errchkjson v0.3.6 h1:VLhVkqSBH96AvXEyclMR37rZslRrY2kcyq+31HCsVrA= github.com/breml/errchkjson v0.3.6/go.mod h1:jhSDoFheAF2RSDOlCfhHO9KqhZgAYLyvHe7bRCX8f/U= -github.com/butuzov/ireturn v0.2.2 h1:jWI36dxXwVrI+RnXDwux2IZOewpmfv930OuIRfaBUJ0= -github.com/butuzov/ireturn v0.2.2/go.mod h1:RfGHUvvAuFFxoHKf4Z8Yxuh6OjlCw1KvR2zM1NFHeBk= +github.com/butuzov/ireturn v0.3.0 h1:hTjMqWw3y5JC3kpnC5vXmFJAWI/m31jaCYQqzkS6PL0= +github.com/butuzov/ireturn v0.3.0/go.mod h1:A09nIiwiqzN/IoVo9ogpa0Hzi9fex1kd9PSD6edP5ZA= github.com/butuzov/mirror v1.1.0 h1:ZqX54gBVMXu78QLoiqdwpl2mgmoOJTk7s4p4o+0avZI= github.com/butuzov/mirror v1.1.0/go.mod h1:8Q0BdQU6rC6WILDiBM60DBfvV78OLJmMmixe7GF45AE= -github.com/catenacyber/perfsprint v0.5.0 h1:FNBJRKm2Lar44u1s7DUfpbiY4iN2LmnK6THY3d5rL40= -github.com/catenacyber/perfsprint v0.5.0/go.mod h1:/wclWYompEyjUD2FuIIDVKNkqz7IgBIWXIH3V0Zol50= +github.com/catenacyber/perfsprint v0.6.0 h1:VSv95RRkk5+BxrU/YTPcnxuMEWar1iMK5Vyh3fWcBfs= +github.com/catenacyber/perfsprint v0.6.0/go.mod h1:/wclWYompEyjUD2FuIIDVKNkqz7IgBIWXIH3V0Zol50= github.com/ccojocar/zxcvbn-go v1.0.1 h1:+sxrANSCj6CdadkcMnvde/GWU1vZiiXRbqYSCalV4/4= github.com/ccojocar/zxcvbn-go v1.0.1/go.mod h1:g1qkXtUSvHP8lhHp5GrSmTz6uWALGRMQdw6Qnz/hi60= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -121,8 +121,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/curioswitch/go-reassign v0.2.0 h1:G9UZyOcpk/d7Gd6mqYgd8XYWFMw/znxwGDUstnC9DIo= github.com/curioswitch/go-reassign v0.2.0/go.mod h1:x6OpXuWvgfQaMGks2BZybTngWjT84hqJfKoO8Tt/Roc= -github.com/daixiang0/gci v0.12.0 h1:EQTG7FfKPlO4Ste+oN0kvz+gP4XswKx29D4fLrmwbiU= -github.com/daixiang0/gci v0.12.0/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI= +github.com/daixiang0/gci v0.12.1 h1:ugsG+KRYny1VK4oqrX4Vtj70bo4akYKa0tgT1DXMYiY= +github.com/daixiang0/gci v0.12.1/go.mod h1:xtHP9N7AHdNvtRNfcx9gwTDfw7FRJx4bZUsiEfiNNAI= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -149,10 +149,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fzipp/gocyclo v0.6.0 h1:lsblElZG7d3ALtGMx9fmxeTKZaLLpU8mET09yN4BBLo= github.com/fzipp/gocyclo v0.6.0/go.mod h1:rXPyn8fnlpa0R2csP/31uerbiVBugk5whMdlyaLkLoA= -github.com/ghostiam/protogetter v0.3.3 h1:EvOuzB/SEifg/c4aMnwcj033Qc1lHO7Yz4QnBDbmbik= -github.com/ghostiam/protogetter v0.3.3/go.mod h1:A0JgIhs0fgVnotGinjQiKaFVG3waItLJNwPmcMzDnvk= -github.com/go-critic/go-critic v0.10.0 h1:tOkke48KSKC4tSS5Dc22ICdChMy0maRj1uDgdV5vXfc= -github.com/go-critic/go-critic v0.10.0/go.mod h1:gW4noMWDewS/WQdqiuJdA0pXbRxito860jj1ucPzy7g= +github.com/ghostiam/protogetter v0.3.4 h1:5SZ+lZSNmNkSbGVSF9hUHhv/b7ELF9Rwchoq7btYo6c= +github.com/ghostiam/protogetter v0.3.4/go.mod h1:A0JgIhs0fgVnotGinjQiKaFVG3waItLJNwPmcMzDnvk= +github.com/go-critic/go-critic v0.11.0 h1:mARtIFX7jPtJ3SzxO9Isa5T2jd2dZxFmQHK3yNf0wrE= +github.com/go-critic/go-critic v0.11.0/go.mod h1:Cz6lr1PlkIu/0Y0U9KqJgcIJJECAF8mEwmzVjKnhbfI= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -185,6 +185,8 @@ github.com/go-toolsmith/strparse v1.1.0 h1:GAioeZUK9TGxnLS+qfdqNbA4z0SSm5zVNtCQi github.com/go-toolsmith/strparse v1.1.0/go.mod h1:7ksGy58fsaQkGQlY8WVoBFNyEPMGuJin1rfoPS4lBSQ= github.com/go-toolsmith/typep v1.1.0 h1:fIRYDyF+JywLfqzyhdiHzRop/GQDxxNhLGQ6gFUNHus= github.com/go-toolsmith/typep v1.1.0/go.mod h1:fVIw+7zjdsMxDA3ITWnH1yOiw1rnTQKCsF/sk2H/qig= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 h1:TQcrn6Wq+sKGkpyPvppOz99zsMBaUOKXq6HSv655U1c= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/go-xmlfmt/xmlfmt v1.1.2 h1:Nea7b4icn8s57fTx1M5AI4qQT5HEM3rVUO8MuE6g80U= github.com/go-xmlfmt/xmlfmt v1.1.2/go.mod h1:aUCEOzzezBEjDBbFBoSiya/gduyIiWYRP6CnSFIV8AM= github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= @@ -233,8 +235,8 @@ github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe h1:6RGUuS7EGotKx6 github.com/golangci/go-misc v0.0.0-20220329215616-d24fe342adfe/go.mod h1:gjqyPShc/m8pEMpk0a3SeagVb0kaqvhscv+i9jI5ZhQ= github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa h1:L0Zq43Px2HrLroRKEgfCsQLMJUkjskJBB1kd1Zjcvvc= github.com/golangci/gofmt v0.0.0-20231019111953-be8c47862aaa/go.mod h1:Pm5KhLPA8gSnQwrQ6ukebRcapGb/BG9iUkdaiCcGHJM= -github.com/golangci/golangci-lint v1.55.2 h1:yllEIsSJ7MtlDBwDJ9IMBkyEUz2fYE0b5B8IUgO1oP8= -github.com/golangci/golangci-lint v1.55.2/go.mod h1:H60CZ0fuqoTwlTvnbyjhpZPWp7KmsjwV2yupIMiMXbM= +github.com/golangci/golangci-lint v1.56.0 h1:uivqYQ8WbkWAE4LgjLLhxsTyb68FlcZF3ZIF0oyn4WQ= +github.com/golangci/golangci-lint v1.56.0/go.mod h1:ZqfKpUUZ+i4MTGqc3G5t18U+5cyXuMeWabXtdL38+Dk= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0 h1:MfyDlzVjl1hoaPzPD4Gpb/QgoRfSBR0jdhwGyAWwMSA= github.com/golangci/lint-1 v0.0.0-20191013205115-297bf364a8e0/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg= github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA= @@ -293,11 +295,6 @@ github.com/gostaticanalysis/nilerr v0.1.1/go.mod h1:wZYb6YI5YAxxq0i1+VJbY0s2YONW github.com/gostaticanalysis/testutil v0.3.1-0.20210208050101-bfb5c8eec0e4/go.mod h1:D+FIZ+7OahH3ePw/izIEeH5I06eKs1IKI4Xr64/Am3M= github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoISdUv3PPQgHY= github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -316,6 +313,8 @@ github.com/jingyugao/rowserrcheck v1.1.1 h1:zibz55j/MJtLsjP1OF4bSdgXxwL1b+Vn7Tjz github.com/jingyugao/rowserrcheck v1.1.1/go.mod h1:4yvlZSDb3IyDTUZJUmpZfm2Hwok+Dtp+nu2qOq+er9c= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af h1:KA9BjwUk7KlCh6S9EAGWBt1oExIUv9WyNCiRz5amv48= github.com/jirfag/go-printf-func-name v0.0.0-20200119135958-7558a9eaa5af/go.mod h1:HEWGJkRDzjJY2sqdDwxccsGicWEf9BQOZsq2tV+xzM0= +github.com/jjti/go-spancheck v0.5.2 h1:WXTZG3efY/ji1Vi8mkH+23O3bLeKR6hp3tI3YB7XwKk= +github.com/jjti/go-spancheck v0.5.2/go.mod h1:ARPNI1JRG1V2Rjnd6/2f2NEfghjSVDZGVmruNKlnXU0= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -361,8 +360,8 @@ github.com/leonklingele/grouper v1.1.1 h1:suWXRU57D4/Enn6pXR0QVqqWWrnJ9Osrz+5rjt github.com/leonklingele/grouper v1.1.1/go.mod h1:uk3I3uDfi9B6PeUjsCKi6ndcf63Uy7snXgR4yDYQVDY= github.com/lufeee/execinquery v1.2.1 h1:hf0Ems4SHcUGBxpGN7Jz78z1ppVkP/837ZlETPCEtOM= github.com/lufeee/execinquery v1.2.1/go.mod h1:EC7DrEKView09ocscGHC+apXMIaorh4xqSxS/dy8SbM= -github.com/macabu/inamedparam v0.1.2 h1:RR5cnayM6Q7cDhQol32DE2BGAPGMnffJ31LFE+UklaU= -github.com/macabu/inamedparam v0.1.2/go.mod h1:Xg25QvY7IBRl1KLPV9Rbml8JOMZtF/iAkNkmV7eQgjw= +github.com/macabu/inamedparam v0.1.3 h1:2tk/phHkMlEL/1GNe/Yf6kkR/hkcUdAEY3L0hjYV1Mk= +github.com/macabu/inamedparam v0.1.3/go.mod h1:93FLICAIk/quk7eaPPQvbzihUdn/QkGDwIZEoLtpH6I= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/maratori/testableexamples v1.0.0 h1:dU5alXRrD8WKSjOUnmJZuzdxWOEQ57+7s93SLMxb2vI= @@ -386,8 +385,8 @@ github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvls github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mbilski/exhaustivestruct v1.2.0 h1:wCBmUnSYufAHO6J4AVWY6ff+oxWxsVFrwgOdMUQePUo= github.com/mbilski/exhaustivestruct v1.2.0/go.mod h1:OeTBVxQWoEmB2J2JCHmXWPJ0aksxSUOUy+nvtVEfzXc= -github.com/mgechev/revive v1.3.4 h1:k/tO3XTaWY4DEHal9tWBkkUMJYO/dLDVyMmAQxmIMDc= -github.com/mgechev/revive v1.3.4/go.mod h1:W+pZCMu9qj8Uhfs1iJMQsEFLRozUfvwFwqVvRbSNLVw= +github.com/mgechev/revive v1.3.6 h1:ZNKZiHb/LciAqzwa/9HnwI8S/OJutYhMvaqgMT1Ylgo= +github.com/mgechev/revive v1.3.6/go.mod h1:75Je+5jKBgdgADNzGhsq7H5J6CmyXSzEk9eLOU4i8Pg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= @@ -409,8 +408,8 @@ github.com/nishanths/exhaustive v0.12.0 h1:vIY9sALmw6T/yxiASewa4TQcFsVYZQQRUQJhK github.com/nishanths/exhaustive v0.12.0/go.mod h1:mEZ95wPIZW+x8kC4TgC+9YCUgiST7ecevsVDTgc2obs= github.com/nishanths/predeclared v0.2.2 h1:V2EPdZPliZymNAn79T8RkNApBjMmVKh5XRpLm/w98Vk= github.com/nishanths/predeclared v0.2.2/go.mod h1:RROzoN6TnGQupbC+lqggsOlcgysk3LMK/HI84Mp280c= -github.com/nunnatsa/ginkgolinter v0.15.1 h1:7UMa3GBehavl0/6etaHPvPtslCRqvxgKRLZ2s5RYnDU= -github.com/nunnatsa/ginkgolinter v0.15.1/go.mod h1:qjfgpFK2AdLpFFJUuGcpf+WFeWF5I8iJW7RUpU/X74A= +github.com/nunnatsa/ginkgolinter v0.15.2 h1:N2ORxUxPU56R9gsfLIlVVvCv/V/VVou5qVI1oBKBNHg= +github.com/nunnatsa/ginkgolinter v0.15.2/go.mod h1:oYxE7dt1vZI8cK2rZOs3RgTaBN2vggkqnENmoJ8kVvc= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= @@ -422,8 +421,8 @@ github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xl github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= -github.com/otiai10/copy v1.11.0 h1:OKBD80J/mLBrwnzXqGtFCzprFSGioo30JcmR4APsNwc= -github.com/otiai10/copy v1.11.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs= github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= @@ -436,8 +435,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/polyfloyd/go-errorlint v1.4.7 h1:BI/hD59Rpapkj68yQsOF3ai9UkGDRrWgO47eqt38RKY= -github.com/polyfloyd/go-errorlint v1.4.7/go.mod h1:WGkLzUkLXGGr6BfD33l7yRidBa+T+8xB8TupCpId2ZE= +github.com/polyfloyd/go-errorlint v1.4.8 h1:jiEjKDH33ouFktyez7sckv6pHWif9B7SuS8cutDXFHw= +github.com/polyfloyd/go-errorlint v1.4.8/go.mod h1:NNCxFcFjZcw3xNjVdCchERkEM6Oz7wta2XJVxRftwO4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -590,8 +589,10 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= gitlab.com/bosi/decorder v0.4.1 h1:VdsdfxhstabyhZovHafFw+9eJ6eU0d2CkFNJcZz/NU4= gitlab.com/bosi/decorder v0.4.1/go.mod h1:jecSqWUew6Yle1pCr2eLWTensJMmsxHsBwt+PVbkAqA= -go-simpler.org/assert v0.6.0 h1:QxSrXa4oRuo/1eHMXSBFHKvJIpWABayzKldqZyugG7E= -go-simpler.org/assert v0.6.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= +go-simpler.org/assert v0.7.0 h1:OzWWZqfNxt8cLS+MlUp6Tgk1HjPkmgdKBq9qvy8lZsA= +go-simpler.org/assert v0.7.0/go.mod h1:74Eqh5eI6vCK6Y5l3PI8ZYFXG4Sa+tkr70OIPJAUr28= +go-simpler.org/musttag v0.8.0 h1:DR4UTgetNNhPRNo02rkK1hwDTRzAPotN+ZqYpdtEwWc= +go-simpler.org/musttag v0.8.0/go.mod h1:fiNdCkXt2S6je9Eblma3okjnlva9NT1Eg/WUt19rWu8= go-simpler.org/sloglint v0.4.0 h1:UVJuUJo63iNQNFEOtZ6o1xAgagVg/giVLLvG9nNLobI= go-simpler.org/sloglint v0.4.0/go.mod h1:v6zJ++j/thFPhefs2wEXoCKwT10yo5nkBDYRCXyqgNQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -599,8 +600,6 @@ go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.tmz.dev/musttag v0.7.2 h1:1J6S9ipDbalBSODNT5jCep8dhZyMr4ttnjQagmGYR5s= -go.tmz.dev/musttag v0.7.2/go.mod h1:m6q5NiiSKMnQYokefa2xGoyoXnrswCbJ0AWYzf4Zs28= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -625,12 +624,12 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp/typeparams v0.0.0-20220428152302-39d4317da171/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/exp/typeparams v0.0.0-20230203172020-98cc5a0785f9/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= -golang.org/x/exp/typeparams v0.0.0-20231214170342-aacd6d4b4611 h1:1ZQbh1EPVJYmWuDORTlWzj5uM76NXdNrdxMBidSVOwk= -golang.org/x/exp/typeparams v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= +golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 h1:UhRVJ0i7bF9n/Hd8YjW3eKjlPVBHzbQdxrBgjbSKl64= +golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848/go.mod h1:AbB0pIl9nAr9wVwH+Z2ZpaocVmF5I4GyWCDIsVjR0bk= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -997,8 +996,8 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wp mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo= mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4= -mvdan.cc/unparam v0.0.0-20230917202934-3ee2d22f45fb h1:xiF91GJnDSbyPdiZB5d52N2VpZfGhjM4Ji75cjzuooQ= -mvdan.cc/unparam v0.0.0-20230917202934-3ee2d22f45fb/go.mod h1:ZzZjEpJDOmx8TdVU6umamY3Xy0UAQUI2DHbf05USVbI= +mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14 h1:zCr3iRRgdk5eIikZNDphGcM6KGVTx3Yu+/Uu9Es254w= +mvdan.cc/unparam v0.0.0-20240104100049-c549a3470d14/go.mod h1:ZzZjEpJDOmx8TdVU6umamY3Xy0UAQUI2DHbf05USVbI= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From fea43a3a75fca67841da99306778883a7a067231 Mon Sep 17 00:00:00 2001 From: Tahmid Alam Date: Thu, 8 Feb 2024 09:01:16 -0800 Subject: [PATCH 31/31] Cilium Load Test with Hubble Enabled. (#2558) * load test with huble enabled config file fix config file fix config file path fix fix: typo removing socket path to use default enabling hubble with parameters chaging the directory for cilium removing extra space setting version for cilium cleaning up exporting cilium image version replacing cilium hubble version var * addressed comments * changing daemonset dir to v1.14.4 * replaced variable with version --- .../cilium-overlay-load-test-template.yaml | 103 +++++++++++++----- .pipelines/cni/pipeline.yaml | 13 +++ .../cilium/cilium-config-hubble.yaml | 1 - 3 files changed, 86 insertions(+), 31 deletions(-) diff --git a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml index 93253051029..1963ed3d65f 100644 --- a/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml +++ b/.pipelines/cni/cilium/cilium-overlay-load-test-template.yaml @@ -8,6 +8,7 @@ parameters: os: "linux" arch: "" osSKU: Ubuntu + hubbleEnabled: false # Condition confirms that: # Previous job has reported Succeeded. Previous job is currently setup which controls variable assignment and we are dependent on its success. @@ -48,38 +49,80 @@ stages: - setup displayName: "Cilium Test - ${{ parameters.name }}" jobs: - - job: deploy_cilium_components - steps: - - task: AzureCLI@1 - displayName: "Install Cilium, CNS, and ip-masq-agent" - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - set -ex - az extension add --name aks-preview - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - ls -lah - pwd - kubectl cluster-info - kubectl get po -owide -A + - ${{if eq(parameters.hubbleEnabled, false)}}: + - job: deploy_cilium_components + displayName: Deploy Cilium + steps: + - task: AzureCLI@1 + displayName: "Install Cilium, CNS, and ip-masq-agent" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml + + echo "install Cilium onto Overlay Cluster" + kubectl apply -f test/integration/manifests/cilium/cilium-agent + kubectl apply -f test/integration/manifests/cilium/cilium-operator + + + echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - + kubectl get po -owide -A + + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + kubectl get po -owide -A + - ${{if eq(parameters.hubbleEnabled, true)}}: + - job: deploy_cilium_components + displayName: Deploy Cilium with Hubble + steps: + - task: AzureCLI@1 + displayName: "Install Cilium, CNS, and ip-masq-agent" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -ex + az extension add --name aks-preview + make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A - echo "deploy Cilium ConfigMap" - kubectl apply -f test/integration/manifests/cilium/cilium-config.yaml - echo "install Cilium onto Overlay Cluster" - kubectl apply -f test/integration/manifests/cilium/cilium-agent - kubectl apply -f test/integration/manifests/cilium/cilium-operator - echo "install Cilium ${CILIUM_VERSION_TAG} onto Overlay Cluster" - # Passes Cilium image to daemonset and deployment - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/daemonset.yaml | kubectl apply -f - - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/deployment.yaml | kubectl apply -f - - kubectl get po -owide -A + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-config/cilium-config-hubble.yaml + + echo "install Cilium onto Overlay Cluster with hubble enabled" + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-agent/files + kubectl apply -f test/integration/manifests/cilium/v1.14.4/cilium-operator/files + + echo "install Cilium v1.14.4 onto Overlay Cluster" + # Passes Cilium image to daemonset and deployment + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-agent/templates/daemonset.tpl | kubectl apply -f - + envsubst '${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v1.14.4/cilium-operator/templates/deployment.tpl | kubectl apply -f - + kubectl get po -owide -A - echo "Deploy Azure-CNS" - sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) - kubectl get po -owide -A + echo "Deploy Azure-CNS" + sudo -E env "PATH=$PATH" make test-integration AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) INSTALL_CNS=true INSTALL_OVERLAY=true CNS_IMAGE_REPO=$(CNS_IMAGE_REPO) + kubectl get po -owide -A + - job: deploy_pods condition: and( and( not(canceled()), not(failed()) ), or( contains(variables.CONTROL_SCENARIO, 'scaleTest') , contains(variables.CONTROL_SCENARIO, 'all') ) ) displayName: "Scale Test" diff --git a/.pipelines/cni/pipeline.yaml b/.pipelines/cni/pipeline.yaml index afa1cc128d2..f917b9bae37 100644 --- a/.pipelines/cni/pipeline.yaml +++ b/.pipelines/cni/pipeline.yaml @@ -162,6 +162,15 @@ stages: nodeCount: ${NODE_COUNT_CILIUM} vmSize: ${VM_SIZE_CILIUM} + - template: cilium/cilium-overlay-load-test-template.yaml + parameters: + name: cilium_overlay_hubble + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cil-over-hub" + hubbleEnabled: true + nodeCount: ${NODE_COUNT_CILIUM} + vmSize: ${VM_SIZE_CILIUM} + - template: cilium/cilium-overlay-load-test-template.yaml parameters: name: cilium_overlay_mariner @@ -269,6 +278,7 @@ stages: condition: always() dependsOn: - cilium_overlay + - cilium_overlay_hubble - cilium_overlay_mariner - cilium_overlay_arm - cilium_overlay_rdma @@ -292,6 +302,9 @@ stages: cilium_overlay: name: cilium_overlay clusterName: "cilium-over" + cilium_overlay_hubble: + name: cilium_overlay_hubble + clusterName: "cil-over-hub" cilium_overlay_mariner: name: cilium_overlay_mariner clusterName: "cil-over-mar" diff --git a/test/integration/manifests/cilium/cilium-config-hubble.yaml b/test/integration/manifests/cilium/cilium-config-hubble.yaml index 492fc08bfcd..63115ae5ff0 100644 --- a/test/integration/manifests/cilium/cilium-config-hubble.yaml +++ b/test/integration/manifests/cilium/cilium-config-hubble.yaml @@ -49,7 +49,6 @@ data: hubble-metrics-server: :9965 hubble-disable-tls: "false" hubble-listen-address: "" - hubble-socket-path: /dev/null hubble-tls-cert-file: /var/lib/cilium/tls/hubble/server.crt hubble-tls-client-ca-files: /var/lib/cilium/tls/hubble/client-ca.crt hubble-tls-key-file: /var/lib/cilium/tls/hubble/server.key