From 227ff33e4e43a5d9f295ae61cb517687ce2d701a Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Wed, 17 Jan 2024 15:18:47 -0800 Subject: [PATCH 1/5] Modyfying stateless CNI state to account for swift 2.0 changes --- cns/api.go | 1 + cns/client/client.go | 3 +- cns/client/client_test.go | 14 +++++-- cns/restserver/ipam.go | 15 +++++++- cns/restserver/restserver.go | 9 +++-- network/endpoint.go | 1 + network/manager.go | 72 ++++++++++++++++++++++++++++-------- 7 files changed, 89 insertions(+), 26 deletions(-) diff --git a/cns/api.go b/cns/api.go index 3894e9aff6..39d5a23a87 100644 --- a/cns/api.go +++ b/cns/api.go @@ -363,4 +363,5 @@ type GetHomeAzResponse struct { type EndpointRequest struct { HnsEndpointID string `json:"hnsEndpointID"` HostVethName string `json:"hostVethName"` + InterfaceName string `json:"InterfaceName"` } diff --git a/cns/client/client.go b/cns/client/client.go index 2ee524cfb9..85ca72b765 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1059,11 +1059,12 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserve // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName, ifName string) (*cns.Response, error) { // build the request updateEndpoint := cns.EndpointRequest{ HnsEndpointID: hnsID, HostVethName: vethName, + InterfaceName: ifName, } var body bytes.Buffer diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 65c427c844..17bcb724db 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2710,6 +2710,7 @@ func TestUpdateEndpoint(t *testing.T) { containerID string hnsID string vethName string + ifName string response *RequestCapture expReq *cns.EndpointRequest shouldErr bool @@ -2719,6 +2720,7 @@ func TestUpdateEndpoint(t *testing.T) { "", "", "", + "", &RequestCapture{ Next: &mockdo{}, }, @@ -2730,6 +2732,7 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "bar", "", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, @@ -2737,6 +2740,7 @@ func TestUpdateEndpoint(t *testing.T) { }, &cns.EndpointRequest{ HnsEndpointID: "bar", + InterfaceName: "eth0", }, false, }, @@ -2745,13 +2749,15 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusOK, }, }, &cns.EndpointRequest{ - HostVethName: "bar", + HostVethName: "bar", + InterfaceName: "eth0", }, false, }, @@ -2760,13 +2766,15 @@ func TestUpdateEndpoint(t *testing.T) { "foo", "", "bar", + "eth0", &RequestCapture{ Next: &mockdo{ httpStatusCodeToReturn: http.StatusBadRequest, }, }, &cns.EndpointRequest{ - HostVethName: "bar", + HostVethName: "bar", + InterfaceName: "eth0", }, true, }, @@ -2784,7 +2792,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 2e1115039c..bb4459775f 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1109,6 +1109,17 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * logger.Response(service.Name, response, response.ReturnCode, err) return } + if req.InterfaceName == "" { + logger.Warnf("[updateEndpoint] No Interface has been provided") + response := cns.Response{ + ReturnCode: types.InvalidRequest, + Message: "[updateEndpoint] No Interface has been provided", + } + w.Header().Set(cnsReturnCode, response.ReturnCode.String()) + err = service.Listener.Encode(w, &response) + logger.Response(service.Name, response, response.ReturnCode, err) + return + } // Update the endpoint state err = service.UpdateEndpointHelper(endpointID, req) if err != nil { @@ -1139,11 +1150,11 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns. if endpointInfo, ok := service.EndpointState[endpointID]; ok { logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) if req.HnsEndpointID != "" { - service.EndpointState[endpointID].HnsEndpointID = req.HnsEndpointID + service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HnsEndpointID = req.HnsEndpointID logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, req.HnsEndpointID) } if req.HostVethName != "" { - service.EndpointState[endpointID].HostVethName = req.HostVethName + service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HostVethName = req.HostVethName logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, req.HostVethName) } diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index d8e9604a7b..fc54046839 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -93,13 +93,14 @@ type EndpointInfo struct { PodName string PodNamespace string IfnameToIPMap map[string]*IPInfo // key : interface name, value : IPInfo - HnsEndpointID string - HostVethName string } type IPInfo struct { - IPv4 []net.IPNet - IPv6 []net.IPNet + IPv4 []net.IPNet + IPv6 []net.IPNet + HnsEndpointID string + HostVethName string + NICType cns.NICType } type GetHTTPServiceDataResponse struct { diff --git a/network/endpoint.go b/network/endpoint.go index db2c2cf01e..0e489d719a 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -93,6 +93,7 @@ type EndpointInfo struct { SkipDefaultRoutes bool HNSEndpointID string HostIfName string + SecondaryInterfaces map[string]*InterfaceInfo } // RouteInfo contains information about an IP route. diff --git a/network/manager.go b/network/manager.go index d0b6c90713..7b8d0ca85c 100644 --- a/network/manager.go +++ b/network/manager.go @@ -10,6 +10,7 @@ import ( "time" cnsclient "github.com/Azure/azure-container-networking/cns/client" + "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/common" "github.com/Azure/azure-container-networking/log" "github.com/Azure/azure-container-networking/netio" @@ -422,7 +423,7 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -437,28 +438,15 @@ func (nm *networkManager) GetEndpointState(networkID, endpointID string) (*Endpo if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } - epInfo := &EndpointInfo{ - Id: endpointID, - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - IfName: endpointResponse.EndpointInfo.HostVethName, - ContainerID: endpointID, - PODName: endpointResponse.EndpointInfo.PodName, - PODNameSpace: endpointResponse.EndpointInfo.PodNamespace, - NetworkContainerID: endpointID, - HNSEndpointID: endpointResponse.EndpointInfo.HnsEndpointID, - } - - for _, ip := range endpointResponse.EndpointInfo.IfnameToIPMap { - epInfo.IPAddresses = ip.IPv4 - epInfo.IPAddresses = append(epInfo.IPAddresses, ip.IPv6...) + epInfo := cnsEndpointInfotoCNIEpInfo(endpointResponse.EndpointInfo, endpointID) - } if epInfo.IsEndpointStateIncomplete() { epInfo, err = epInfo.GetEndpointInfoByIPImpl(epInfo.IPAddresses, networkID) if err != nil { return nil, errors.Wrapf(err, "Get endpoint API returend with error") } } + logger.Info("returning getEndpoint API with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", epInfo.HNSEndpointID)) return epInfo, nil } @@ -512,6 +500,7 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint EnableSnatOnHost: false, EnableMultitenancy: false, NetworkContainerID: epInfo.Id, + SecondaryInterfaces: epInfo.SecondaryInterfaces, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) @@ -698,3 +687,54 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { } return containerID + "-" + ifName } + +func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID string) *EndpointInfo { + epInfo := &EndpointInfo{ + Id: endpointID, + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + ContainerID: endpointID, + PODName: endpointInfo.PodName, + PODNameSpace: endpointInfo.PodNamespace, + NetworkContainerID: endpointID, + SecondaryInterfaces: make(map[string]*InterfaceInfo), + } + // filling out the InfraNIC from the state + for ifName, ipInfo := range endpointInfo.IfnameToIPMap { + if ifName == InfraInterfaceName { + epInfo.IPAddresses = ipInfo.IPv4 + epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) + epInfo.IfName = ifName + epInfo.HostIfName = ipInfo.HostVethName + epInfo.HNSEndpointID = ipInfo.HnsEndpointID + } else { // filling out the SecondaryNICs from the state + interfaceInfo := &InterfaceInfo{ + Name: ifName, + IPConfigs: generateIPConfigfromState(ipInfo), + NICType: ipInfo.NICType, + } + epInfo.SecondaryInterfaces[ifName] = interfaceInfo + } + } + return epInfo +} + +func generateIPConfigfromState(ipInfo *restserver.IPInfo) []*IPConfig { + ipConfigs := []*IPConfig{} + for _, ip := range ipInfo.IPv4 { + ipConfig := &IPConfig{ + Address: net.IPNet{ + IP: ip.IP, + }, + } + ipConfigs = append(ipConfigs, ipConfig) + } + for _, ip := range ipInfo.IPv6 { + ipConfig := &IPConfig{ + Address: net.IPNet{ + IP: ip.IP, + }, + } + ipConfigs = append(ipConfigs, ipConfig) + } + return ipConfigs +} From 40bd8d9cbdde3d7f2caf6e1c7b3e72f25e53eea3 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 28 Mar 2024 10:42:22 -0700 Subject: [PATCH 2/5] Removing SecondaryNICInfor fro EPInfo. --- network/endpoint.go | 1 - network/manager.go | 34 +++++++++++++--------------------- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/network/endpoint.go b/network/endpoint.go index 0e489d719a..db2c2cf01e 100644 --- a/network/endpoint.go +++ b/network/endpoint.go @@ -93,7 +93,6 @@ type EndpointInfo struct { SkipDefaultRoutes bool HNSEndpointID string HostIfName string - SecondaryInterfaces map[string]*InterfaceInfo } // RouteInfo contains information about an IP route. diff --git a/network/manager.go b/network/manager.go index 7b8d0ca85c..337f87f040 100644 --- a/network/manager.go +++ b/network/manager.go @@ -71,7 +71,7 @@ type EndpointClient interface { // NetworkManager manages the set of container networking resources. type networkManager struct { - StatelessCniMode bool + statelessCniMode bool CnsClient *cnsclient.Client Version string TimeStamp time.Time @@ -150,7 +150,7 @@ func (nm *networkManager) Uninitialize() { // SetStatelessCNIMode enable the statelessCNI falg and inititlizes a CNSClient func (nm *networkManager) SetStatelessCNIMode() error { - nm.StatelessCniMode = true + nm.statelessCniMode = true // Create CNS client client, err := cnsclient.New(cnsBaseURL, cnsReqTimeout) if err != nil { @@ -162,7 +162,7 @@ func (nm *networkManager) SetStatelessCNIMode() error { // IsStatelessCNIMode checks if the Stateless CNI mode has been enabled or not func (nm *networkManager) IsStatelessCNIMode() bool { - return nm.StatelessCniMode + return nm.statelessCniMode } // Restore reads network manager state from persistent store. @@ -500,7 +500,6 @@ func (nm *networkManager) DeleteEndpointState(networkID string, epInfo *Endpoint EnableSnatOnHost: false, EnableMultitenancy: false, NetworkContainerID: epInfo.Id, - SecondaryInterfaces: epInfo.SecondaryInterfaces, } logger.Info("Deleting endpoint with", zap.String("Endpoint Info: ", epInfo.PrettyString()), zap.String("HNISID : ", ep.HnsId)) return nw.deleteEndpointImpl(netlink.NewNetlink(), platform.NewExecClient(logger), nil, nil, nil, nil, ep) @@ -690,30 +689,23 @@ func (nm *networkManager) GetEndpointID(containerID, ifName string) string { func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID string) *EndpointInfo { epInfo := &EndpointInfo{ - Id: endpointID, - IfIndex: EndpointIfIndex, // Azure CNI supports only one interface - ContainerID: endpointID, - PODName: endpointInfo.PodName, - PODNameSpace: endpointInfo.PodNamespace, - NetworkContainerID: endpointID, - SecondaryInterfaces: make(map[string]*InterfaceInfo), - } - // filling out the InfraNIC from the state + Id: endpointID, + IfIndex: EndpointIfIndex, // Azure CNI supports only one interface + ContainerID: endpointID, + PODName: endpointInfo.PodName, + PODNameSpace: endpointInfo.PodNamespace, + NetworkContainerID: endpointID, + } + for ifName, ipInfo := range endpointInfo.IfnameToIPMap { - if ifName == InfraInterfaceName { + if ifName == InfraInterfaceName { // filling out the InfraNIC from the state epInfo.IPAddresses = ipInfo.IPv4 epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) epInfo.IfName = ifName epInfo.HostIfName = ipInfo.HostVethName epInfo.HNSEndpointID = ipInfo.HnsEndpointID - } else { // filling out the SecondaryNICs from the state - interfaceInfo := &InterfaceInfo{ - Name: ifName, - IPConfigs: generateIPConfigfromState(ipInfo), - NICType: ipInfo.NICType, - } - epInfo.SecondaryInterfaces[ifName] = interfaceInfo } + // TODO: filling out the SecondaryNICs from the state for Swift 2.0 } return epInfo } From 2d13070a1ce39fd5ef917be72a084a0e901f09cc Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Thu, 28 Mar 2024 15:36:12 -0700 Subject: [PATCH 3/5] removing SecondaryNic from the epInfo. --- network/manager.go | 37 +++++++++---------------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/network/manager.go b/network/manager.go index 337f87f040..c4e3bd0674 100644 --- a/network/manager.go +++ b/network/manager.go @@ -698,35 +698,16 @@ func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID } for ifName, ipInfo := range endpointInfo.IfnameToIPMap { - if ifName == InfraInterfaceName { // filling out the InfraNIC from the state - epInfo.IPAddresses = ipInfo.IPv4 - epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) - epInfo.IfName = ifName - epInfo.HostIfName = ipInfo.HostVethName - epInfo.HNSEndpointID = ipInfo.HnsEndpointID + if ifName != InfraInterfaceName { + // TODO: filling out the SecondaryNICs from the state for Swift 2.0 + continue } - // TODO: filling out the SecondaryNICs from the state for Swift 2.0 + // filling out the InfraNIC from the state + epInfo.IPAddresses = ipInfo.IPv4 + epInfo.IPAddresses = append(epInfo.IPAddresses, ipInfo.IPv6...) + epInfo.IfName = ifName + epInfo.HostIfName = ipInfo.HostVethName + epInfo.HNSEndpointID = ipInfo.HnsEndpointID } return epInfo } - -func generateIPConfigfromState(ipInfo *restserver.IPInfo) []*IPConfig { - ipConfigs := []*IPConfig{} - for _, ip := range ipInfo.IPv4 { - ipConfig := &IPConfig{ - Address: net.IPNet{ - IP: ip.IP, - }, - } - ipConfigs = append(ipConfigs, ipConfig) - } - for _, ip := range ipInfo.IPv6 { - ipConfig := &IPConfig{ - Address: net.IPNet{ - IP: ip.IP, - }, - } - ipConfigs = append(ipConfigs, ipConfig) - } - return ipConfigs -} From 7cdbaf52b052adc2a3e34f6d655628a030e5c44a Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Fri, 12 Apr 2024 11:06:40 -0700 Subject: [PATCH 4/5] Make change to UpdateEndpointState API to support SwiftV2 for Stateless CNI --- cns/client/client.go | 9 ++----- cns/client/client_test.go | 32 +++++++++++++--------- cns/restserver/ipam.go | 57 ++++++++++++++++++++++----------------- network/manager.go | 20 +++++++++++++- 4 files changed, 72 insertions(+), 46 deletions(-) diff --git a/cns/client/client.go b/cns/client/client.go index 85ca72b765..804580b8d8 100644 --- a/cns/client/client.go +++ b/cns/client/client.go @@ -1059,16 +1059,11 @@ func (c *Client) GetEndpoint(ctx context.Context, endpointID string) (*restserve // UpdateEndpoint calls the EndpointHandlerAPI in CNS // to update the state of a given EndpointID with either HNSEndpointID or HostVethName -func (c *Client) UpdateEndpoint(ctx context.Context, endpointID, hnsID, vethName, ifName string) (*cns.Response, error) { +func (c *Client) UpdateEndpoint(ctx context.Context, endpointID string, ipInfo map[string]*restserver.IPInfo) (*cns.Response, error) { // build the request - updateEndpoint := cns.EndpointRequest{ - HnsEndpointID: hnsID, - HostVethName: vethName, - InterfaceName: ifName, - } var body bytes.Buffer - if err := json.NewEncoder(&body).Encode(updateEndpoint); err != nil { + if err := json.NewEncoder(&body).Encode(ipInfo); err != nil { return nil, errors.Wrap(err, "failed to encode updateEndpoint") } diff --git a/cns/client/client_test.go b/cns/client/client_test.go index 17bcb724db..b10a47f8a5 100644 --- a/cns/client/client_test.go +++ b/cns/client/client_test.go @@ -2712,7 +2712,7 @@ func TestUpdateEndpoint(t *testing.T) { vethName string ifName string response *RequestCapture - expReq *cns.EndpointRequest + expReq map[string]*restserver.IPInfo shouldErr bool }{ { @@ -2738,9 +2738,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusOK, }, }, - &cns.EndpointRequest{ - HnsEndpointID: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HnsEndpointID: "bar", + NICType: cns.InfraNIC, + }, }, false, }, @@ -2755,9 +2757,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusOK, }, }, - &cns.EndpointRequest{ - HostVethName: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HostVethName: "bar", + NICType: cns.InfraNIC, + }, }, false, }, @@ -2772,9 +2776,11 @@ func TestUpdateEndpoint(t *testing.T) { httpStatusCodeToReturn: http.StatusBadRequest, }, }, - &cns.EndpointRequest{ - HostVethName: "bar", - InterfaceName: "eth0", + map[string]*restserver.IPInfo{ + "eth0": { + HostVethName: "bar", + NICType: cns.InfraNIC, + }, }, true, }, @@ -2792,7 +2798,7 @@ func TestUpdateEndpoint(t *testing.T) { } // execute the method under test - res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.hnsID, test.vethName, test.ifName) + res, err := client.UpdateEndpoint(context.TODO(), test.containerID, test.expReq) if err != nil && !test.shouldErr { t.Fatal("unexpected error: err: ", err, res.Message) } @@ -2809,7 +2815,7 @@ func TestUpdateEndpoint(t *testing.T) { // if a request was expected to be sent, decode it and ensure that it // matches expectations if test.expReq != nil { - var gotReq cns.EndpointRequest + var gotReq map[string]*restserver.IPInfo err = json.NewDecoder(test.response.Request.Body).Decode(&gotReq) if err != nil { t.Fatal("error decoding the received request: err:", err) @@ -2818,7 +2824,7 @@ func TestUpdateEndpoint(t *testing.T) { // a nil expReq is semantically meaningful (i.e. "no request"), but in // order for cmp to work properly, the outer types should be identical. // Thus we have to dereference it explicitly: - expReq := *test.expReq + expReq := test.expReq // ensure that the received request is what was expected if !cmp.Equal(gotReq, expReq) { diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index bb4459775f..420bbb13f4 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1083,7 +1083,7 @@ func (service *HTTPRestService) GetEndpointHelper(endpointID string) (*EndpointI func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r *http.Request) { logger.Printf("[updateEndpoint] updateEndpoint for %s", r.URL.Path) - var req cns.EndpointRequest + var req map[string]*IPInfo err := service.Listener.Decode(w, r, &req) endpointID := strings.TrimPrefix(r.URL.Path, cns.EndpointPath) logger.Request(service.Name, &req, err) @@ -1098,22 +1098,10 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * logger.Response(service.Name, response, response.ReturnCode, err) return } - if req.HostVethName == "" && req.HnsEndpointID == "" { - logger.Warnf("[updateEndpoint] No HnsEndpointID or HostVethName has been provided") + if err = verifyUpdateEndpointStateRequest(req); err != nil { response := cns.Response{ ReturnCode: types.InvalidRequest, - Message: "[updateEndpoint] No HnsEndpointID or HostVethName has been provided", - } - w.Header().Set(cnsReturnCode, response.ReturnCode.String()) - err = service.Listener.Encode(w, &response) - logger.Response(service.Name, response, response.ReturnCode, err) - return - } - if req.InterfaceName == "" { - logger.Warnf("[updateEndpoint] No Interface has been provided") - response := cns.Response{ - ReturnCode: types.InvalidRequest, - Message: "[updateEndpoint] No Interface has been provided", + Message: err.Error(), } w.Header().Set(cnsReturnCode, response.ReturnCode.String()) err = service.Listener.Encode(w, &response) @@ -1142,22 +1130,27 @@ func (service *HTTPRestService) UpdateEndpointHandler(w http.ResponseWriter, r * } // UpdateEndpointHelper updates the state of the given endpointId with HNSId or VethName -func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns.EndpointRequest) error { +func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[string]*IPInfo) error { if service.EndpointStateStore == nil { return ErrStoreEmpty } logger.Printf("[updateEndpoint] Updating endpoint state for infra container %s", endpointID) if endpointInfo, ok := service.EndpointState[endpointID]; ok { - logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) - if req.HnsEndpointID != "" { - service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HnsEndpointID = req.HnsEndpointID - logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, req.HnsEndpointID) - } - if req.HostVethName != "" { - service.EndpointState[endpointID].IfnameToIPMap[req.InterfaceName].HostVethName = req.HostVethName - logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, req.HostVethName) + for ifName, InterfaceInfo := range req { + logger.Printf("[updateEndpoint] Found existing endpoint state for infra container %s", endpointID) + if InterfaceInfo.HnsEndpointID != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].HnsEndpointID = InterfaceInfo.HnsEndpointID + logger.Printf("[updateEndpoint] update the endpoint %s with HNSID %s", endpointID, InterfaceInfo.HnsEndpointID) + } + if InterfaceInfo.HostVethName != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].HostVethName = InterfaceInfo.HostVethName + logger.Printf("[updateEndpoint] update the endpoint %s with vethName %s", endpointID, InterfaceInfo.HostVethName) + } + if InterfaceInfo.NICType != "" { + service.EndpointState[endpointID].IfnameToIPMap[ifName].NICType = InterfaceInfo.NICType + logger.Printf("[updateEndpoint] update the endpoint %s with NICType %s", endpointID, InterfaceInfo.NICType) + } } - err := service.EndpointStateStore.Write(EndpointStoreKey, service.EndpointState) if err != nil { return fmt.Errorf("[updateEndpoint] failed to write endpoint state to store for pod %s : %w", endpointInfo.PodName, err) @@ -1165,4 +1158,18 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req cns. return nil } return errors.New("[updateEndpoint] endpoint could not be found in the statefile") + +} + +// verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API +func verifyUpdateEndpointStateRequest(req map[string]*IPInfo) error { + for ifName, InterfaceInfo := range req { + if InterfaceInfo.HostVethName == "" && InterfaceInfo.HnsEndpointID == "" && InterfaceInfo.NICType == "" { + return errors.New("[updateEndpoint] No NicType, HnsEndpointID or HostVethName has been provided") + } + if ifName == "" { + return errors.New("[updateEndpoint] No Interface has been provided") + } + } + return nil } diff --git a/network/manager.go b/network/manager.go index c4e3bd0674..d5f3fd960d 100644 --- a/network/manager.go +++ b/network/manager.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/Azure/azure-container-networking/cns" cnsclient "github.com/Azure/azure-container-networking/cns/client" "github.com/Azure/azure-container-networking/cns/restserver" "github.com/Azure/azure-container-networking/common" @@ -422,8 +423,9 @@ func (nm *networkManager) CreateEndpoint(cli apipaClient, networkID string, epIn // UpdateEndpointState will make a call to CNS updatEndpointState API in the stateless CNI mode // It will add HNSEndpointID or HostVeth name to the endpoint state func (nm *networkManager) UpdateEndpointState(ep *endpoint) error { + ifnameToIPInfoMap := generateCNSIPInfoMap(ep) // key : interface name, value : IPInfo logger.Info("Calling cns updateEndpoint API with ", zap.String("containerID: ", ep.ContainerID), zap.String("HnsId: ", ep.HnsId), zap.String("HostIfName: ", ep.HostIfName)) - response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ep.HnsId, ep.HostIfName, ep.IfName) + response, err := nm.CnsClient.UpdateEndpoint(context.TODO(), ep.ContainerID, ifnameToIPInfoMap) if err != nil { return errors.Wrapf(err, "Update endpoint API returend with error") } @@ -711,3 +713,19 @@ func cnsEndpointInfotoCNIEpInfo(endpointInfo restserver.EndpointInfo, endpointID } return epInfo } + +func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo { + ifNametoIPInfoMap := make(map[string]*restserver.IPInfo) // key : interface name, value : IPInfo + if ep.IfName != "" { + ifNametoIPInfoMap[ep.IfName].NICType = cns.InfraNIC + ifNametoIPInfoMap[ep.IfName].HnsEndpointID = ep.HnsId + ifNametoIPInfoMap[ep.IfName].HostVethName = ep.HostIfName + } + if ep.SecondaryInterfaces != nil { + for ifName, InterfaceInfo := range ep.SecondaryInterfaces { + ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType + } + + } + return ifNametoIPInfoMap +} From c7e9b452856bf08f9d54b175fddb02277fdd7195 Mon Sep 17 00:00:00 2001 From: AzureAhai Date: Fri, 19 Apr 2024 13:56:50 -0700 Subject: [PATCH 5/5] updating Makefile to include azure CNI binary. --- Makefile | 14 +++++++++++++- cns/restserver/ipam.go | 1 - cns/restserver/restserver.go | 6 +++--- network/manager.go | 1 - 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 38823b578d..90a723d753 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ AZURE_IPAM_DIR = $(REPO_ROOT)/azure-ipam CNM_DIR = $(REPO_ROOT)/cnm/plugin CNI_NET_DIR = $(REPO_ROOT)/cni/network/plugin CNI_IPAM_DIR = $(REPO_ROOT)/cni/ipam/plugin +STATELESS_CNI_NET_DIR = $(REPO_ROOT)/cni/network/stateless CNI_IPAMV6_DIR = $(REPO_ROOT)/cni/ipam/pluginv6 CNI_TELEMETRY_DIR = $(REPO_ROOT)/cni/telemetry/service ACNCLI_DIR = $(REPO_ROOT)/tools/acncli @@ -59,10 +60,13 @@ IMAGE_DIR = $(OUTPUT_DIR)/images CNM_BUILD_DIR = $(BUILD_DIR)/cnm CNI_BUILD_DIR = $(BUILD_DIR)/cni ACNCLI_BUILD_DIR = $(BUILD_DIR)/acncli +STATELESS_CNI_BUILD_DIR = $(CNI_BUILD_DIR)/stateless CNI_MULTITENANCY_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy CNI_MULTITENANCY_TRANSPARENT_VLAN_BUILD_DIR = $(BUILD_DIR)/cni-multitenancy-transparent-vlan CNI_SWIFT_BUILD_DIR = $(BUILD_DIR)/cni-swift CNI_OVERLAY_BUILD_DIR = $(BUILD_DIR)/cni-overlay +STATELESS_CNI_OVERLAY_BUILD_DIR = $(CNI_OVERLAY_BUILD_DIR)/stateless +STATELESS_CNI_SWIFT_BUILD_DIR = $(CNI_SWIFT_BUILD_DIR)/stateless CNI_BAREMETAL_BUILD_DIR = $(BUILD_DIR)/cni-baremetal CNI_DUALSTACK_BUILD_DIR = $(BUILD_DIR)/cni-dualstack CNS_BUILD_DIR = $(BUILD_DIR)/cns @@ -130,7 +134,7 @@ endif # Shorthand target names for convenience. azure-cnm-plugin: cnm-binary cnm-archive -azure-cni-plugin: azure-vnet-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive +azure-cni-plugin: azure-vnet-binary azure-vnet-stateless-binary azure-vnet-ipam-binary azure-vnet-ipamv6-binary azure-vnet-telemetry-binary cni-archive azure-cns: azure-cns-binary cns-archive acncli: acncli-binary acncli-archive azure-npm: azure-npm-binary npm-archive @@ -179,6 +183,10 @@ cnm-binary: azure-vnet-binary: cd $(CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" +# Build the Azure CNI stateless network binary +azure-vnet-stateless-binary: + cd $(STATELESS_CNI_NET_DIR) && CGO_ENABLED=0 go build -v -o $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" + # Build the Azure CNI IPAM binary. azure-vnet-ipam-binary: cd $(CNI_IPAM_DIR) && CGO_ENABLED=0 go build -v -o $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) -ldflags "-X main.version=$(CNI_VERSION)" -gcflags="-dwarflocationlists=true" @@ -674,12 +682,16 @@ endif cp cni/azure-$(GOOS)-swift.conflist $(CNI_SWIFT_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_SWIFT_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_SWIFT_BUILD_DIR) + $(MKDIR) $(STATELESS_CNI_SWIFT_BUILD_DIR) + cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_SWIFT_BUILD_DIR) cd $(CNI_SWIFT_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_SWIFT_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_OVERLAY_BUILD_DIR) cp cni/azure-$(GOOS)-swift-overlay.conflist $(CNI_OVERLAY_BUILD_DIR)/10-azure.conflist cp telemetry/azure-vnet-telemetry.config $(CNI_OVERLAY_BUILD_DIR)/azure-vnet-telemetry.config cp $(CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-ipam$(EXE_EXT) $(CNI_BUILD_DIR)/azure-vnet-telemetry$(EXE_EXT) $(CNI_OVERLAY_BUILD_DIR) + $(MKDIR) $(STATELESS_CNI_OVERLAY_BUILD_DIR) + cp $(STATELESS_CNI_BUILD_DIR)/azure-vnet$(EXE_EXT) $(STATELESS_CNI_OVERLAY_BUILD_DIR) cd $(CNI_OVERLAY_BUILD_DIR) && $(ARCHIVE_CMD) $(CNI_OVERLAY_ARCHIVE_NAME) azure-vnet$(EXE_EXT) azure-vnet-ipam$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) 10-azure.conflist azure-vnet-telemetry.config $(MKDIR) $(CNI_DUALSTACK_BUILD_DIR) diff --git a/cns/restserver/ipam.go b/cns/restserver/ipam.go index 420bbb13f4..1031b15c3c 100644 --- a/cns/restserver/ipam.go +++ b/cns/restserver/ipam.go @@ -1158,7 +1158,6 @@ func (service *HTTPRestService) UpdateEndpointHelper(endpointID string, req map[ return nil } return errors.New("[updateEndpoint] endpoint could not be found in the statefile") - } // verifyUpdateEndpointStateRequest verify the CNI request body for the UpdateENdpointState API diff --git a/cns/restserver/restserver.go b/cns/restserver/restserver.go index fc54046839..6fccc0e07a 100644 --- a/cns/restserver/restserver.go +++ b/cns/restserver/restserver.go @@ -97,9 +97,9 @@ type EndpointInfo struct { type IPInfo struct { IPv4 []net.IPNet - IPv6 []net.IPNet - HnsEndpointID string - HostVethName string + IPv6 []net.IPNet `json:",omitempty"` + HnsEndpointID string `json:",omitempty"` + HostVethName string `json:",omitempty"` NICType cns.NICType } diff --git a/network/manager.go b/network/manager.go index d5f3fd960d..d82a95684f 100644 --- a/network/manager.go +++ b/network/manager.go @@ -725,7 +725,6 @@ func generateCNSIPInfoMap(ep *endpoint) map[string]*restserver.IPInfo { for ifName, InterfaceInfo := range ep.SecondaryInterfaces { ifNametoIPInfoMap[ifName].NICType = InterfaceInfo.NICType } - } return ifNametoIPInfoMap }