From 2e34ed9f7297dac8a24e1876af3f5d22c202ea8d Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 30 Jul 2024 15:03:47 -0700 Subject: [PATCH 01/13] Add support for nodes managing their own networks Signed-off-by: nithyar --- dist/images/daemonset.sh | 8 ++++++++ dist/images/ovnkube.sh | 8 ++++++++ dist/templates/ovnkube-master.yaml.j2 | 2 ++ go-controller/pkg/ovn/master.go | 3 +++ .../templates/deployment-ovnkube-master.yaml | 2 ++ helm/ovn-kubernetes/values.yaml | 2 ++ 6 files changed, 25 insertions(+) diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index 2c46442c812..de944cde69e 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -92,6 +92,7 @@ OVN_ENABLE_INTERCONNECT= OVN_ENABLE_OVNKUBE_IDENTITY="true" OVN_ENABLE_PERSISTENT_IPS= OVN_ENABLE_SVC_TEMPLATE_SUPPORT="true" +OVN_NOHOSTSUBNET_LABEL="" # IN_UPGRADE is true only if called by upgrade-ovn.sh during the upgrade test, # it will render only the parts in ovn-setup.yaml related to RBAC permissions. IN_UPGRADE= @@ -346,6 +347,9 @@ while [ "$1" != "" ]; do --enable-svc-template-support) OVN_ENABLE_SVC_TEMPLATE_SUPPORT=$VALUE ;; + --no-hostsubnet-label) + OVN_NOHOSTSUBNET_LABEL=$VALUE + ;; *) echo "WARNING: unknown parameter \"$PARAM\"" exit 1 @@ -531,6 +535,9 @@ echo "ovn_enable_persistent_ips: ${ovn_enable_persistent_ips}" ovn_enable_svc_template_support=${OVN_ENABLE_SVC_TEMPLATE_SUPPORT} echo "ovn_enable_svc_template_support: ${ovn_enable_svc_template_support}" +ovn_nohostsubnet_label=${OVN_NOHOSTSUBNET_LABEL} +echo "ovn_nohostsubnet_label: ${ovn_nohostsubnet_label}" + ovn_image=${ovnkube_image} \ ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ ovn_image_pull_policy=${image_pull_policy} \ @@ -716,6 +723,7 @@ ovn_image=${ovnkube_image} \ ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \ ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \ ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \ + ovn_nohostsubnet_label=${ovn_nohostsubnet_label} \ jinjanate ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml ovn_image=${ovnkube_image} \ diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 1a9cbffb325..48c9b4a3de1 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -305,6 +305,8 @@ ovnkube_compact_mode_enable=${OVNKUBE_COMPACT_MODE_ENABLE:-false} ovn_northd_backoff_interval=${OVN_NORTHD_BACKOFF_INTERVAL:-"300"} # OVN_ENABLE_SVC_TEMPLATE_SUPPORT - enable svc template support ovn_enable_svc_template_support=${OVN_ENABLE_SVC_TEMPLATE_SUPPORT:-true} +# OVN_NOHOSTSUBNET_LABEL - node label indicating nodes managing their own network +ovn_nohostsubnet_label=${OVN_NOHOSTSUBNET_LABEL:-""} # Determine the ovn rundir. if [[ -f /usr/bin/ovn-appctl ]]; then @@ -1247,6 +1249,11 @@ ovn-master() { ovn_enable_svc_template_support_flag="--enable-svc-template-support" fi echo "ovn_enable_svc_template_support_flag=${ovn_enable_svc_template_support_flag}" + + nohostsubnet_label_option= + if [[ ${ovn_nohostsubnet_label} != "" ]]; then + nohostsubnet_label_option="--no-hostsubnet-nodes=${ovn_nohostsubnet_label}" + fi init_node_flags= if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then @@ -1291,6 +1298,7 @@ ovn-master() { ${ovn_v6_join_subnet_opt} \ ${ovn_v6_masquerade_subnet_opt} \ ${persistent_ips_enabled_flag} \ + ${nohostsubnet_label_option} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ --host-network-namespace ${ovn_host_network_namespace} \ diff --git a/dist/templates/ovnkube-master.yaml.j2 b/dist/templates/ovnkube-master.yaml.j2 index 3c67c26a136..a926898728c 100644 --- a/dist/templates/ovnkube-master.yaml.j2 +++ b/dist/templates/ovnkube-master.yaml.j2 @@ -293,6 +293,8 @@ spec: value: "{{ ovn_enable_multi_external_gateway }}" - name: OVN_ENABLE_SVC_TEMPLATE_SUPPORT value: "{{ ovn_enable_svc_template_support }}" + - name: OVN_NOHOSTSUBNET_LABEL + value: "{{ ovn_nohostsubnet_label }}" - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: diff --git a/go-controller/pkg/ovn/master.go b/go-controller/pkg/ovn/master.go index 011615fd9db..d7e588cfc84 100644 --- a/go-controller/pkg/ovn/master.go +++ b/go-controller/pkg/ovn/master.go @@ -924,6 +924,9 @@ func (oc *DefaultNetworkController) deleteHoNodeEvent(node *kapi.Node) error { func (oc *DefaultNetworkController) addIPToHostNetworkNamespaceAddrSet(node *kapi.Node) error { var hostNetworkPolicyIPs []net.IP + if util.NoHostSubnet(node) { + return nil + } hostNetworkPolicyIPs, err := oc.getHostNamespaceAddressesForNode(node) if err != nil { parsedErr := err diff --git a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml index 5db066021db..58da075a576 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml @@ -266,6 +266,8 @@ spec: value: {{ hasKey .Values.global "enableMultiExternalGateway" | ternary .Values.global.enableMultiExternalGateway false | quote }} - name: OVN_ENABLE_SVC_TEMPLATE_SUPPORT value: {{ hasKey .Values.global "enableSvcTemplate" | ternary .Values.global.enableSvcTemplate true | quote }} + - name: OVN_NOHOSTSUBNET_LABEL + value: {{ default "k8s.ovn.org/ovn-managed=false" .Values.global.noHostSubnetLabel | quote }} - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: diff --git a/helm/ovn-kubernetes/values.yaml b/helm/ovn-kubernetes/values.yaml index 3173250e8c4..6e3aca21b2c 100644 --- a/helm/ovn-kubernetes/values.yaml +++ b/helm/ovn-kubernetes/values.yaml @@ -136,6 +136,8 @@ global: ofctrlWaitBeforeClear: "" # -- Separate log file for libovsdb client libovsdbClientLogFile: "" + # -- Label for indicating that the node is managing its own network + noHostSubnetLabel: "" image: # -- Image repository for ovn-kubernetes components repository: ghcr.io/ovn-org/ovn-kubernetes/ovn-kube-ubuntu From 339f8cc2e4fcfd39e13f5bc3b9e5f393228e5291 Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 30 Jul 2024 17:09:12 -0700 Subject: [PATCH 02/13] Helm chart fixes for DPUs Signed-off-by: nithyar --- dist/images/Dockerfile.ubuntu.arm64 | 55 +++++++++++ .../templates/ovnkube-node-dpu.yaml | 91 ++++++++++++++++++- .../charts/ovnkube-node-dpu/values.yaml | 2 +- helm/ovn-kubernetes/templates/_helpers.tpl | 17 ++++ helm/ovn-kubernetes/values.yaml | 15 +++ 5 files changed, 175 insertions(+), 5 deletions(-) create mode 100644 dist/images/Dockerfile.ubuntu.arm64 diff --git a/dist/images/Dockerfile.ubuntu.arm64 b/dist/images/Dockerfile.ubuntu.arm64 new file mode 100644 index 00000000000..b33df55add5 --- /dev/null +++ b/dist/images/Dockerfile.ubuntu.arm64 @@ -0,0 +1,55 @@ +# +# The standard name for this image is ovn-kube-ubuntu + +# Notes: +# This is for a development build where the ovn-kubernetes utilities +# are built in this Dockerfile and included in the image (instead of the deb package) +# +# +# So this file will change over time. + +FROM ubuntu:24.10 + +USER root + +RUN apt-get update && apt-get install -y iproute2 curl software-properties-common util-linux + +RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - + +# Install OVS and OVN packages. +RUN apt-get update && apt-get install -y openvswitch-switch openvswitch-common ovn-central ovn-common ovn-host + +RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/arm64/kubectl" \ + && install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + +RUN mkdir -p /var/run/openvswitch + +# Built in ../../go_controller, then the binaries are copied here. +# put things where they are in the pkg +RUN mkdir -p /usr/libexec/cni/ +COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity ovnkube-observ /usr/bin/ +COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay + +# ovnkube.sh is the entry point. This script examines environment +# variables to direct operation and configure ovn +COPY ovnkube.sh /root/ +COPY ovndb-raft-functions.sh /root/ +# override the pkg's ovn_k8s.conf with this local copy +COPY ovn_k8s.conf /etc/openvswitch/ovn_k8s.conf + +# copy git commit number into image +COPY git_info /root + +# iptables wrappers +COPY ./iptables-scripts/iptables /usr/sbin/ +COPY ./iptables-scripts/iptables-save /usr/sbin/ +COPY ./iptables-scripts/iptables-restore /usr/sbin/ +COPY ./iptables-scripts/ip6tables /usr/sbin/ +COPY ./iptables-scripts/ip6tables-save /usr/sbin/ +COPY ./iptables-scripts/ip6tables-restore /usr/sbin/ + +LABEL io.k8s.display-name="ovn-kubernetes" \ + io.k8s.description="ovnkube ubuntu image" + +WORKDIR /root +ENTRYPOINT /root/ovnkube.sh diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml index 8f0795ac59e..4df63e974a7 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml @@ -36,7 +36,7 @@ spec: containers: {{- if eq (hasKey .Values.global "enableCompactMode" | ternary .Values.global.enableCompactMode false) false }} - name: ovnkube-node - image: {{ include "getImage" . }} + image: {{ include "getDPUImage" . }} imagePullPolicy: {{ default "IfNotPresent" .Values.global.image.pullPolicy }} {{- if eq (hasKey .Values.global "dummyGatewayBridge" | ternary .Values.global.dummyGatewayBridge false) true }} lifecycle: @@ -160,7 +160,7 @@ spec: - name: OVN_EGRESSIP_ENABLE value: {{ default "" .Values.global.enableEgressIp | quote }} - name: OVN_EGRESSIP_HEALTHCHECK_PORT - value: {{ default "" .Values.global.enableEgressIp | quote }} + value: {{ default "" .Values.global.egressIpHealthCheckPort | quote }} - name: OVN_EGRESSSERVICE_ENABLE value: {{ default "" .Values.global.enableEgressService | quote }} - name: OVN_HYBRID_OVERLAY_NET_CIDR @@ -225,8 +225,6 @@ spec: value: {{ hasKey .Values.global "enableMultiExternalGateway" | ternary .Values.global.enableMultiExternalGateway false | quote }} - name: OVNKUBE_NODE_MODE value: "dpu" - - name: OVNKUBE_NODE_MGMT_PORT_NETDEV - value: {{ default "" .Values.global.nodeMgmtPortNetdev | quote }} - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: @@ -244,6 +242,91 @@ spec: timeoutSeconds: 30 periodSeconds: 60 {{- end }} + - name: ovn-controller + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.image.pullPolicy }} + command: ["/root/ovnkube.sh", "ovn-controller"] + securityContext: + runAsUser: 0 + capabilities: + add: ["SYS_NICE"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/log/ovn/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + - mountPath: /var/run/ovn/ + name: host-var-run-ovs + - mountPath: /ovn-cert + name: host-ovn-cert + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.0.0" + - name: OVN_LOGLEVEL_CONTROLLER + value: {{ default "-vconsole:info" .Values.ovnControllerLogLevel | quote }} + - name: K8S_APISERVER + valueFrom: + configMapKeyRef: + name: ovn-config + key: k8s_apiserver + - name: K8S_NODE + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_SSL_ENABLE + value: {{ include "isSslEnabled" . | quote }} + readinessProbe: + exec: + command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovn-controller"] + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + # ovs-metrics-exporter - v3 + - name: ovs-metrics-exporter + image: {{ include "getDPUImage" . }} + imagePullPolicy: {{ default "IfNotPresent" .Values.global.image.pullPolicy }} + command: ["/root/ovnkube.sh", "ovs-metrics"] + securityContext: + runAsUser: 0 + capabilities: + add: ["NET_ADMIN"] + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /var/run/dbus/ + name: host-var-run-dbus + readOnly: true + - mountPath: /var/log/openvswitch/ + name: host-var-log-ovs + - mountPath: /var/run/openvswitch/ + name: host-var-run-ovs + readOnly: true + resources: + requests: + cpu: 100m + memory: 300Mi + env: + - name: OVN_DAEMONSET_VERSION + value: "1.0.0" + - name: K8S_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + # end of container nodeSelector: kubernetes.io/os: "linux" k8s.ovn.org/dpu: "" diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml index 584127efca9..22101b6b4da 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml @@ -2,4 +2,4 @@ logLevel: 4 logFileMaxSize: 100 logFileMaxBackups: 5 logFileMaxAge: 5 -ovnControllerLogLevel: 4 \ No newline at end of file +ovnControllerLogLevel: "-vconsole:info" diff --git a/helm/ovn-kubernetes/templates/_helpers.tpl b/helm/ovn-kubernetes/templates/_helpers.tpl index 570370d519b..8237c728548 100644 --- a/helm/ovn-kubernetes/templates/_helpers.tpl +++ b/helm/ovn-kubernetes/templates/_helpers.tpl @@ -34,6 +34,23 @@ Generate image {{- end }} {{- end }} +{{/* +Generate DPU image +*/}} +{{- define "getDPUImage" -}} + {{- $image := "" }} + {{- if and (ne .Values.global.dpuImage.repository "") (ne .Values.global.dpuImage.tag "") }} + {{- $image = printf "%s:%s" .Values.global.dpuImage.repository .Values.global.dpuImage.tag }} + {{- else if and (ne .Values.dpuImage.repository "") (ne .Values.dpuImage.tag "") }} + {{- $image = printf "%s:%s" .Values.dpuImage.repository .Values.dpuImage.tag }} + {{- end }} + {{- if eq $image "" }} + {{ fail "dpu image not found" }} + {{- else }} + {{- print $image }} + {{- end }} +{{- end }} + {{/* Output "yes" if enableSsl is true, otherwise "no" */}} diff --git a/helm/ovn-kubernetes/values.yaml b/helm/ovn-kubernetes/values.yaml index 6e3aca21b2c..c6ce1dd2f60 100644 --- a/helm/ovn-kubernetes/values.yaml +++ b/helm/ovn-kubernetes/values.yaml @@ -145,6 +145,21 @@ global: tag: master # -- Image pull policy pullPolicy: IfNotPresent + # -- Image for DPUs + dpuImage: + # -- Image repository for ovn-kubernetes components + repository: ghcr.io/ovn-org/ovn-kubernetes/ovn-kube-ubuntu + # -- Specify image tag to run + tag: master + # -- Image pull policy + pullPolicy: IfNotPresent + # -- The name of secret used for pulling image. Use only if needed + imagePullSecretName: "" + # -- The secret used for pulling image. Use only if needed. Set create to have have secret created by helm + dockerConfigSecret: + registry: ghcr.io + auth: blah_blah_blah + create: false ovnkube-identity: # -- number of ovnube-identity pods, co-located with kube-apiserver process, so need to be the same number of control plane nodes From 581122a5e1ae5d8b707d41ab71a2095f7099c5ec Mon Sep 17 00:00:00 2001 From: nithyar Date: Wed, 31 Jul 2024 10:38:11 -0700 Subject: [PATCH 03/13] Capture dpu->dpu_host relation during ovnkube-node-dpu bringup Signed-off-by: nithyar --- dist/images/ovnkube.sh | 9 ++++++++ dist/templates/ovnkube-node.yaml.j2 | 2 +- .../node/default_node_network_controller.go | 7 +++++- .../default_node_network_controller_test.go | 22 ++++++++++--------- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 48c9b4a3de1..4137ecf3249 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -2401,6 +2401,15 @@ ovn-node() { node_mgmt_port_netdev_flags="$node_mgmt_port_netdev_flags --ovnkube-node-mgmt-port-dp-resource-name ${ovnkube_node_mgmt_port_dp_resource_name}" fi + if [[ ${ovnkube_node_mode} == "dpu" ]]; then + # in the case of dpu mode we want the host K8s Node Name and not the DPU K8s Node Name + K8S_NODE=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:host-k8s-nodename | tr -d '\"') + if [[ ${K8S_NODE} == "" ]]; then + echo "Couldn't get the required Host K8s Nodename. Exiting..." + exit 1 + fi + fi + local ovn_node_ssl_opts="" if [[ ${ovnkube_node_mode} != "dpu-host" ]]; then [[ "yes" == ${OVN_SSL_ENABLE} ]] && { diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index 10f578a0276..20816b662c9 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -261,7 +261,7 @@ spec: timeoutSeconds: 30 periodSeconds: 60 {% endif %} - {% if ovnkube_app_name=="ovnkube-node" -%} + {% if ovnkube_app_name!="ovnkube-node-dpu-host" -%} - name: ovn-controller image: "{{ ovn_image | default('docker.io/ovnkube/ovn-daemonset:latest') }}" imagePullPolicy: "{{ ovn_image_pull_policy | default('IfNotPresent') }}" diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index 96d87dd07a0..3e6c95519d0 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -305,7 +305,6 @@ func setupOVNNode(node *kapi.Node) error { // to finish computation specially with complex acl configuration with port range. fmt.Sprintf("other_config:bundle-idle-timeout=%d", config.Default.OpenFlowProbe), - fmt.Sprintf("external_ids:hostname=\"%s\"", node.Name), // If Interconnect feature is enabled, we want to tell ovn-controller to // make this node/chassis as an interconnect gateway. fmt.Sprintf("external_ids:ovn-is-interconn=%s", strconv.FormatBool(config.OVNKubernetesFeature.EnableInterconnect)), @@ -328,6 +327,12 @@ func setupOVNNode(node *kapi.Node) error { ) } + // In the case of DPU, the hostname should be that of the DPU and not + // the K8s Node's. So skip setting the incorrect hostname. + if config.OvnKubeNode.Mode != types.NodeModeDPU { + setExternalIdsCmd = append(setExternalIdsCmd, fmt.Sprintf("external_ids:hostname=\"%s\"", node.Name)) + } + _, stderr, err := util.RunOVSVsctl(setExternalIdsCmd...) if err != nil { return fmt.Errorf("error setting OVS external IDs: %v\n %q", err, stderr) diff --git a/go-controller/pkg/node/default_node_network_controller_test.go b/go-controller/pkg/node/default_node_network_controller_test.go index 5182a5693dc..5a0de916805 100644 --- a/go-controller/pkg/node/default_node_network_controller_test.go +++ b/go-controller/pkg/node/default_node_network_controller_test.go @@ -278,12 +278,12 @@ var _ = Describe("Node", func() { "external_ids:ovn-remote-probe-interval=%d "+ "external_ids:ovn-openflow-probe-interval=%d "+ "other_config:bundle-idle-timeout=%d "+ - "external_ids:hostname=\"%s\" "+ "external_ids:ovn-is-interconn=false "+ "external_ids:ovn-monitor-all=true "+ "external_ids:ovn-ofctrl-wait-before-clear=0 "+ "external_ids:ovn-enable-lflow-cache=true "+ - "external_ids:ovn-set-local-ip=\"true\"", + "external_ids:ovn-set-local-ip=\"true\" "+ + "external_ids:hostname=\"%s\"", nodeIP, interval, ofintval, ofintval, nodeName), }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ @@ -299,6 +299,7 @@ var _ = Describe("Node", func() { _, err = config.InitConfig(ctx, fexec, nil) Expect(err).NotTo(HaveOccurred()) + config.OvnKubeNode.Mode = types.NodeModeFull err = setupOVNNode(&node) Expect(err).NotTo(HaveOccurred()) @@ -383,14 +384,14 @@ var _ = Describe("Node", func() { "external_ids:ovn-remote-probe-interval=%d "+ "external_ids:ovn-openflow-probe-interval=%d "+ "other_config:bundle-idle-timeout=%d "+ - "external_ids:hostname=\"%s\" "+ "external_ids:ovn-is-interconn=false "+ "external_ids:ovn-monitor-all=true "+ "external_ids:ovn-ofctrl-wait-before-clear=0 "+ "external_ids:ovn-enable-lflow-cache=false "+ "external_ids:ovn-set-local-ip=\"true\" "+ "external_ids:ovn-limit-lflow-cache=1000 "+ - "external_ids:ovn-memlimit-lflow-cache-kb=100000", + "external_ids:ovn-memlimit-lflow-cache-kb=100000 "+ + "external_ids:hostname=\"%s\"", nodeIP, interval, ofintval, ofintval, nodeName), }) fexec.AddFakeCmd(&ovntest.ExpectedCmd{ @@ -409,6 +410,7 @@ var _ = Describe("Node", func() { config.Default.LFlowCacheEnable = false config.Default.LFlowCacheLimit = 1000 config.Default.LFlowCacheLimitKb = 100000 + config.OvnKubeNode.Mode = types.NodeModeFull err = setupOVNNode(&node) Expect(err).NotTo(HaveOccurred()) @@ -452,12 +454,12 @@ var _ = Describe("Node", func() { "external_ids:ovn-remote-probe-interval=%d "+ "external_ids:ovn-openflow-probe-interval=%d "+ "other_config:bundle-idle-timeout=%d "+ - "external_ids:hostname=\"%s\" "+ "external_ids:ovn-is-interconn=false "+ "external_ids:ovn-monitor-all=true "+ "external_ids:ovn-ofctrl-wait-before-clear=0 "+ "external_ids:ovn-enable-lflow-cache=true "+ - "external_ids:ovn-set-local-ip=\"true\"", + "external_ids:ovn-set-local-ip=\"true\" "+ + "external_ids:hostname=\"%s\"", nodeIP, interval, ofintval, ofintval, nodeName), }) @@ -527,12 +529,12 @@ var _ = Describe("Node", func() { "external_ids:ovn-remote-probe-interval=%d "+ "external_ids:ovn-openflow-probe-interval=%d "+ "other_config:bundle-idle-timeout=%d "+ - "external_ids:hostname=\"%s\" "+ "external_ids:ovn-is-interconn=false "+ "external_ids:ovn-monitor-all=true "+ "external_ids:ovn-ofctrl-wait-before-clear=0 "+ "external_ids:ovn-enable-lflow-cache=true "+ - "external_ids:ovn-set-local-ip=\"true\"", + "external_ids:ovn-set-local-ip=\"true\" "+ + "external_ids:hostname=\"%s\"", nodeIP, interval, ofintval, ofintval, nodeName), }) @@ -602,12 +604,12 @@ var _ = Describe("Node", func() { "external_ids:ovn-remote-probe-interval=%d "+ "external_ids:ovn-openflow-probe-interval=%d "+ "other_config:bundle-idle-timeout=%d "+ - "external_ids:hostname=\"%s\" "+ "external_ids:ovn-is-interconn=false "+ "external_ids:ovn-monitor-all=true "+ "external_ids:ovn-ofctrl-wait-before-clear=0 "+ "external_ids:ovn-enable-lflow-cache=true "+ - "external_ids:ovn-set-local-ip=\"true\"", + "external_ids:ovn-set-local-ip=\"true\" "+ + "external_ids:hostname=\"%s\"", nodeIP, interval, ofintval, ofintval, nodeName), }) From 7e76714ec3f3186243392f693f6ca4bcc6d85231 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Mon, 26 Feb 2024 15:38:59 -0800 Subject: [PATCH 04/13] [upstream] not to update ovs openflow in dpu-host mode Signed-off-by: Yun Zhou --- .../pkg/node/default_node_network_controller.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/go-controller/pkg/node/default_node_network_controller.go b/go-controller/pkg/node/default_node_network_controller.go index 3e6c95519d0..529b94a18ff 100644 --- a/go-controller/pkg/node/default_node_network_controller.go +++ b/go-controller/pkg/node/default_node_network_controller.go @@ -703,15 +703,18 @@ func (nc *DefaultNodeNetworkController) Start(ctx context.Context) error { if err := level.Set("5"); err != nil { klog.Errorf("Setting klog \"loglevel\" to 5 failed, err: %v", err) } + nc.wg.Add(1) go func() { defer nc.wg.Done() nc.routeManager.Run(nc.stopChan, 4*time.Minute) }() - // Bootstrap flows in OVS if just normal flow is present - if err := bootstrapOVSFlows(); err != nil { - return fmt.Errorf("failed to bootstrap OVS flows: %w", err) + if config.OvnKubeNode.Mode != types.NodeModeDPUHost { + // Bootstrap flows in OVS if just normal flow is present + if err := bootstrapOVSFlows(); err != nil { + return fmt.Errorf("failed to bootstrap OVS flows: %w", err) + } } if node, err = nc.Kube.GetNode(nc.name); err != nil { From 65941949011d323209ad1468f3e608e725adf131 Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Tue, 5 Mar 2024 12:59:52 -0800 Subject: [PATCH 05/13] [upstream] dpu bridge mac should not be used in openflow rules Signed-off-by: Yun Zhou --- go-controller/pkg/node/gateway_init.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index cd11e085831..cfe58c98c90 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -541,6 +541,13 @@ func CleanupClusterNode(name string) error { } func (nc *DefaultNodeNetworkController) updateGatewayMAC(link netlink.Link) error { + // TBD-merge for dpu-host mode: if interface mac of the dpu-host interface that connects to the + // gateway bridge on the dpu changes, we need to update dpu's gatewayBridge.macAddress L3 gateway + // annotation (see bridgeForInterface) + if config.OvnKubeNode.Mode != types.NodeModeFull { + return nil + } + if nc.gateway.GetGatewayBridgeIface() != link.Attrs().Name { return nil } From c15dca83619eced9dafcfd2bd89adcf5e510a93c Mon Sep 17 00:00:00 2001 From: Yun Zhou Date: Tue, 5 Mar 2024 11:57:19 -0800 Subject: [PATCH 06/13] service traffic route through Masquerade node IP on dpu host node Signed-off-by: Yun Zhou --- go-controller/pkg/node/gateway_init.go | 2 +- go-controller/pkg/node/gateway_init_linux_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index cfe58c98c90..1210ec9bd3e 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -476,7 +476,7 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHost(kubeNodeIP net.IP) er return fmt.Errorf("failed to set the node masquerade route to OVN: %v", err) } - err = configureSvcRouteViaInterface(nc.routeManager, gatewayIntf, gatewayNextHops) + err = configureSvcRouteViaInterface(nc.routeManager, gatewayIntf, DummyNextHopIPs()) if err != nil { return err } diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 8a1c67e6dbd..6f9dff88a95 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -769,7 +769,7 @@ func shareGatewayInterfaceDPUHostTest(app *cli.App, testNS ns.NetNS, uplinkName, expRoute := &netlink.Route{ Dst: ovntest.MustParseIPNet(svcCIDR), LinkIndex: link.Attrs().Index, - Gw: ovntest.MustParseIP(gwIP), + Gw: ovntest.MustParseIP(config.Gateway.MasqueradeIPs.V4DummyNextHopMasqueradeIP.String()), } Eventually(func() error { r, err := util.LinkRouteGetFilteredRoute( From 5b60819322d377fa49e8f5319412bcbc434c1afb Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 3 Sep 2024 12:27:49 -0700 Subject: [PATCH 07/13] Add gateway options in DPU mode Signed-off-by: nithyar --- dist/images/ovnkube.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index 4137ecf3249..ba6fe8049d3 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -2408,6 +2408,30 @@ ovn-node() { echo "Couldn't get the required Host K8s Nodename. Exiting..." exit 1 fi + if [[ ${ovn_gateway_opts} == "" ]]; then + # get the gateway interface + gw_iface=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-interface | tr -d \") + if [[ ${gw_iface} == "" ]]; then + echo "Couldn't get the required OVN Gateway Interface. Exiting..." + exit 1 + fi + ovn_gateway_opts="--gateway-interface=${gw_iface} " + + # get the gateway nexthop + gw_nexthop=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-nexthop | tr -d \") + if [[ ${gw_nexthop} == "" ]]; then + echo "Couldn't get the required OVN Gateway NextHop. Exiting..." + exit 1 + fi + ovn_gateway_opts+="--gateway-nexthop=${gw_nexthop} " + fi + + # this is required if the DPU and DPU Host are in different subnets + if [[ ${ovn_gateway_router_subnet} == "" ]]; then + # get the gateway router subnet + ovn_gateway_router_subnet=$(ovs-vsctl --if-exists get Open_vSwitch . external_ids:ovn-gw-router-subnet | tr -d \") + fi + fi local ovn_node_ssl_opts="" From 6e287fe55087b5ecf15f9471dab5659418605060 Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 3 Sep 2024 12:47:33 -0700 Subject: [PATCH 08/13] Fix node certificate handling for DPUs Signed-off-by: nithyar --- dist/templates/ovnkube-node.yaml.j2 | 8 ++++++++ go-controller/cmd/ovnkube/ovnkube.go | 9 ++++++++- .../ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml | 4 ++-- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index 20816b662c9..8f357ee587b 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -149,7 +149,11 @@ spec: name: ovn-config key: routable_mtu optional: true + {%-if ovnkube_app_name=="ovnkube-node-dpu" %} + - name: K8S_NODE_DPU + {%- else %} - name: K8S_NODE + {%- endif %} valueFrom: fieldRef: fieldPath: spec.nodeName @@ -304,7 +308,11 @@ spec: configMapKeyRef: name: ovn-config key: k8s_apiserver + {%- if ovnkube_app_name=="ovnkube-node-dpu" %} + - name: K8S_NODE_DPU + {%- else %} - name: K8S_NODE + {%- endif %} valueFrom: fieldRef: fieldPath: spec.nodeName diff --git a/go-controller/cmd/ovnkube/ovnkube.go b/go-controller/cmd/ovnkube/ovnkube.go index a67cb4e8210..8dca95f1cf9 100644 --- a/go-controller/cmd/ovnkube/ovnkube.go +++ b/go-controller/cmd/ovnkube/ovnkube.go @@ -278,7 +278,14 @@ func startOvnKube(ctx *cli.Context, cancel context.CancelFunc) error { }() if config.Kubernetes.BootstrapKubeconfig != "" { - if err := util.StartNodeCertificateManager(ctx.Context, ovnKubeStartWg, os.Getenv("K8S_NODE"), &config.Kubernetes); err != nil { + // In the case of dpus K8S_NODE will be set to dpu host's name + var csrNodeName string + if config.OvnKubeNode.Mode == types.NodeModeDPU { + csrNodeName = os.Getenv("K8S_NODE_DPU") + } else { + csrNodeName = os.Getenv("K8S_NODE") + } + if err := util.StartNodeCertificateManager(ctx.Context, ovnKubeStartWg, csrNodeName, &config.Kubernetes); err != nil { return fmt.Errorf("failed to start the node certificate manager: %w", err) } } diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml index 4df63e974a7..4fb7165f699 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml @@ -141,7 +141,7 @@ spec: name: ovn-config key: routable_mtu optional: true - - name: K8S_NODE + - name: K8S_NODE_DPU valueFrom: fieldRef: fieldPath: spec.nodeName @@ -280,7 +280,7 @@ spec: configMapKeyRef: name: ovn-config key: k8s_apiserver - - name: K8S_NODE + - name: K8S_NODE_DPU valueFrom: fieldRef: fieldPath: spec.nodeName From 56ce58bb1a2480a12a948ce5a2c0f2a9ee89c4ea Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 3 Sep 2024 12:55:29 -0700 Subject: [PATCH 09/13] Disable nodeIPManager on dpu nodes Signed-off-by: nithyar --- go-controller/pkg/node/gateway_init_linux_test.go | 9 --------- go-controller/pkg/node/node_ip_handler_linux.go | 8 ++++++++ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/go-controller/pkg/node/gateway_init_linux_test.go b/go-controller/pkg/node/gateway_init_linux_test.go index 6f9dff88a95..1bc46f9fc7c 100644 --- a/go-controller/pkg/node/gateway_init_linux_test.go +++ b/go-controller/pkg/node/gateway_init_linux_test.go @@ -538,15 +538,6 @@ func shareGatewayInterfaceDPUTest(app *cli.App, testNS ns.NetNS, Cmd: "ovs-vsctl --timeout=15 get interface " + hostRep + " ofport", Output: "9", }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovs-vsctl --timeout=15 get Open_vSwitch . external_ids:ovn-encap-ip", - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovs-vsctl --timeout=15 set Open_vSwitch . external_ids:ovn-encap-ip=192.168.1.101", - }) - fexec.AddFakeCmdsNoOutputNoError([]string{ - "ovn-appctl --timeout=5 -t ovn-controller exit --restart", - }) // cleanup flows fexec.AddFakeCmdsNoOutputNoError([]string{ "ovs-ofctl -O OpenFlow13 --bundle replace-flows " + brphys + " -", diff --git a/go-controller/pkg/node/node_ip_handler_linux.go b/go-controller/pkg/node/node_ip_handler_linux.go index 7145d605a91..313f239c131 100644 --- a/go-controller/pkg/node/node_ip_handler_linux.go +++ b/go-controller/pkg/node/node_ip_handler_linux.go @@ -115,6 +115,10 @@ func (c *addressManager) ListAddresses() []net.IP { type subscribeFn func() (bool, chan netlink.AddrUpdate, error) func (c *addressManager) Run(stopChan <-chan struct{}, doneWg *sync.WaitGroup) { + if config.OvnKubeNode.Mode == types.NodeModeDPU { + return + } + c.addHandlerForPrimaryAddrChange() doneWg.Add(1) go func() { @@ -407,6 +411,10 @@ func (c *addressManager) isValidNodeIP(addr net.IP) bool { } func (c *addressManager) sync() { + if config.OvnKubeNode.Mode == types.NodeModeDPU { + return + } + var addrs []netlink.Addr if c.useNetlink { From 35427943cf246af176d0092dab5b8db1bca53c73 Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 3 Sep 2024 13:57:57 -0700 Subject: [PATCH 10/13] requested-chassis should not be set for dpu-host nodes Signed-off-by: nithyar --- dist/images/daemonset.sh | 8 +++ dist/images/ovnkube.sh | 10 +++ dist/templates/ovnkube-master.yaml.j2 | 2 + go-controller/pkg/config/config.go | 63 +++++++++++-------- go-controller/pkg/config/config_test.go | 3 + .../pkg/ovn/base_network_controller_pods.go | 5 +- .../templates/deployment-ovnkube-master.yaml | 2 + helm/ovn-kubernetes/values.yaml | 2 + 8 files changed, 67 insertions(+), 28 deletions(-) diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh index de944cde69e..9d60fbb8add 100755 --- a/dist/images/daemonset.sh +++ b/dist/images/daemonset.sh @@ -93,6 +93,7 @@ OVN_ENABLE_OVNKUBE_IDENTITY="true" OVN_ENABLE_PERSISTENT_IPS= OVN_ENABLE_SVC_TEMPLATE_SUPPORT="true" OVN_NOHOSTSUBNET_LABEL="" +OVN_DISABLE_REQUESTEDCHASSIS="false" # IN_UPGRADE is true only if called by upgrade-ovn.sh during the upgrade test, # it will render only the parts in ovn-setup.yaml related to RBAC permissions. IN_UPGRADE= @@ -350,6 +351,9 @@ while [ "$1" != "" ]; do --no-hostsubnet-label) OVN_NOHOSTSUBNET_LABEL=$VALUE ;; + --ovn_disable_requestedchassis) + OVN_DISABLE_REQUESTEDCHASSIS=$value + ;; *) echo "WARNING: unknown parameter \"$PARAM\"" exit 1 @@ -538,6 +542,9 @@ echo "ovn_enable_svc_template_support: ${ovn_enable_svc_template_support}" ovn_nohostsubnet_label=${OVN_NOHOSTSUBNET_LABEL} echo "ovn_nohostsubnet_label: ${ovn_nohostsubnet_label}" +ovn_disable_requestedchassis=${OVN_DISABLE_REQUESTEDCHASSIS} +echo "ovn_disable_requestedchassis: ${ovn_disable_requestedchassis}" + ovn_image=${ovnkube_image} \ ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \ ovn_image_pull_policy=${image_pull_policy} \ @@ -724,6 +731,7 @@ ovn_image=${ovnkube_image} \ ovn_enable_persistent_ips=${ovn_enable_persistent_ips} \ ovn_enable_svc_template_support=${ovn_enable_svc_template_support} \ ovn_nohostsubnet_label=${ovn_nohostsubnet_label} \ + ovn_disable_requestedchassis=${ovn_disable_requestedchassis} \ jinjanate ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml ovn_image=${ovnkube_image} \ diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh index ba6fe8049d3..3331ff28251 100755 --- a/dist/images/ovnkube.sh +++ b/dist/images/ovnkube.sh @@ -307,6 +307,9 @@ ovn_northd_backoff_interval=${OVN_NORTHD_BACKOFF_INTERVAL:-"300"} ovn_enable_svc_template_support=${OVN_ENABLE_SVC_TEMPLATE_SUPPORT:-true} # OVN_NOHOSTSUBNET_LABEL - node label indicating nodes managing their own network ovn_nohostsubnet_label=${OVN_NOHOSTSUBNET_LABEL:-""} +# OVN_DISABLE_REQUESTEDCHASSIS - disable requested-chassis option during pod creation +# should be set to true when dpu nodes are in the cluster +ovn_disable_requestedchassis=${OVN_DISABLE_REQUESTEDCHASSIS:-false} # Determine the ovn rundir. if [[ -f /usr/bin/ovn-appctl ]]; then @@ -1255,6 +1258,12 @@ ovn-master() { nohostsubnet_label_option="--no-hostsubnet-nodes=${ovn_nohostsubnet_label}" fi + ovn_disable_requestedchassis_flag= + if [[ ${ovn_disable_requestedchassis} == "true" ]]; then + ovn_disable_requestedchassis_flag="--disable-requestedchassis" + fi + echo "ovn_disable_requestedchassis_flag=${ovn_disable_requestedchassis_flag}" + init_node_flags= if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then init_node_flags="--init-node ${K8S_NODE} --nodeport" @@ -1299,6 +1308,7 @@ ovn-master() { ${ovn_v6_masquerade_subnet_opt} \ ${persistent_ips_enabled_flag} \ ${nohostsubnet_label_option} \ + ${ovn_disable_requestedchassis_flag} \ --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \ --gateway-mode=${ovn_gateway_mode} ${ovn_gateway_opts} \ --host-network-namespace ${ovn_host_network_namespace} \ diff --git a/dist/templates/ovnkube-master.yaml.j2 b/dist/templates/ovnkube-master.yaml.j2 index a926898728c..a79a97c44f0 100644 --- a/dist/templates/ovnkube-master.yaml.j2 +++ b/dist/templates/ovnkube-master.yaml.j2 @@ -295,6 +295,8 @@ spec: value: "{{ ovn_enable_svc_template_support }}" - name: OVN_NOHOSTSUBNET_LABEL value: "{{ ovn_nohostsubnet_label }}" + - name: OVN_DISABLE_REQUESTEDCHASSIS + value: "{{ ovn_disable_requestedchassis }}" - name: OVN_HOST_NETWORK_NAMESPACE valueFrom: configMapKeyRef: diff --git a/go-controller/pkg/config/config.go b/go-controller/pkg/config/config.go index 717a2b3d516..538e3ac5c5d 100644 --- a/go-controller/pkg/config/config.go +++ b/go-controller/pkg/config/config.go @@ -107,13 +107,14 @@ var ( // Kubernetes holds Kubernetes-related parsed config file parameters and command-line overrides Kubernetes = KubernetesConfig{ - APIServer: DefaultAPIServer, - RawServiceCIDRs: "172.16.1.0/24", - OVNConfigNamespace: "ovn-kubernetes", - HostNetworkNamespace: "", - PlatformType: "", - DNSServiceNamespace: "kube-system", - DNSServiceName: "kube-dns", + APIServer: DefaultAPIServer, + RawServiceCIDRs: "172.16.1.0/24", + OVNConfigNamespace: "ovn-kubernetes", + HostNetworkNamespace: "", + DisableRequestedChassis: false, + PlatformType: "", + DNSServiceNamespace: "kube-system", + DNSServiceName: "kube-dns", // By default, use a short lifetime length for certificates to ensure that the automatic rotation works well, // might revisit in the future to use a more sensible value CertDuration: 10 * time.Minute, @@ -340,26 +341,27 @@ type CNIConfig struct { // KubernetesConfig holds Kubernetes-related parsed config file parameters and command-line overrides type KubernetesConfig struct { - BootstrapKubeconfig string `gcfg:"bootstrap-kubeconfig"` - CertDir string `gcfg:"cert-dir"` - CertDuration time.Duration `gcfg:"cert-duration"` - Kubeconfig string `gcfg:"kubeconfig"` - CACert string `gcfg:"cacert"` - CAData []byte - APIServer string `gcfg:"apiserver"` - Token string `gcfg:"token"` - TokenFile string `gcfg:"tokenFile"` - CompatServiceCIDR string `gcfg:"service-cidr"` - RawServiceCIDRs string `gcfg:"service-cidrs"` - ServiceCIDRs []*net.IPNet - OVNConfigNamespace string `gcfg:"ovn-config-namespace"` - OVNEmptyLbEvents bool `gcfg:"ovn-empty-lb-events"` - PodIP string `gcfg:"pod-ip"` // UNUSED - RawNoHostSubnetNodes string `gcfg:"no-hostsubnet-nodes"` - NoHostSubnetNodes labels.Selector - HostNetworkNamespace string `gcfg:"host-network-namespace"` - PlatformType string `gcfg:"platform-type"` - HealthzBindAddress string `gcfg:"healthz-bind-address"` + BootstrapKubeconfig string `gcfg:"bootstrap-kubeconfig"` + CertDir string `gcfg:"cert-dir"` + CertDuration time.Duration `gcfg:"cert-duration"` + Kubeconfig string `gcfg:"kubeconfig"` + CACert string `gcfg:"cacert"` + CAData []byte + APIServer string `gcfg:"apiserver"` + Token string `gcfg:"token"` + TokenFile string `gcfg:"tokenFile"` + CompatServiceCIDR string `gcfg:"service-cidr"` + RawServiceCIDRs string `gcfg:"service-cidrs"` + ServiceCIDRs []*net.IPNet + OVNConfigNamespace string `gcfg:"ovn-config-namespace"` + OVNEmptyLbEvents bool `gcfg:"ovn-empty-lb-events"` + PodIP string `gcfg:"pod-ip"` // UNUSED + RawNoHostSubnetNodes string `gcfg:"no-hostsubnet-nodes"` + NoHostSubnetNodes labels.Selector + HostNetworkNamespace string `gcfg:"host-network-namespace"` + DisableRequestedChassis bool `gcfg:"disable-requestedchassis"` + PlatformType string `gcfg:"platform-type"` + HealthzBindAddress string `gcfg:"healthz-bind-address"` // CompatMetricsBindAddress is overridden by the corresponding option in MetricsConfig CompatMetricsBindAddress string `gcfg:"metrics-bind-address"` @@ -628,6 +630,7 @@ func PrepareTestConfig() error { HybridOverlay = savedHybridOverlay OvnKubeNode = savedOvnKubeNode ClusterManager = savedClusterManager + Kubernetes.DisableRequestedChassis = false EnableMulticast = false if err := completeConfig(); err != nil { @@ -1156,6 +1159,12 @@ var K8sFlags = []cli.Flag{ Destination: &cliConfig.Kubernetes.HostNetworkNamespace, Value: Kubernetes.HostNetworkNamespace, }, + &cli.BoolFlag{ + Name: "disable-requestedchassis", + Usage: "If set to true, requested-chassis option will not be set during pod creation", + Destination: &cliConfig.Kubernetes.DisableRequestedChassis, + Value: Kubernetes.DisableRequestedChassis, + }, &cli.StringFlag{ Name: "platform-type", Usage: "The cloud provider platform type ovn-kubernetes is deployed on. " + diff --git a/go-controller/pkg/config/config_test.go b/go-controller/pkg/config/config_test.go index e32270f3b55..86a33c23aa9 100644 --- a/go-controller/pkg/config/config_test.go +++ b/go-controller/pkg/config/config_test.go @@ -154,6 +154,7 @@ no-hostsubnet-nodes=label=another-test-label healthz-bind-address=0.0.0.0:1234 dns-service-namespace=kube-system-f dns-service-name=kube-dns-f +disable-requestedchassis=false [metrics] bind-address=1.1.1.1:8080 @@ -739,6 +740,7 @@ var _ = Describe("Config Operations", func() { gomega.Expect(Kubernetes.HealthzBindAddress).To(gomega.Equal("0.0.0.0:4321")) gomega.Expect(Kubernetes.DNSServiceNamespace).To(gomega.Equal("kube-system-2")) gomega.Expect(Kubernetes.DNSServiceName).To(gomega.Equal("kube-dns-2")) + gomega.Expect(Kubernetes.DisableRequestedChassis).To(gomega.BeTrue()) gomega.Expect(Default.ClusterSubnets).To(gomega.Equal([]CIDRNetworkEntry{ {ovntest.MustParseIPNet("10.130.0.0/15"), 24}, })) @@ -863,6 +865,7 @@ var _ = Describe("Config Operations", func() { "-zone=bar", "-dns-service-namespace=kube-system-2", "-dns-service-name=kube-dns-2", + "-disable-requestedchassis=true", "-cluster-manager-v4-transit-switch-subnet=100.90.0.0/16", "-cluster-manager-v6-transit-switch-subnet=fd96::/64", } diff --git a/go-controller/pkg/ovn/base_network_controller_pods.go b/go-controller/pkg/ovn/base_network_controller_pods.go index cdb4aaf8c71..f8e046c90b0 100644 --- a/go-controller/pkg/ovn/base_network_controller_pods.go +++ b/go-controller/pkg/ovn/base_network_controller_pods.go @@ -529,7 +529,10 @@ func (bnc *BaseNetworkController) addLogicalPortToNetwork(pod *kapi.Pod, nadName // chassis if ovnkube-node isn't running correctly and hasn't cleared // out iface-id for an old instance of this pod, and the pod got // rescheduled. - lsp.Options["requested-chassis"] = pod.Spec.NodeName + + if !config.Kubernetes.DisableRequestedChassis { + lsp.Options["requested-chassis"] = pod.Spec.NodeName + } // Although we have different code to allocate the pod annotation for the // default network and secondary networks, at the time of this writing they diff --git a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml index 58da075a576..661c5f2c6ec 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-master/templates/deployment-ovnkube-master.yaml @@ -273,6 +273,8 @@ spec: configMapKeyRef: name: ovn-config key: host_network_namespace + - name: OVN_DISABLE_REQUESTEDCHASSIS + value: {{ default "false" .Values.global.disableRequestedchassis | quote }} # end of container volumes: # TODO: Need to check why we need this? diff --git a/helm/ovn-kubernetes/values.yaml b/helm/ovn-kubernetes/values.yaml index c6ce1dd2f60..09f6892f585 100644 --- a/helm/ovn-kubernetes/values.yaml +++ b/helm/ovn-kubernetes/values.yaml @@ -101,6 +101,8 @@ global: disablePacketMtuCheck: "" # -- Deprecated: iface-id-ver is always enabled disableIfaceIdVer: false + # -- Enable/disable requested-chassis option on lsp during pod creation. Must be set to true when cluster has DPUs + disableRequestedchassis: false # -- The largest number of messages per second that gets logged before drop # @default 20 aclLoggingRateLimit: 20 From f63cc7ce2c7da3ee4464e490a81fbd38807bdced Mon Sep 17 00:00:00 2001 From: nithyar Date: Tue, 3 Sep 2024 16:05:01 -0700 Subject: [PATCH 11/13] Update readinessprobe for DPUs Signed-off-by: nithyar --- dist/templates/ovnkube-node.yaml.j2 | 10 ++++++++++ .../ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml | 10 ++++++---- .../ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml | 2 ++ 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/dist/templates/ovnkube-node.yaml.j2 b/dist/templates/ovnkube-node.yaml.j2 index 8f357ee587b..54592d5aab0 100644 --- a/dist/templates/ovnkube-node.yaml.j2 +++ b/dist/templates/ovnkube-node.yaml.j2 @@ -259,11 +259,21 @@ spec: fieldPath: metadata.name readinessProbe: + {%- if ovnkube_app_name!="ovnkube-node-dpu" %} exec: command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnkube-node"] initialDelaySeconds: 30 timeoutSeconds: 30 periodSeconds: 60 + {%- else %} + httpGet: + path: /metrics + port: 9476 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + periodSeconds: 30 + {%- endif %} {% endif %} {% if ovnkube_app_name!="ovnkube-node-dpu-host" -%} - name: ovn-controller diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml index 4fb7165f699..0140e3e23cf 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/templates/ovnkube-node-dpu.yaml @@ -236,11 +236,13 @@ spec: apiVersion: v1 fieldPath: metadata.name readinessProbe: - exec: - command: ["/usr/bin/ovn-kube-util", "readiness-probe", "-t", "ovnkube-node"] + httpGet: + path: /metrics + port: {{ .Values.metricsPort }} + scheme: HTTP initialDelaySeconds: 30 - timeoutSeconds: 30 - periodSeconds: 60 + timeoutSeconds: 5 + periodSeconds: 30 {{- end }} - name: ovn-controller image: {{ include "getDPUImage" . }} diff --git a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml index 22101b6b4da..fe2b401adcd 100644 --- a/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml +++ b/helm/ovn-kubernetes/charts/ovnkube-node-dpu/values.yaml @@ -3,3 +3,5 @@ logFileMaxSize: 100 logFileMaxBackups: 5 logFileMaxAge: 5 ovnControllerLogLevel: "-vconsole:info" +# -- TCP port serving metrics +metricsPort: 9476 From c905d0a55e69284bf900ed0f0c985948391e16c7 Mon Sep 17 00:00:00 2001 From: even-k8s-master Date: Tue, 8 Oct 2024 11:31:25 +0800 Subject: [PATCH 12/13] Fixed unused declare --- go-controller/pkg/node/gateway_init.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go-controller/pkg/node/gateway_init.go b/go-controller/pkg/node/gateway_init.go index 1210ec9bd3e..9c73a2d1801 100644 --- a/go-controller/pkg/node/gateway_init.go +++ b/go-controller/pkg/node/gateway_init.go @@ -458,7 +458,7 @@ func (nc *DefaultNodeNetworkController) initGatewayDPUHost(kubeNodeIP net.IP) er } config.Gateway.Interface = gwIntf - gatewayNextHops, gatewayIntf, err := getGatewayNextHops() + _, gatewayIntf, err := getGatewayNextHops() if err != nil { return err } From afe9fedee5227ca86d4f1bd0688e5894cf2f4092 Mon Sep 17 00:00:00 2001 From: even-k8s-master Date: Tue, 8 Oct 2024 14:22:21 +0800 Subject: [PATCH 13/13] Remove the ovnkube-observ that haven't been backported yet. --- dist/images/Dockerfile.ubuntu.arm64 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/images/Dockerfile.ubuntu.arm64 b/dist/images/Dockerfile.ubuntu.arm64 index b33df55add5..6c336c6749e 100644 --- a/dist/images/Dockerfile.ubuntu.arm64 +++ b/dist/images/Dockerfile.ubuntu.arm64 @@ -27,7 +27,7 @@ RUN mkdir -p /var/run/openvswitch # Built in ../../go_controller, then the binaries are copied here. # put things where they are in the pkg RUN mkdir -p /usr/libexec/cni/ -COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity ovnkube-observ /usr/bin/ +COPY ovnkube ovn-kube-util ovndbchecker hybrid-overlay-node ovnkube-identity /usr/bin/ COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay # ovnkube.sh is the entry point. This script examines environment