diff --git a/internal/controller/testdata/deletion-no-delete/before/hpa.yaml b/internal/controller/testdata/deletion-no-delete/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/deletion-no-delete/before/hpa.yaml +++ b/internal/controller/testdata/deletion-no-delete/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/deletion-policy-all/before/hpa.yaml b/internal/controller/testdata/deletion-policy-all/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/deletion-policy-all/before/hpa.yaml +++ b/internal/controller/testdata/deletion-policy-all/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/hpa.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/hpa.yaml index 99305dee..7c627ee7 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/hpa.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/after/hpa.yaml @@ -47,5 +47,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/hpa.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/hpa.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-add-another-horizontal/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/deployment.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/deployment.yaml index 96d10022..0d41a091 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/deployment.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/deployment.yaml @@ -8,8 +8,7 @@ spec: strategy: {} template: metadata: - annotations: - kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + annotations: null creationTimestamp: null labels: app: mercari diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml index ece6ec11..9af0d17e 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-no-hpa-and-add-horizontal/after/tortoise.yaml @@ -20,110 +20,97 @@ status: cpu: Vertical memory: Vertical conditions: + containerResourceRequests: + - containerName: app + resource: + cpu: "10" + memory: 10Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile containerRecommendationFromVPA: - containerName: app maxRecommendation: cpu: - quantity: "3" - updatedAt: "2023-01-01T00:00:00Z" + quantity: "10" + updatedAt: null memory: - quantity: 3Gi - updatedAt: "2023-01-01T00:00:00Z" + quantity: 10Gi + updatedAt: null recommendation: cpu: - quantity: "3" - updatedAt: "2023-01-01T00:00:00Z" + quantity: "10" + updatedAt: null memory: - quantity: 3Gi - updatedAt: "2023-01-01T00:00:00Z" + quantity: 10Gi + updatedAt: null - containerName: istio-proxy maxRecommendation: cpu: - quantity: "3" - updatedAt: "2023-01-01T00:00:00Z" + quantity: "4" + updatedAt: null memory: - quantity: 3Gi - updatedAt: "2023-01-01T00:00:00Z" + quantity: 4Gi + updatedAt: null recommendation: cpu: - quantity: "3" - updatedAt: "2023-01-01T00:00:00Z" + quantity: "4" + updatedAt: null memory: - quantity: 3Gi - updatedAt: "2023-01-01T00:00:00Z" - containerResourceRequests: - - containerName: app - resource: - cpu: "10" - memory: 3Gi - - containerName: istio-proxy - resource: - cpu: "3" - memory: 3Gi - tortoiseConditions: - - lastTransitionTime: "2023-01-01T00:00:00Z" - lastUpdateTime: "2023-01-01T00:00:00Z" - message: the current number of replicas is not bigger than the preferred max - replica number - reason: ScaledUpBasedOnPreferredMaxReplicas - status: "False" - type: ScaledUpBasedOnPreferredMaxReplicas - - lastTransitionTime: "2023-01-01T00:00:00Z" - lastUpdateTime: "2023-01-01T00:00:00Z" - message: The recommendation is provided - status: "True" - type: VerticalRecommendationUpdated - - lastTransitionTime: "2023-01-01T00:00:00Z" - lastUpdateTime: "2023-01-01T00:00:00Z" - status: "False" - type: FailedToReconcile - containerResourcePhases: - - containerName: app - resourcePhases: - cpu: - lastTransitionTime: "2023-01-01T00:00:00Z" - phase: GatheringData - memory: - lastTransitionTime: "2023-01-01T00:00:00Z" - phase: Working - - containerName: istio-proxy - resourcePhases: - cpu: - lastTransitionTime: "2023-01-01T00:00:00Z" - phase: Working - memory: - lastTransitionTime: "2023-01-01T00:00:00Z" - phase: Working + quantity: 4Gi + updatedAt: null recommendations: horizontal: maxReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" + updatedAt: "2023-10-06T01:15:47Z" value: 20 minReplicas: - from: 0 timezone: Local to: 24 - updatedAt: "2023-01-01T00:00:00Z" + updatedAt: "2023-10-06T01:15:47Z" value: 5 targetUtilizations: - containerName: app - targetUtilization: - cpu: 70 + targetUtilization: {} - containerName: istio-proxy targetUtilization: {} vertical: containerResourceRecommendation: - RecommendedResource: cpu: "10" - memory: 3Gi + memory: 10Gi containerName: app - RecommendedResource: - cpu: "3" - memory: 3Gi + cpu: "4" + memory: 4Gi containerName: istio-proxy + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: GatheringData + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working targets: horizontalPodAutoscaler: tortoise-hpa-mercari scaleTargetRef: @@ -132,4 +119,4 @@ status: verticalPodAutoscalers: - name: tortoise-monitor-mercari role: Monitor - tortoisePhase: Working + tortoisePhase: PartlyWorking diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/hpa.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/hpa.yaml index acce44a3..28c7019f 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/hpa.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/after/hpa.yaml @@ -32,5 +32,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/before/hpa.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/before/hpa.yaml index 21aec690..0c317754 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/before/hpa.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal-2/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal/before/hpa.yaml b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal/before/hpa.yaml +++ b/internal/controller/testdata/mutable-autoscalingpolicy-remove-horizontal/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/deployment.yaml new file mode 100644 index 00000000..96d10022 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/deployment.yaml @@ -0,0 +1,30 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + - image: awesome-istio-proxy-image + name: istio-proxy + resources: + requests: + cpu: "4" + memory: 4Gi +status: {} diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/hpa.yaml new file mode 100644 index 00000000..a8b794f3 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/hpa.yaml @@ -0,0 +1,60 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 20 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 55 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 55 + type: Utilization + type: ContainerResource + minReplicas: 5 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/tortoise.yaml new file mode 100644 index 00000000..e36185be --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/tortoise.yaml @@ -0,0 +1,142 @@ +metadata: + finalizers: + - tortoise.autoscaling.mercari.com/finalizer + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - containerName: app + policy: + cpu: Horizontal + memory: Vertical + - containerName: istio-proxy + policy: + cpu: Horizontal + memory: Vertical + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + containerResourceRequests: + - containerName: app + resource: + cpu: "10" + memory: 3Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 3Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: the current number of replicas is not bigger than the preferred max replica number + reason: ScaledUpBasedOnPreferredMaxReplicas + status: "False" + type: ScaledUpBasedOnPreferredMaxReplicas + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: The recommendation is provided + status: "True" + type: VerticalRecommendationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 20 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-01-01T00:00:00Z" + value: 5 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 90 + - containerName: istio-proxy + targetUtilization: + cpu: 55 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "10" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + scaleTargetRef: + kind: "" + name: "" + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/vpa-Monitor.yaml new file mode 100644 index 00000000..81846608 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/after/vpa-Monitor.yaml @@ -0,0 +1,39 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/deployment.yaml new file mode 100644 index 00000000..693bb43d --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/deployment.yaml @@ -0,0 +1,27 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/proxyCPU: "4" + sidecar.istio.io/proxyMemory: "4Gi" + labels: + app: mercari + spec: + containers: + - name: istio-proxy # will be ignored. + image: auto + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + replicas: 10 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/hpa.yaml new file mode 100644 index 00000000..fe5f26b0 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/hpa.yaml @@ -0,0 +1,61 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 15 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + minReplicas: 3 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/tortoise.yaml new file mode 100644 index 00000000..2757b8f3 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/tortoise.yaml @@ -0,0 +1,116 @@ +metadata: + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - policy: + cpu: Horizontal + memory: Vertical + containerName: app + - policy: + cpu: Horizontal + memory: Vertical + containerName: istio-proxy + conditions: + containerResourceRequests: + - containerName: "app" + resource: + cpu: "10" + memory: 10Gi + - containerName: "istio-proxy" + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 30 + - containerName: istio-proxy + targetUtilization: + cpu: 30 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working + containerResourcePhases: + - containerName: "app" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working + - containerName: "istio-proxy" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/vpa-Monitor.yaml new file mode 100644 index 00000000..2b8063f1 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-back-to-working/before/vpa-Monitor.yaml @@ -0,0 +1,43 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + autoscalingPolicy: + containerPolicies: + - containerName: app + - containerName: istio-proxy + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/deployment.yaml new file mode 100644 index 00000000..96d10022 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/deployment.yaml @@ -0,0 +1,30 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + - image: awesome-istio-proxy-image + name: istio-proxy + resources: + requests: + cpu: "4" + memory: 4Gi +status: {} diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml new file mode 100644 index 00000000..9ec7057c --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/hpa.yaml @@ -0,0 +1,61 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 15 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + minReplicas: 15 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "False" + type: ScalingActive + message: "the HPA was unable to compute the replica count: failed to get cpu utilization" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 0 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 0 + desiredReplicas: 0 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml new file mode 100644 index 00000000..85478b77 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/tortoise.yaml @@ -0,0 +1,136 @@ +metadata: + finalizers: + - tortoise.autoscaling.mercari.com/finalizer + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - containerName: app + policy: + cpu: Horizontal + memory: Vertical + - containerName: istio-proxy + policy: + cpu: Horizontal + memory: Vertical + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + containerResourceRequests: + - containerName: app + resource: + cpu: "6" + memory: 3Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 3Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: The recommendation is provided + status: "True" + type: VerticalRecommendationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 30 + - containerName: istio-proxy + targetUtilization: + cpu: 30 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + scaleTargetRef: + kind: "" + name: "" + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Emergency diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/vpa-Monitor.yaml new file mode 100644 index 00000000..81846608 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/after/vpa-Monitor.yaml @@ -0,0 +1,39 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/deployment.yaml new file mode 100644 index 00000000..693bb43d --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/deployment.yaml @@ -0,0 +1,27 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/proxyCPU: "4" + sidecar.istio.io/proxyMemory: "4Gi" + labels: + app: mercari + spec: + containers: + - name: istio-proxy # will be ignored. + image: auto + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + replicas: 10 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/hpa.yaml new file mode 100644 index 00000000..5f9ba0c9 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/hpa.yaml @@ -0,0 +1,61 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "False" + type: ScalingActive + message: "the HPA was unable to compute the replica count: failed to get cpu utilization" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 0 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 0 +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 15 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + minReplicas: 3 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml new file mode 100644 index 00000000..2757b8f3 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/tortoise.yaml @@ -0,0 +1,116 @@ +metadata: + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - policy: + cpu: Horizontal + memory: Vertical + containerName: app + - policy: + cpu: Horizontal + memory: Vertical + containerName: istio-proxy + conditions: + containerResourceRequests: + - containerName: "app" + resource: + cpu: "10" + memory: 10Gi + - containerName: "istio-proxy" + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 30 + - containerName: istio-proxy + targetUtilization: + cpu: 30 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working + containerResourcePhases: + - containerName: "app" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working + - containerName: "istio-proxy" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/vpa-Monitor.yaml new file mode 100644 index 00000000..2b8063f1 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-condition/before/vpa-Monitor.yaml @@ -0,0 +1,43 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + autoscalingPolicy: + containerPolicies: + - containerName: app + - containerName: istio-proxy + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/deployment.yaml new file mode 100644 index 00000000..96d10022 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/deployment.yaml @@ -0,0 +1,30 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + kubectl.kubernetes.io/restartedAt: "2023-01-01T00:00:00Z" + creationTimestamp: null + labels: + app: mercari + spec: + containers: + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + - image: awesome-istio-proxy-image + name: istio-proxy + resources: + requests: + cpu: "4" + memory: 4Gi +status: {} diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml new file mode 100644 index 00000000..45fa2a67 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/hpa.yaml @@ -0,0 +1,61 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 15 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + minReplicas: 15 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 0 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 0 + desiredReplicas: 0 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml new file mode 100644 index 00000000..85478b77 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/tortoise.yaml @@ -0,0 +1,136 @@ +metadata: + finalizers: + - tortoise.autoscaling.mercari.com/finalizer + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - containerName: app + policy: + cpu: Horizontal + memory: Vertical + - containerName: istio-proxy + policy: + cpu: Horizontal + memory: Vertical + conditions: + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + recommendation: + cpu: + quantity: "3" + updatedAt: "2023-01-01T00:00:00Z" + memory: + quantity: 3Gi + updatedAt: "2023-01-01T00:00:00Z" + containerResourceRequests: + - containerName: app + resource: + cpu: "6" + memory: 3Gi + - containerName: istio-proxy + resource: + cpu: "4" + memory: 3Gi + tortoiseConditions: + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: HPA target utilization is updated + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + message: The recommendation is provided + status: "True" + type: VerticalRecommendationUpdated + - lastTransitionTime: "2023-01-01T00:00:00Z" + lastUpdateTime: "2023-01-01T00:00:00Z" + status: "False" + type: FailedToReconcile + containerResourcePhases: + - containerName: app + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + - containerName: istio-proxy + resourcePhases: + cpu: + lastTransitionTime: null + phase: Working + memory: + lastTransitionTime: "2023-01-01T00:00:00Z" + phase: Working + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 30 + - containerName: istio-proxy + targetUtilization: + cpu: 30 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + scaleTargetRef: + kind: "" + name: "" + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Emergency diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/vpa-Monitor.yaml new file mode 100644 index 00000000..81846608 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/after/vpa-Monitor.yaml @@ -0,0 +1,39 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/deployment.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/deployment.yaml new file mode 100644 index 00000000..693bb43d --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/deployment.yaml @@ -0,0 +1,27 @@ +metadata: + name: mercari-app + namespace: default +spec: + selector: + matchLabels: + app: mercari + strategy: {} + template: + metadata: + annotations: + sidecar.istio.io/inject: "true" + sidecar.istio.io/proxyCPU: "4" + sidecar.istio.io/proxyMemory: "4Gi" + labels: + app: mercari + spec: + containers: + - name: istio-proxy # will be ignored. + image: auto + - image: awesome-mercari-app-image + name: app + resources: + requests: + cpu: "10" + memory: 10Gi + replicas: 10 diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml new file mode 100644 index 00000000..2d42fb84 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/hpa.yaml @@ -0,0 +1,60 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-hpa-mercari + namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 0 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 0 +spec: + behavior: + scaleDown: + policies: + - periodSeconds: 90 + type: Percent + value: 2 + selectPolicy: Max + scaleUp: + policies: + - periodSeconds: 60 + type: Percent + value: 100 + selectPolicy: Max + stabilizationWindowSeconds: 0 + maxReplicas: 15 + metrics: + - containerResource: + container: app + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + - containerResource: + container: istio-proxy + name: cpu + target: + averageUtilization: 30 + type: Utilization + type: ContainerResource + minReplicas: 3 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml new file mode 100644 index 00000000..2757b8f3 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/tortoise.yaml @@ -0,0 +1,116 @@ +metadata: + name: mercari + namespace: default +spec: + targetRefs: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updateMode: Auto +status: + autoscalingPolicy: + - policy: + cpu: Horizontal + memory: Vertical + containerName: app + - policy: + cpu: Horizontal + memory: Vertical + containerName: istio-proxy + conditions: + containerResourceRequests: + - containerName: "app" + resource: + cpu: "10" + memory: 10Gi + - containerName: "istio-proxy" + resource: + cpu: "4" + memory: 4Gi + tortoiseConditions: + - message: "HPA target utilization is updated" + reason: HPATargetUtilizationUpdated + status: "True" + type: HPATargetUtilizationUpdated + containerRecommendationFromVPA: + - containerName: app + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + - containerName: istio-proxy + maxRecommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendation: + cpu: + quantity: "3" + updatedAt: null + memory: + quantity: 3Gi + updatedAt: null + recommendations: + horizontal: + maxReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 15 + minReplicas: + - from: 0 + timezone: Local + to: 24 + updatedAt: "2023-10-06T01:15:47Z" + value: 3 + targetUtilizations: + - containerName: app + targetUtilization: + cpu: 30 + - containerName: istio-proxy + targetUtilization: + cpu: 30 + vertical: + containerResourceRecommendation: + - RecommendedResource: + cpu: "6" + memory: 3Gi + containerName: app + - RecommendedResource: + cpu: "4" + memory: 3Gi + containerName: istio-proxy + targets: + horizontalPodAutoscaler: tortoise-hpa-mercari + verticalPodAutoscalers: + - name: tortoise-monitor-mercari + role: Monitor + tortoisePhase: Working + containerResourcePhases: + - containerName: "app" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working + - containerName: "istio-proxy" + resourcePhases: + cpu: + phase: Working + memory: + phase: Working diff --git a/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/vpa-Monitor.yaml b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/vpa-Monitor.yaml new file mode 100644 index 00000000..2b8063f1 --- /dev/null +++ b/internal/controller/testdata/reconcile-automatic-emergency-mode-hpa-no-metrics/before/vpa-Monitor.yaml @@ -0,0 +1,43 @@ +metadata: + annotations: + tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" + name: tortoise-monitor-mercari + namespace: default +spec: + autoscalingPolicy: + containerPolicies: + - containerName: app + - containerName: istio-proxy + targetRef: + apiVersion: apps/v1 + kind: Deployment + name: mercari-app + updatePolicy: + updateMode: "Off" +status: + conditions: + - lastTransitionTime: null + status: "True" + type: RecommendationProvided + recommendation: + containerRecommendations: + - containerName: app + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi + - containerName: istio-proxy + lowerBound: + cpu: "3" + memory: 3Gi + target: + cpu: "3" + memory: 3Gi + upperBound: + cpu: "5" + memory: 5Gi diff --git a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/hpa.yaml index 836c01e8..2d976100 100644 --- a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/after/hpa.yaml @@ -40,5 +40,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-istio-enabled-pod-working/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/hpa.yaml index 0cb536c2..a102d9c1 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/after/hpa.yaml @@ -40,5 +40,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/before/hpa.yaml index aeeabf2b..3a70606b 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-during-emergency/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/hpa.yaml index 0cb536c2..a102d9c1 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/after/hpa.yaml @@ -40,5 +40,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/before/hpa.yaml index d1d756fd..fe5f26b0 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-emergency-started/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/hpa.yaml index 0eb6b7e6..bfc7d4d1 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-one-off/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/hpa.yaml index 47e4e117..3dbd5407 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/after/hpa.yaml @@ -40,5 +40,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 \ No newline at end of file diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/before/hpa.yaml index 4f9031a6..9dd43455 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-suggested-too-small/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/hpa.yaml index 836c01e8..2d976100 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/after/hpa.yaml @@ -40,5 +40,21 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/before/hpa.yaml index 54cef9df..9bbc7c95 100644 --- a/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-multiple-containers-pod-working/before/hpa.yaml @@ -3,6 +3,25 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 + - containerResource: + container: istio-proxy + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/after/hpa.yaml index 1b871784..87aabcd3 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/before/hpa.yaml index 9c2126fa..e5c83a34 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-backtonormal/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/after/hpa.yaml index 2924abbc..a0719772 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 \ No newline at end of file diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-dryrun/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/after/hpa.yaml index 16758a47..b3fee935 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/before/hpa.yaml index 087cff6f..7836cac4 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-during-emergency/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/after/hpa.yaml index dc35b0ab..b1b60f03 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/before/hpa.yaml index bdf08d5d..2e1bdfb5 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-emergency-started/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/hpa.yaml index 7c9ab2d5..450a8cd5 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data-finished/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/after/hpa.yaml index 2924abbc..a0c6bcf1 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-gathering-data/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml index d4accc33..cf3a7e97 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/after/hpa.yaml @@ -40,5 +40,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml index 63f804f9..9b68a123 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-hpa-changed/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml index 2924abbc..a0c6bcf1 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-initializing/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml index 2924abbc..a0c6bcf1 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/after/hpa.yaml @@ -33,5 +33,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/before/hpa.yaml index 98160014..bdd7234f 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-partly-working/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/hpa.yaml index b8db9d51..8ccddef9 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/after/hpa.yaml @@ -40,5 +40,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/before/hpa.yaml index 8f992d7e..663e7e30 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-too-big/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/hpa.yaml index 0ea1203f..7c1e45ef 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/after/hpa.yaml @@ -40,5 +40,16 @@ spec: kind: Deployment name: mercari-app status: - currentMetrics: null - desiredReplicas: 0 + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 diff --git a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/before/hpa.yaml b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/before/hpa.yaml index 8f992d7e..663e7e30 100644 --- a/internal/controller/testdata/reconcile-for-the-single-container-pod-working/before/hpa.yaml +++ b/internal/controller/testdata/reconcile-for-the-single-container-pod-working/before/hpa.yaml @@ -3,6 +3,20 @@ metadata: tortoise.autoscaling.mercari.com/managed-by-tortoise: "true" name: tortoise-hpa-mercari namespace: default +status: + conditions: + - status: "True" + type: AbleToScale + message: "recommended size matches current size" + - status: "True" + type: ScalingActive + message: "the HPA was able to compute the replica count" + currentMetrics: + - containerResource: + container: app + name: cpu + current: + value: 3 spec: behavior: scaleDown: diff --git a/internal/controller/tortoise_controller.go b/internal/controller/tortoise_controller.go index f08fe9f6..3ce632c8 100644 --- a/internal/controller/tortoise_controller.go +++ b/internal/controller/tortoise_controller.go @@ -185,6 +185,7 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ return ctrl.Result{}, err } tortoise = tortoiseService.UpdateTortoiseAutoscalingPolicyInStatus(tortoise, hpa, now) + tortoise = r.TortoiseService.UpdateTortoisePhase(tortoise, now) if tortoise.Status.TortoisePhase == autoscalingv1beta3.TortoisePhaseInitializing { logger.Info("initializing tortoise", "tortoise", req.NamespacedName) @@ -233,12 +234,25 @@ func (r *TortoiseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_ tortoise = vpa.SetAllVerticalContainerResourcePhaseWorking(tortoise, now) logger.Info("VPA created by tortoise is ready, proceeding to generate the recommendation", "tortoise", req.NamespacedName) - hpa, err = r.HpaService.GetHPAOnTortoise(ctx, tortoise) + hpa, isReady, err := r.HpaService.GetHPAOnTortoise(ctx, tortoise) if err != nil { logger.Error(err, "failed to get HPA", "tortoise", req.NamespacedName) return ctrl.Result{}, err } + if !isReady { + // HPA is correctly fetched, but looks like not ready yet. We won't be able to calculate things correctly, and hence stop the reconciliation here. + logger.Info("HPA on tortoise is not ready, don't reconcile now and will retry later", "hpa", hpa.Name) + return ctrl.Result{RequeueAfter: r.Interval}, nil + } + scalingActive := r.HpaService.IsHpaMetricAvailable(ctx, hpa) + + tortoise, err = r.TortoiseService.UpdateTortoisePhaseIfHPAIsUnhealthy(ctx, scalingActive, tortoise) + if err != nil { + logger.Error(err, "Tortoise could not switch to emergency mode", "tortoise", req.NamespacedName) + return ctrl.Result{}, err + } + tortoise = r.TortoiseService.UpdateContainerRecommendationFromVPA(tortoise, monitorvpa, now) tortoise, err = r.RecommenderService.UpdateRecommendations(ctx, tortoise, hpa, currentDesiredReplicaNum, now) diff --git a/internal/controller/tortoise_controller_test.go b/internal/controller/tortoise_controller_test.go index 93a9e33d..2f7f01e4 100644 --- a/internal/controller/tortoise_controller_test.go +++ b/internal/controller/tortoise_controller_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "reflect" "time" "github.com/google/go-cmp/cmp" @@ -121,11 +122,25 @@ func createTortoiseWithStatus(ctx context.Context, k8sClient client.Client, tort Expect(err).NotTo(HaveOccurred()) } +func createHPAWithStatus(ctx context.Context, k8sClient client.Client, hpa *v2.HorizontalPodAutoscaler) { + err := k8sClient.Create(ctx, hpa.DeepCopy()) + Expect(err).NotTo(HaveOccurred()) + + h := &v2.HorizontalPodAutoscaler{} + err = k8sClient.Get(ctx, client.ObjectKey{Namespace: hpa.Namespace, Name: hpa.Name}, h) + Expect(err).NotTo(HaveOccurred()) + + if !reflect.DeepEqual(hpa.Status, v2.HorizontalPodAutoscalerStatus{}) { + h.Status = hpa.Status + err = k8sClient.Status().Update(ctx, h) + Expect(err).NotTo(HaveOccurred()) + } +} + func initializeResourcesFromFiles(ctx context.Context, k8sClient client.Client, path string) resources { resource := newResource(path) if resource.hpa != nil { - err := k8sClient.Create(ctx, resource.hpa) - Expect(err).NotTo(HaveOccurred()) + createHPAWithStatus(ctx, k8sClient, resource.hpa) } createDeploymentWithStatus(ctx, k8sClient, resource.deployment) @@ -171,7 +186,7 @@ func updateResourcesInTestCaseFile(path string, resource resources) error { } if resource.hpa != nil { - err = writeToFile(filepath.Join(path, "hpa.yaml"), removeUnnecessaryFieldsFromHPA(resource.hpa)) + err = writeToFile(filepath.Join(path, "hpa.yaml"), resource.hpa) if err != nil { return err } @@ -180,11 +195,6 @@ func updateResourcesInTestCaseFile(path string, resource resources) error { return nil } -func removeUnnecessaryFieldsFromHPA(hpa *v2.HorizontalPodAutoscaler) *v2.HorizontalPodAutoscaler { - hpa.Status = v2.HorizontalPodAutoscalerStatus{} - return hpa -} - func removeUnnecessaryFieldsFromDeployment(deployment *v1.Deployment) *v1.Deployment { // remove all default values @@ -477,6 +487,17 @@ var _ = Describe("Test TortoiseController", func() { runTest(filepath.Join("testdata", "mutable-autoscalingpolicy-remove-horizontal-2")) }) }) + Context("automatic switch to emergency mode", func() { + It("HPA scalingactive condition false", func() { + runTest(filepath.Join("testdata", "reconcile-automatic-emergency-mode-hpa-condition")) + }) + It("HPA scalingactive no metrics", func() { + runTest(filepath.Join("testdata", "reconcile-automatic-emergency-mode-hpa-no-metrics")) + }) + It("Tortoise changes the status back to Working if it finds HPA is working fine now", func() { + runTest(filepath.Join("testdata", "reconcile-automatic-emergency-mode-hpa-back-to-working")) + }) + }) Context("DeletionPolicy is handled correctly", func() { It("[DeletionPolicy = DeleteAll] delete HPA and VPA when Tortoise is deleted", func() { resource := initializeResourcesFromFiles(ctx, k8sClient, "testdata/deletion-policy-all/before") diff --git a/pkg/hpa/service.go b/pkg/hpa/service.go index 444f98da..8574982a 100644 --- a/pkg/hpa/service.go +++ b/pkg/hpa/service.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "reflect" "regexp" "sort" "time" @@ -285,19 +286,23 @@ func (c *Service) GetHPAOnTortoiseSpec(ctx context.Context, tortoise *autoscalin return hpa, nil } -func (c *Service) GetHPAOnTortoise(ctx context.Context, tortoise *autoscalingv1beta3.Tortoise) (*v2.HorizontalPodAutoscaler, error) { +func (c *Service) GetHPAOnTortoise(ctx context.Context, tortoise *autoscalingv1beta3.Tortoise) (*v2.HorizontalPodAutoscaler, bool, error) { if !HasHorizontal(tortoise) { // there should be no HPA - return nil, nil + return nil, true, nil } hpa := &v2.HorizontalPodAutoscaler{} if err := c.c.Get(ctx, types.NamespacedName{Namespace: tortoise.Namespace, Name: tortoise.Status.Targets.HorizontalPodAutoscaler}, hpa); err != nil { - return nil, fmt.Errorf("failed to get hpa on tortoise: %w", err) + return nil, false, fmt.Errorf("failed to get hpa on tortoise: %w", err) + } + if reflect.DeepEqual(hpa.Status, v2.HorizontalPodAutoscalerStatus{}) || hpa.Status.Conditions == nil || hpa.Status.CurrentMetrics == nil { + // Most likely, HPA is just created and not yet handled by HPA controller. + return nil, false, nil } recordHPAMetric(ctx, tortoise, hpa) - return hpa, nil + return hpa, true, nil } func (s *Service) UpdatingHPATargetUtilizationAllowed(tortoise *autoscalingv1beta3.Tortoise, now time.Time) (*autoscalingv1beta3.Tortoise, bool) { @@ -765,3 +770,33 @@ func (c *Service) excludeExternalMetric(ctx context.Context, hpa *v2.HorizontalP return newHPA } + +func (c *Service) IsHpaMetricAvailable(ctx context.Context, currenthpa *v2.HorizontalPodAutoscaler) bool { + logger := log.FromContext(ctx) + if currenthpa == nil || reflect.DeepEqual(currenthpa.Status, v2.HorizontalPodAutoscalerStatus{}) || len(currenthpa.Status.Conditions) == 0 || len(currenthpa.Status.CurrentMetrics) == 0 { + // shouldn't reach here because, in this HPA unready case, the controller should stop the reconciliation at the point of fetching hpa. + logger.Error(nil, "invalid container resource metric", "hpa", klog.KObj(currenthpa)) + return false + } + + conditions := currenthpa.Status.Conditions + currentMetrics := currenthpa.Status.CurrentMetrics + + for _, condition := range conditions { + if condition.Type == "ScalingActive" && condition.Status == "False" && condition.Reason == "FailedGetResourceMetric" { + // switch to Emergency mode since no metrics + logger.Info("HPA failed to get resource metrics, switch to emergency mode") + return false + } + } + + for _, currentMetric := range currentMetrics { + if !currentMetric.ContainerResource.Current.Value.IsZero() { + // Can still get metrics for some containers, they can scale based on those + return true + } + } + logger.Info("HPA looks unready because all the metrics indicate zero", "hpa", currenthpa.Name) + return false + +} diff --git a/pkg/hpa/service_test.go b/pkg/hpa/service_test.go index 4891613f..248dc59b 100644 --- a/pkg/hpa/service_test.go +++ b/pkg/hpa/service_test.go @@ -10,6 +10,7 @@ import ( v2 "k8s.io/api/autoscaling/v2" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + resource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" "k8s.io/utils/ptr" @@ -4666,3 +4667,349 @@ func TestService_UpdateHPASpecFromTortoiseAutoscalingPolicy(t *testing.T) { }) } } + +func TestService_IsHpaMetricAvailable(t *testing.T) { + tests := []struct { + name string + HPA *v2.HorizontalPodAutoscaler + result bool + }{ + { + name: "metric server down, should return false", + HPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-hpa", + Namespace: "default", + }, + Status: v2.HorizontalPodAutoscalerStatus{ + Conditions: []v2.HorizontalPodAutoscalerCondition{ + { + Status: "True", + Type: v2.AbleToScale, + Message: "recommended size matches current size", + }, + { + Status: "False", + Type: v2.ScalingActive, + Message: "the HPA was unable to compute the replica count: failed to get cpu utilization", + }, + { + Status: "False", + Type: v2.ScalingLimited, + Message: "the desired count is within the acceptable range", + }, + }, + CurrentMetrics: []v2.MetricStatus{ + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "app", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](0), + AverageValue: resource.NewQuantity(0, resource.DecimalSI), + Value: resource.NewQuantity(0, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "istio-proxy", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](0), + AverageValue: resource.NewQuantity(0, resource.DecimalSI), + Value: resource.NewQuantity(0, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 1000, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + }, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + result: false, + }, + { + name: "Container resource metric missing", + HPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-hpa", + Namespace: "default", + }, + Status: v2.HorizontalPodAutoscalerStatus{ + Conditions: []v2.HorizontalPodAutoscalerCondition{ + { + Status: "True", + Type: v2.AbleToScale, + Message: "recommended size matches current size", + }, + { + Status: "True", + Type: v2.ScalingActive, + Message: "the HPA was able to successfully calculate a replica count from cpu container resource utilization (percentage of request)", + }, + { + Status: "False", + Type: v2.ScalingLimited, + Message: "the desired count is within the acceptable range", + }, + }, + CurrentMetrics: []v2.MetricStatus{ + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "app", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](0), + AverageValue: resource.NewQuantity(0, resource.DecimalSI), + Value: resource.NewQuantity(0, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "istio-proxy", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](0), + AverageValue: resource.NewQuantity(0, resource.DecimalSI), + Value: resource.NewQuantity(0, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 1000, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + }, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + result: false, + }, + { + name: "HPA working normally", + HPA: &v2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Name: "existing-hpa", + Namespace: "default", + }, + Status: v2.HorizontalPodAutoscalerStatus{ + Conditions: []v2.HorizontalPodAutoscalerCondition{ + { + Status: "True", + Type: v2.AbleToScale, + Message: "recommended size matches current size", + }, + { + Status: "True", + Type: v2.ScalingActive, + Message: "the HPA was able to successfully calculate a replica count from cpu container resource utilization (percentage of request)", + }, + { + Status: "False", + Type: v2.ScalingLimited, + Message: "the desired count is within the acceptable range", + }, + }, + CurrentMetrics: []v2.MetricStatus{ + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "app", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](70), + AverageValue: resource.NewQuantity(5, resource.DecimalSI), + Value: resource.NewQuantity(5, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + { + Type: "ContainerResource", + ContainerResource: &v2.ContainerResourceMetricStatus{ + Container: "istio-proxy", + Current: v2.MetricValueStatus{ + AverageUtilization: ptr.To[int32](70), + AverageValue: resource.NewQuantity(5, resource.DecimalSI), + Value: resource.NewQuantity(5, resource.DecimalSI), + }, + Name: v1.ResourceCPU, + }, + }, + }, + }, + Spec: v2.HorizontalPodAutoscalerSpec{ + MinReplicas: ptrInt32(3), + MaxReplicas: 1000, + Metrics: []v2.MetricSpec{ + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "app", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + { + Type: v2.ContainerResourceMetricSourceType, + ContainerResource: &v2.ContainerResourceMetricSource{ + Name: v1.ResourceCPU, + Container: "istio-proxy", + Target: v2.MetricTarget{ + AverageUtilization: ptr.To[int32](70), + Type: v2.UtilizationMetricType, + }, + }, + }, + }, + ScaleTargetRef: v2.CrossVersionObjectReference{ + Kind: "Deployment", + Name: "deployment", + APIVersion: "apps/v1", + }, + Behavior: &v2.HorizontalPodAutoscalerBehavior{ + ScaleUp: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 100, + PeriodSeconds: 60, + }, + }, + }, + ScaleDown: &v2.HPAScalingRules{ + Policies: []v2.HPAScalingPolicy{ + { + Type: v2.PercentScalingPolicy, + Value: 2, + PeriodSeconds: 90, + }, + }, + }, + }, + }, + }, + result: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c, err := New(fake.NewClientBuilder().Build(), record.NewFakeRecorder(10), 0.95, 90, 100, time.Hour, 100, 1000, 3, "") + if err != nil { + t.Fatalf("New() error = %v", err) + } + status := c.IsHpaMetricAvailable(context.Background(), tt.HPA) + if status != tt.result { + t.Errorf("Service.checkHpaMetricStatus() status test: %s failed", tt.name) + return + } + }) + } +} diff --git a/pkg/tortoise/tortoise.go b/pkg/tortoise/tortoise.go index ef1e0490..a49a57f7 100644 --- a/pkg/tortoise/tortoise.go +++ b/pkg/tortoise/tortoise.go @@ -816,3 +816,11 @@ func recommendationIncreaseAnyResource(oldTortoise, newTortoise *v1beta3.Tortois return false } + +func (c *Service) UpdateTortoisePhaseIfHPAIsUnhealthy(ctx context.Context, scalingActive bool, tortoise *v1beta3.Tortoise) (*v1beta3.Tortoise, error) { + if !scalingActive && tortoise.Spec.UpdateMode == v1beta3.UpdateModeAuto && tortoise.Status.TortoisePhase == v1beta3.TortoisePhaseWorking { + log.FromContext(ctx).Info("switching Tortoise to Emergency mode because looks like HPA isn't working properly") + tortoise.Status.TortoisePhase = v1beta3.TortoisePhaseEmergency + } + return tortoise, nil +} diff --git a/pkg/tortoise/tortoise_test.go b/pkg/tortoise/tortoise_test.go index b90e26f1..20a80012 100644 --- a/pkg/tortoise/tortoise_test.go +++ b/pkg/tortoise/tortoise_test.go @@ -4605,3 +4605,114 @@ func TestService_UpdateResourceRequest(t *testing.T) { }) } } + +func TestService_UpdateTortoisePhaseIfHPAIsUnhealthy(t *testing.T) { + type args struct { + t *v1beta3.Tortoise + scalingActive bool + } + tests := []struct { + name string + args args + wantTortoise *v1beta3.Tortoise + }{ + { + name: "healthy HPA tortoise working", + args: args{ + t: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + }, + scalingActive: true, + }, + wantTortoise: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test", ResourceVersion: "1"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + }, + }, + { + name: "unhealthy HPA tortoise working", + args: args{ + t: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + }, + scalingActive: false, + }, + wantTortoise: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test", ResourceVersion: "1"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseEmergency, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeAuto, + }, + }, + }, + { + name: "unhealthy HPA tortoise off", + args: args{ + t: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeOff, + }, + }, + scalingActive: false, + }, + wantTortoise: &v1beta3.Tortoise{ + ObjectMeta: metav1.ObjectMeta{Name: "t", Namespace: "test", ResourceVersion: "1"}, + Status: v1beta3.TortoiseStatus{ + TortoisePhase: v1beta3.TortoisePhaseWorking, + }, + Spec: v1beta3.TortoiseSpec{ + UpdateMode: v1beta3.UpdateModeOff, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + scheme := runtime.NewScheme() + err := v1beta3.AddToScheme(scheme) + if err != nil { + t.Fatalf("failed to add to scheme: %v", err) + } + c := fake.NewClientBuilder().WithScheme(scheme).Build() + err = c.Create(context.Background(), tt.args.t) + if err != nil { + t.Fatalf("create tortoise: %v", err) + } + s := &Service{ + c: c, + lastTimeUpdateTortoise: make(map[client.ObjectKey]time.Time), + } + + tortoise, err := s.UpdateTortoisePhaseIfHPAIsUnhealthy(context.Background(), tt.args.scalingActive, tt.args.t) + if err != nil { + t.Fatalf("failed to update tortoise phase: %v", err) + } + if d := cmp.Diff(tortoise, tt.wantTortoise); d != "" { + t.Errorf("UpdateTortoiseStatus() diff = %v", d) + } + }) + } +}