From 477b94a7dbfcea24ba631afb92ed3cd25db02dea Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 29 Jul 2021 17:47:22 +0800 Subject: [PATCH 01/28] move to autoscaling v2beta2 --- controllers/function.go | 4 ++-- controllers/function_controller.go | 4 ++-- controllers/sink.go | 4 ++-- controllers/sink_controller.go | 4 ++-- controllers/source.go | 4 ++-- controllers/source_controller.go | 4 ++-- controllers/spec/common.go | 28 ++++++++++++++++++++-------- controllers/spec/function.go | 4 ++-- controllers/spec/sink.go | 4 ++-- controllers/spec/source.go | 4 ++-- 10 files changed, 38 insertions(+), 26 deletions(-) diff --git a/controllers/function.go b/controllers/function.go index ea888b8c4..6bdb73460 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -23,7 +23,7 @@ import ( "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -186,7 +186,7 @@ func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Re return nil } - hpa := &autov1.HorizontalPodAutoscaler{} + hpa := &autov2beta2.HorizontalPodAutoscaler{} err := r.Get(ctx, types.NamespacedName{Namespace: function.Namespace, Name: spec.MakeFunctionObjectMeta(function).Name}, hpa) if err != nil { diff --git a/controllers/function_controller.go b/controllers/function_controller.go index 83669558a..2d4f51adb 100644 --- a/controllers/function_controller.go +++ b/controllers/function_controller.go @@ -23,7 +23,7 @@ import ( "github.com/go-logr/logr" "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -106,7 +106,7 @@ func (r *FunctionReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&v1alpha1.Function{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). - Owns(&autov1.HorizontalPodAutoscaler{}). + Owns(&autov2beta2.HorizontalPodAutoscaler{}). Owns(&corev1.Secret{}). Complete(r) } diff --git a/controllers/sink.go b/controllers/sink.go index 965a36c54..2242db35a 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -23,7 +23,7 @@ import ( "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -166,7 +166,7 @@ func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, s Action: v1alpha1.NoAction, } - hpa := &autov1.HorizontalPodAutoscaler{} + hpa := &autov2beta2.HorizontalPodAutoscaler{} err := r.Get(ctx, types.NamespacedName{Namespace: sink.Namespace, Name: spec.MakeSinkObjectMeta(sink).Name}, hpa) if err != nil { diff --git a/controllers/sink_controller.go b/controllers/sink_controller.go index e78ec8727..708663467 100644 --- a/controllers/sink_controller.go +++ b/controllers/sink_controller.go @@ -23,7 +23,7 @@ import ( "github.com/go-logr/logr" computev1alpha1 "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -104,6 +104,6 @@ func (r *SinkReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&computev1alpha1.Sink{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). - Owns(&autov1.HorizontalPodAutoscaler{}). + Owns(&autov2beta2.HorizontalPodAutoscaler{}). Complete(r) } diff --git a/controllers/source.go b/controllers/source.go index 79f904a93..8bb06ba16 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -23,7 +23,7 @@ import ( "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -168,7 +168,7 @@ func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Reques Action: v1alpha1.NoAction, } - hpa := &autov1.HorizontalPodAutoscaler{} + hpa := &autov2beta2.HorizontalPodAutoscaler{} err := r.Get(ctx, types.NamespacedName{Namespace: source.Namespace, Name: spec.MakeSourceObjectMeta(source).Name}, hpa) if err != nil { diff --git a/controllers/source_controller.go b/controllers/source_controller.go index 21eba31b4..8963a4a66 100644 --- a/controllers/source_controller.go +++ b/controllers/source_controller.go @@ -23,7 +23,7 @@ import ( "github.com/go-logr/logr" computev1alpha1 "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -104,6 +104,6 @@ func (r *SourceReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&computev1alpha1.Source{}). Owns(&appsv1.StatefulSet{}). Owns(&corev1.Service{}). - Owns(&autov1.HorizontalPodAutoscaler{}). + Owns(&autov2beta2.HorizontalPodAutoscaler{}). Complete(r) } diff --git a/controllers/spec/common.go b/controllers/spec/common.go index d3f766086..76479a9f6 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -24,11 +24,12 @@ import ( "strings" "time" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" + "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/proto" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -97,24 +98,35 @@ func MakeHeadlessServiceName(serviceName string) string { } func MakeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, - kind string) *autov1.HorizontalPodAutoscaler { + kind string) *autov2beta2.HorizontalPodAutoscaler { // TODO: configurable cpu percentage cpuPercentage := int32(80) - return &autov1.HorizontalPodAutoscaler{ + return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ Kind: "autoscaling/v1", APIVersion: "HorizontalPodAutoscaler", }, ObjectMeta: *objectMeta, - Spec: autov1.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov1.CrossVersionObjectReference{ + Spec: autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ Kind: kind, Name: objectMeta.Name, APIVersion: "compute.functionmesh.io/v1alpha1", }, - MinReplicas: &minReplicas, - MaxReplicas: maxReplicas, - TargetCPUUtilizationPercentage: &cpuPercentage, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: []autov2beta2.MetricSpec{ + { + Type: autov2beta2.ResourceMetricSourceType, + Resource: &autov2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autov2beta2.MetricTarget{ + Type: autov2beta2.UtilizationMetricType, + AverageUtilization: &cpuPercentage, + }, + }, + }, + }, }, } } diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 66b391686..595d0f39a 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -21,7 +21,7 @@ import ( "github.com/gogo/protobuf/jsonpb" "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" logf "sigs.k8s.io/controller-runtime/pkg/log" @@ -30,7 +30,7 @@ import ( // log is for logging in this package. var log = logf.Log.WithName("sink-resource") -func MakeFunctionHPA(function *v1alpha1.Function) *autov1.HorizontalPodAutoscaler { +func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeFunctionObjectMeta(function) return MakeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) } diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index 2cead0730..4e1e58050 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -21,12 +21,12 @@ import ( "github.com/gogo/protobuf/jsonpb" "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func MakeSinkHPA(sink *v1alpha1.Sink) *autov1.HorizontalPodAutoscaler { +func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) return MakeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) } diff --git a/controllers/spec/source.go b/controllers/spec/source.go index 5c33d1204..c3e87266f 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -21,12 +21,12 @@ import ( "github.com/gogo/protobuf/jsonpb" "github.com/streamnative/function-mesh/api/v1alpha1" appsv1 "k8s.io/api/apps/v1" - autov1 "k8s.io/api/autoscaling/v1" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func MakeSourceHPA(source *v1alpha1.Source) *autov1.HorizontalPodAutoscaler { +func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSourceObjectMeta(source) return MakeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) } From 62a25ada9ad9a5a5688c12255af4771908380bfa Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Fri, 30 Jul 2021 10:50:39 +0800 Subject: [PATCH 02/28] add defaultHPAMetrics --- controllers/spec/common.go | 15 +------------- controllers/spec/hpa.go | 41 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 controllers/spec/hpa.go diff --git a/controllers/spec/common.go b/controllers/spec/common.go index 76479a9f6..a7340cec9 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -99,8 +99,6 @@ func MakeHeadlessServiceName(serviceName string) string { func MakeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, kind string) *autov2beta2.HorizontalPodAutoscaler { - // TODO: configurable cpu percentage - cpuPercentage := int32(80) return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ Kind: "autoscaling/v1", @@ -115,18 +113,7 @@ func MakeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, }, MinReplicas: &minReplicas, MaxReplicas: maxReplicas, - Metrics: []autov2beta2.MetricSpec{ - { - Type: autov2beta2.ResourceMetricSourceType, - Resource: &autov2beta2.ResourceMetricSource{ - Name: corev1.ResourceCPU, - Target: autov2beta2.MetricTarget{ - Type: autov2beta2.UtilizationMetricType, - AverageUtilization: &cpuPercentage, - }, - }, - }, - }, + Metrics: defaultHPAMetrics(), }, } } diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go new file mode 100644 index 000000000..e84baaea1 --- /dev/null +++ b/controllers/spec/hpa.go @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package spec + +import ( + autov2beta2 "k8s.io/api/autoscaling/v2beta2" + corev1 "k8s.io/api/core/v1" +) + +// defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. +func defaultHPAMetrics() []autov2beta2.MetricSpec { + // TODO: configurable cpu percentage + cpuPercentage := int32(80) + return []autov2beta2.MetricSpec{ + { + Type: autov2beta2.ResourceMetricSourceType, + Resource: &autov2beta2.ResourceMetricSource{ + Name: corev1.ResourceCPU, + Target: autov2beta2.MetricTarget{ + Type: autov2beta2.UtilizationMetricType, + AverageUtilization: &cpuPercentage, + }, + }, + }, + } +} From 7d2e0c533d1587359e05e0b9d51855954357cf77 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 16:09:47 +0800 Subject: [PATCH 03/28] add HorizontalPodAutoscalerSpec to PodPolicy --- api/v1alpha1/common.go | 8 + api/v1alpha1/zz_generated.deepcopy.go | 6 + ...ompute.functionmesh.io_functionmeshes.yaml | 813 +++++++ .../compute.functionmesh.io_functions.yaml | 271 +++ .../bases/compute.functionmesh.io_sinks.yaml | 271 +++ .../compute.functionmesh.io_sources.yaml | 271 +++ controllers/function.go | 4 +- controllers/sink.go | 4 +- controllers/source.go | 4 +- controllers/spec/common.go | 24 +- controllers/spec/function.go | 6 +- controllers/spec/sink.go | 6 +- controllers/spec/source.go | 6 +- manifests/crd.yaml | 1918 +++++++++++++++-- 14 files changed, 3456 insertions(+), 156 deletions(-) diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index 9c7ac7660..f582940bf 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -20,6 +20,7 @@ package v1alpha1 import ( "encoding/json" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -93,6 +94,13 @@ type PodPolicy struct { // ServiceAccountName is the name of the ServiceAccount to use to run this pod. // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` + + // HPAutoscaler allows user to configure the HPA with k8s.io/api/autoscaling/v2beta2 + // This config will override the maxReplicas in Function/Source/Sink Spec + // If user set HPAutoscaler, then controller will use HPAutoscaler to create HPA + // More info: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ + // +optional + HPAutoscaler *autov2beta2.HorizontalPodAutoscalerSpec `json:"hpAutoscaler,omitempty"` } type Runtime struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 20a45b24a..b68c1509d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -21,6 +21,7 @@ limitations under the License. package v1alpha1 import ( + "k8s.io/api/autoscaling/v2beta2" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -582,6 +583,11 @@ func (in *PodPolicy) DeepCopyInto(out *PodPolicy) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.HPAutoscaler != nil { + in, out := &in.HPAutoscaler, &out.HPAutoscaler + *out = new(v2beta2.HorizontalPodAutoscalerSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodPolicy. diff --git a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml index dfdec7497..08d0e71ea 100644 --- a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml +++ b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml @@ -478,6 +478,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: @@ -2651,6 +2922,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: @@ -4804,6 +5346,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_functions.yaml b/config/crd/bases/compute.functionmesh.io_functions.yaml index bf21dd5c9..28ed3134f 100644 --- a/config/crd/bases/compute.functionmesh.io_functions.yaml +++ b/config/crd/bases/compute.functionmesh.io_functions.yaml @@ -479,6 +479,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sinks.yaml b/config/crd/bases/compute.functionmesh.io_sinks.yaml index 652705200..c49dc8b8f 100644 --- a/config/crd/bases/compute.functionmesh.io_sinks.yaml +++ b/config/crd/bases/compute.functionmesh.io_sinks.yaml @@ -413,6 +413,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sources.yaml b/config/crd/bases/compute.functionmesh.io_sources.yaml index 006d7cf83..8dc3aa1e3 100644 --- a/config/crd/bases/compute.functionmesh.io_sources.yaml +++ b/config/crd/bases/compute.functionmesh.io_sources.yaml @@ -390,6 +390,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: diff --git a/controllers/function.go b/controllers/function.go index 6bdb73460..6157baa68 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -167,7 +167,7 @@ func (r *FunctionReconciler) ApplyFunctionService(ctx context.Context, req ctrl. func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Request, function *v1alpha1.Function) error { - if function.Spec.MaxReplicas == nil { + if function.Spec.Pod.HPAutoscaler == nil && function.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -205,7 +205,7 @@ func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Re func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Request, function *v1alpha1.Function) error { - if function.Spec.MaxReplicas == nil { + if function.Spec.Pod.HPAutoscaler == nil && function.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } diff --git a/controllers/sink.go b/controllers/sink.go index 2242db35a..5c7839a76 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -155,7 +155,7 @@ func (r *SinkReconciler) ApplySinkService(ctx context.Context, req ctrl.Request, } func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, sink *v1alpha1.Sink) error { - if sink.Spec.MaxReplicas == nil { + if sink.Spec.Pod.HPAutoscaler == nil && sink.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -187,7 +187,7 @@ func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, s } func (r *SinkReconciler) ApplySinkHPA(ctx context.Context, req ctrl.Request, sink *v1alpha1.Sink) error { - if sink.Spec.MaxReplicas == nil { + if sink.Spec.Pod.HPAutoscaler == nil && sink.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } diff --git a/controllers/source.go b/controllers/source.go index 8bb06ba16..21eba8aae 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -157,7 +157,7 @@ func (r *SourceReconciler) ApplySourceService(ctx context.Context, req ctrl.Requ } func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Request, source *v1alpha1.Source) error { - if source.Spec.MaxReplicas == nil { + if source.Spec.Pod.HPAutoscaler == nil && source.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -189,7 +189,7 @@ func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Reques } func (r *SourceReconciler) ApplySourceHPA(ctx context.Context, req ctrl.Request, source *v1alpha1.Source) error { - if source.Spec.MaxReplicas == nil { + if source.Spec.Pod.HPAutoscaler == nil && source.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } diff --git a/controllers/spec/common.go b/controllers/spec/common.go index a7340cec9..7230c8df2 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -97,7 +97,7 @@ func MakeHeadlessServiceName(serviceName string) string { return fmt.Sprintf("%s-headless", serviceName) } -func MakeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, +func MakeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, kind string) *autov2beta2.HorizontalPodAutoscaler { return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ @@ -118,6 +118,28 @@ func MakeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, } } +func MakeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, + kind string) *autov2beta2.HorizontalPodAutoscaler { + return &autov2beta2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + Kind: "autoscaling/v1", + APIVersion: "HorizontalPodAutoscaler", + }, + ObjectMeta: *objectMeta, + Spec: autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ + Kind: kind, + Name: objectMeta.Name, + APIVersion: "compute.functionmesh.io/v1alpha1", + }, + MinReplicas: autoscalerSpec.MinReplicas, + MaxReplicas: autoscalerSpec.MaxReplicas, + Metrics: autoscalerSpec.Metrics, + Behavior: autoscalerSpec.Behavior, + }, + } +} + func MakeStatefulSet(objectMeta *metav1.ObjectMeta, replicas *int32, container *corev1.Container, volumes []corev1.Volume, labels map[string]string, policy v1alpha1.PodPolicy) *appsv1.StatefulSet { return &appsv1.StatefulSet{ diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 595d0f39a..def77bc71 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -32,7 +32,11 @@ var log = logf.Log.WithName("sink-resource") func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeFunctionObjectMeta(function) - return MakeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) + if function.Spec.Pod.HPAutoscaler == nil { + return MakeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) + } else { + return MakeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) + } } func MakeFunctionService(function *v1alpha1.Function) *corev1.Service { diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index 4e1e58050..f150cee8a 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -28,7 +28,11 @@ import ( func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) - return MakeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) + if sink.Spec.Pod.HPAutoscaler == nil { + return MakeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) + } else { + return MakeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) + } } func MakeSinkService(sink *v1alpha1.Sink) *corev1.Service { diff --git a/controllers/spec/source.go b/controllers/spec/source.go index c3e87266f..8e1e6ba53 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -28,7 +28,11 @@ import ( func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSourceObjectMeta(source) - return MakeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) + if source.Spec.Pod.HPAutoscaler == nil { + return MakeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) + } else { + return MakeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) + } } func MakeSourceService(source *v1alpha1.Source) *corev1.Service { diff --git a/manifests/crd.yaml b/manifests/crd.yaml index ab34957c6..59ada4b0f 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -476,6 +476,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: @@ -2649,59 +2920,330 @@ spec: additionalProperties: type: string type: object - imagePullSecrets: - items: - properties: - name: - type: string - type: object - type: array - initContainers: - items: - properties: - args: - items: - type: string - type: array - command: - items: - type: string - type: array - env: - items: + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: properties: - name: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: type: string - value: + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: type: string - valueFrom: - properties: - configMapKeyRef: - properties: - key: - type: string - name: - type: string - optional: - type: boolean - required: - - key - type: object - fieldRef: - properties: - apiVersion: - type: string - fieldPath: - type: string - required: - - fieldPath - type: object - resourceFieldRef: - properties: - containerName: - type: string - divisor: - type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object + imagePullSecrets: + items: + properties: + name: + type: string + type: object + type: array + initContainers: + items: + properties: + args: + items: + type: string + type: array + command: + items: + type: string + type: array + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + type: string + optional: + type: boolean + required: + - key + type: object + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + resourceFieldRef: + properties: + containerName: + type: string + divisor: + type: string resource: type: string required: @@ -4718,9 +5260,271 @@ spec: preferredDuringSchedulingIgnoredDuringExecution: items: properties: - podAffinityTerm: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - labelSelector: + name: + type: string + selector: properties: matchExpressions: items: @@ -4743,64 +5547,73 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string required: - - topologyKey + - type type: object - weight: - format: int32 - type: integer required: - - podAffinityTerm - - weight + - metric + - target type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + resource: properties: - labelSelector: + name: + type: string + target: properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string required: - - topologyKey + - name + - target type: object - type: array + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name type: object - type: object - annotations: - additionalProperties: - type: string + required: + - maxReplicas + - scaleTargetRef type: object imagePullSecrets: items: @@ -7098,21 +7911,292 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - topologyKey + - name + - target type: object - type: array + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name type: object - type: object - annotations: - additionalProperties: - type: string + required: + - maxReplicas + - scaleTargetRef type: object imagePullSecrets: items: @@ -9308,48 +10392,319 @@ spec: - podAffinityTerm - weight type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: properties: - labelSelector: + name: + type: string + target: properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string required: - - topologyKey + - name + - target type: object - type: array + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name type: object - type: object - annotations: - additionalProperties: - type: string + required: + - maxReplicas + - scaleTargetRef type: object imagePullSecrets: items: @@ -11568,6 +12923,277 @@ spec: additionalProperties: type: string type: object + hpAutoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + type: integer + metrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + type: integer + scaleTargetRef: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + required: + - maxReplicas + - scaleTargetRef + type: object imagePullSecrets: items: properties: From aa20dcb9495407e318b221d8606aca4a2a052410 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 17:57:07 +0800 Subject: [PATCH 04/28] fix style --- controllers/spec/common.go | 45 ------------------------------------ controllers/spec/function.go | 3 +-- controllers/spec/hpa.go | 44 +++++++++++++++++++++++++++++++++++ controllers/spec/sink.go | 3 +-- controllers/spec/source.go | 3 +-- 5 files changed, 47 insertions(+), 51 deletions(-) diff --git a/controllers/spec/common.go b/controllers/spec/common.go index 7230c8df2..15218c0f4 100644 --- a/controllers/spec/common.go +++ b/controllers/spec/common.go @@ -24,8 +24,6 @@ import ( "strings" "time" - autov2beta2 "k8s.io/api/autoscaling/v2beta2" - "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/proto" @@ -97,49 +95,6 @@ func MakeHeadlessServiceName(serviceName string) string { return fmt.Sprintf("%s-headless", serviceName) } -func MakeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, - kind string) *autov2beta2.HorizontalPodAutoscaler { - return &autov2beta2.HorizontalPodAutoscaler{ - TypeMeta: metav1.TypeMeta{ - Kind: "autoscaling/v1", - APIVersion: "HorizontalPodAutoscaler", - }, - ObjectMeta: *objectMeta, - Spec: autov2beta2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: kind, - Name: objectMeta.Name, - APIVersion: "compute.functionmesh.io/v1alpha1", - }, - MinReplicas: &minReplicas, - MaxReplicas: maxReplicas, - Metrics: defaultHPAMetrics(), - }, - } -} - -func MakeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, - kind string) *autov2beta2.HorizontalPodAutoscaler { - return &autov2beta2.HorizontalPodAutoscaler{ - TypeMeta: metav1.TypeMeta{ - Kind: "autoscaling/v1", - APIVersion: "HorizontalPodAutoscaler", - }, - ObjectMeta: *objectMeta, - Spec: autov2beta2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: kind, - Name: objectMeta.Name, - APIVersion: "compute.functionmesh.io/v1alpha1", - }, - MinReplicas: autoscalerSpec.MinReplicas, - MaxReplicas: autoscalerSpec.MaxReplicas, - Metrics: autoscalerSpec.Metrics, - Behavior: autoscalerSpec.Behavior, - }, - } -} - func MakeStatefulSet(objectMeta *metav1.ObjectMeta, replicas *int32, container *corev1.Container, volumes []corev1.Volume, labels map[string]string, policy v1alpha1.PodPolicy) *appsv1.StatefulSet { return &appsv1.StatefulSet{ diff --git a/controllers/spec/function.go b/controllers/spec/function.go index def77bc71..9225293d3 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -34,9 +34,8 @@ func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAuto objectMeta := MakeFunctionObjectMeta(function) if function.Spec.Pod.HPAutoscaler == nil { return MakeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) - } else { - return MakeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) } + return MakeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) } func MakeFunctionService(function *v1alpha1.Function) *corev1.Service { diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index e84baaea1..845cc0dab 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -20,6 +20,7 @@ package spec import ( autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. @@ -39,3 +40,46 @@ func defaultHPAMetrics() []autov2beta2.MetricSpec { }, } } + +func MakeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, + kind string) *autov2beta2.HorizontalPodAutoscaler { + return &autov2beta2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + Kind: "autoscaling/v1", + APIVersion: "HorizontalPodAutoscaler", + }, + ObjectMeta: *objectMeta, + Spec: autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ + Kind: kind, + Name: objectMeta.Name, + APIVersion: "compute.functionmesh.io/v1alpha1", + }, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: defaultHPAMetrics(), + }, + } +} + +func MakeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, + kind string) *autov2beta2.HorizontalPodAutoscaler { + return &autov2beta2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + Kind: "autoscaling/v1", + APIVersion: "HorizontalPodAutoscaler", + }, + ObjectMeta: *objectMeta, + Spec: autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ + Kind: kind, + Name: objectMeta.Name, + APIVersion: "compute.functionmesh.io/v1alpha1", + }, + MinReplicas: autoscalerSpec.MinReplicas, + MaxReplicas: autoscalerSpec.MaxReplicas, + Metrics: autoscalerSpec.Metrics, + Behavior: autoscalerSpec.Behavior, + }, + } +} diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index f150cee8a..c51c5cc8f 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -30,9 +30,8 @@ func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) if sink.Spec.Pod.HPAutoscaler == nil { return MakeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) - } else { - return MakeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) } + return MakeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) } func MakeSinkService(sink *v1alpha1.Sink) *corev1.Service { diff --git a/controllers/spec/source.go b/controllers/spec/source.go index 8e1e6ba53..d68013177 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -30,9 +30,8 @@ func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler objectMeta := MakeSourceObjectMeta(source) if source.Spec.Pod.HPAutoscaler == nil { return MakeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) - } else { - return MakeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) } + return MakeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) } func MakeSourceService(source *v1alpha1.Source) *corev1.Service { From 7c375398166432674ea236a2a25ac3d4b81a2d8d Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 19:09:25 +0800 Subject: [PATCH 05/28] fix style --- api/v1alpha1/function_types.go | 14 +++++++++----- api/v1alpha1/sink_types.go | 16 ++++++++++------ api/v1alpha1/source_types.go | 16 ++++++++++------ 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/api/v1alpha1/function_types.go b/api/v1alpha1/function_types.go index c5fb89375..a79b73dd7 100644 --- a/api/v1alpha1/function_types.go +++ b/api/v1alpha1/function_types.go @@ -29,11 +29,15 @@ import ( type FunctionSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - Name string `json:"name,omitempty"` - ClassName string `json:"className,omitempty"` - Tenant string `json:"tenant,omitempty"` - ClusterName string `json:"clusterName,omitempty"` - Replicas *int32 `json:"replicas,omitempty"` + Name string `json:"name,omitempty"` + ClassName string `json:"className,omitempty"` + Tenant string `json:"tenant,omitempty"` + ClusterName string `json:"clusterName,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + + // MaxReplicas indicates the maximum number of replicas and enables the HorizontalPodAutoscaler + // If provided, a default HPA with CPU at average of 80% will be used. + // For complex HPA strategies, please refer to Pod.HPAutoscaler. MaxReplicas *int32 `json:"maxReplicas,omitempty"` // if provided, turn on autoscaling Input InputConf `json:"input,omitempty"` Output OutputConf `json:"output,omitempty"` diff --git a/api/v1alpha1/sink_types.go b/api/v1alpha1/sink_types.go index a29a5f1b8..1a60a986e 100644 --- a/api/v1alpha1/sink_types.go +++ b/api/v1alpha1/sink_types.go @@ -29,12 +29,16 @@ import ( type SinkSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - Name string `json:"name,omitempty"` - ClassName string `json:"className,omitempty"` - ClusterName string `json:"clusterName,omitempty"` - Tenant string `json:"tenant,omitempty"` - SinkType string `json:"sinkType,omitempty"` // refer to `--sink-type` as builtin connector - Replicas *int32 `json:"replicas,omitempty"` + Name string `json:"name,omitempty"` + ClassName string `json:"className,omitempty"` + ClusterName string `json:"clusterName,omitempty"` + Tenant string `json:"tenant,omitempty"` + SinkType string `json:"sinkType,omitempty"` // refer to `--sink-type` as builtin connector + Replicas *int32 `json:"replicas,omitempty"` + + // MaxReplicas indicates the maximum number of replicas and enables the HorizontalPodAutoscaler + // If provided, a default HPA with CPU at average of 80% will be used. + // For complex HPA strategies, please refer to Pod.HPAutoscaler. MaxReplicas *int32 `json:"maxReplicas,omitempty"` // if provided, turn on autoscaling Input InputConf `json:"input,omitempty"` diff --git a/api/v1alpha1/source_types.go b/api/v1alpha1/source_types.go index daba57645..861466a40 100644 --- a/api/v1alpha1/source_types.go +++ b/api/v1alpha1/source_types.go @@ -29,12 +29,16 @@ import ( type SourceSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file - Name string `json:"name,omitempty"` - ClassName string `json:"className,omitempty"` - Tenant string `json:"tenant,omitempty"` - ClusterName string `json:"clusterName,omitempty"` - SourceType string `json:"sourceType,omitempty"` // refer to `--source-type` as builtin connector - Replicas *int32 `json:"replicas,omitempty"` + Name string `json:"name,omitempty"` + ClassName string `json:"className,omitempty"` + Tenant string `json:"tenant,omitempty"` + ClusterName string `json:"clusterName,omitempty"` + SourceType string `json:"sourceType,omitempty"` // refer to `--source-type` as builtin connector + Replicas *int32 `json:"replicas,omitempty"` + + // MaxReplicas indicates the maximum number of replicas and enables the HorizontalPodAutoscaler + // If provided, a default HPA with CPU at average of 80% will be used. + // For complex HPA strategies, please refer to Pod.HPAutoscaler. MaxReplicas *int32 `json:"maxReplicas,omitempty"` // if provided, turn on autoscaling Output OutputConf `json:"output,omitempty"` From ec3c46689a34adef297f3c99cb6f11cc0fc926da Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 19:21:42 +0800 Subject: [PATCH 06/28] fix CI --- controllers/spec/function.go | 4 ++-- controllers/spec/hpa.go | 22 +++++++++------------- controllers/spec/sink.go | 4 ++-- controllers/spec/source.go | 4 ++-- 4 files changed, 15 insertions(+), 19 deletions(-) diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 9225293d3..62d78e050 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -33,9 +33,9 @@ var log = logf.Log.WithName("sink-resource") func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeFunctionObjectMeta(function) if function.Spec.Pod.HPAutoscaler == nil { - return MakeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) + return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) } - return MakeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) + return makeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) } func MakeFunctionService(function *v1alpha1.Function) *corev1.Service { diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index 845cc0dab..0b66b8333 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -41,7 +41,7 @@ func defaultHPAMetrics() []autov2beta2.MetricSpec { } } -func MakeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, +func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, kind string) *autov2beta2.HorizontalPodAutoscaler { return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ @@ -62,24 +62,20 @@ func MakeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int3 } } -func MakeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, +func makeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, kind string) *autov2beta2.HorizontalPodAutoscaler { + spec := *autoscalerSpec + spec.ScaleTargetRef = autov2beta2.CrossVersionObjectReference{ + Kind: kind, + Name: objectMeta.Name, + APIVersion: "compute.functionmesh.io/v1alpha1", + } return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ Kind: "autoscaling/v1", APIVersion: "HorizontalPodAutoscaler", }, ObjectMeta: *objectMeta, - Spec: autov2beta2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: kind, - Name: objectMeta.Name, - APIVersion: "compute.functionmesh.io/v1alpha1", - }, - MinReplicas: autoscalerSpec.MinReplicas, - MaxReplicas: autoscalerSpec.MaxReplicas, - Metrics: autoscalerSpec.Metrics, - Behavior: autoscalerSpec.Behavior, - }, + Spec: spec, } } diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index c51c5cc8f..15f483f61 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -29,9 +29,9 @@ import ( func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) if sink.Spec.Pod.HPAutoscaler == nil { - return MakeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) + return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) } - return MakeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) + return makeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) } func MakeSinkService(sink *v1alpha1.Sink) *corev1.Service { diff --git a/controllers/spec/source.go b/controllers/spec/source.go index d68013177..820176fae 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -29,9 +29,9 @@ import ( func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSourceObjectMeta(source) if source.Spec.Pod.HPAutoscaler == nil { - return MakeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) + return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) } - return MakeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) + return makeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) } func MakeSourceService(source *v1alpha1.Source) *corev1.Service { From 3933d9fdbf91860cc2ce9f80bc2fc0224c8475a7 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 21:14:52 +0800 Subject: [PATCH 07/28] add tests --- .../compute_v1alpha1_function_hpa.yaml | 79 + .ci/helm.sh | 5 + .ci/verify_function_mesh.sh | 8 + .../test-integration-kind-samples.yml | 14 + api/v1alpha1/common.go | 16 +- api/v1alpha1/zz_generated.deepcopy.go | 13 +- ...ompute.functionmesh.io_functionmeshes.yaml | 1544 +++++---- .../compute.functionmesh.io_functions.yaml | 454 ++- .../bases/compute.functionmesh.io_sinks.yaml | 454 ++- .../compute.functionmesh.io_sources.yaml | 454 ++- controllers/function.go | 5 +- controllers/function_controller_test.go | 47 + controllers/sink.go | 4 +- controllers/source.go | 4 +- controllers/spec/function.go | 6 +- controllers/spec/hpa.go | 38 +- controllers/spec/sink.go | 6 +- controllers/spec/source.go | 6 +- hack/kind-cluster-build.sh | 6 + manifests/crd.yaml | 2860 ++++++++--------- 20 files changed, 2958 insertions(+), 3065 deletions(-) create mode 100644 .ci/clusters/compute_v1alpha1_function_hpa.yaml diff --git a/.ci/clusters/compute_v1alpha1_function_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_hpa.yaml new file mode 100644 index 000000000..e32a12973 --- /dev/null +++ b/.ci/clusters/compute_v1alpha1_function_hpa.yaml @@ -0,0 +1,79 @@ +apiVersion: compute.functionmesh.io/v1alpha1 +kind: Function +metadata: + name: function-hpa-sample + namespace: default +spec: + image: streamnative/pulsar-functions-java-sample:2.8.0.7 + className: org.apache.pulsar.functions.api.examples.ExclamationFunction + forwardSourceMessageProperty: true + MaxPendingAsyncRequests: 1000 + replicas: 1 + maxReplicas: 5 + logTopic: persistent://public/default/logging-function-logs + input: + topics: + - persistent://public/default/input-java-topic + typeClassName: java.lang.String + output: + topic: persistent://public/default/output-java-topic + typeClassName: java.lang.String + resources: + requests: + cpu: "0.1" + memory: 1G + limits: + cpu: "0.2" + memory: 1.1G + # each secret will be loaded ad an env variable from the `path` secret with the `key` in that secret in the name of `name` + secretsMap: + "name": + path: "test-secret" + key: "username" + "pwd": + path: "test-secret" + key: "password" + pulsar: + pulsarConfig: "test-pulsar" + #authConfig: "test-auth" + java: + jar: /pulsar/examples/api-examples.jar + # to be delete & use admission hook + clusterName: test + autoAck: true + autoScalingMetrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-pulsar +data: + webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080 + brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650 +#--- +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: test-auth +#data: +# clientAuthenticationPlugin: "abc" +# clientAuthenticationParameters: "xyz" +# tlsTrustCertsFilePath: "uvw" +# useTls: "true" +# tlsAllowInsecureConnection: "false" +# tlsHostnameVerificationEnable: "true" +--- +apiVersion: v1 +data: + username: YWRtaW4= + password: MWYyZDFlMmU2N2Rm +kind: Secret +metadata: + name: test-secret +type: Opaque diff --git a/.ci/helm.sh b/.ci/helm.sh index 7e69ac18f..9c500269e 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -126,6 +126,11 @@ function ci::verify_function_mesh() { ${KUBECTL} describe pod -lname=${FUNCTION_NAME} } +function ci::verify_hpa() { + FUNCTION_NAME=$1 + ${KUBECTL} get hpa.v2beta2.autoscaling +} + function ci::test_function_runners() { ${KUBECTL} exec -n ${NAMESPACE} ${CLUSTER}-pulsar-broker-0 -- bin/pulsar-admin functions create --tenant public --namespace default --name test-java --className org.apache.pulsar.functions.api.examples.ExclamationFunction --inputs persistent://public/default/test-java-input --jar /pulsar/examples/api-examples.jar --cpu 0.1 sleep 15 diff --git a/.ci/verify_function_mesh.sh b/.ci/verify_function_mesh.sh index 8e235a633..af57cd8fa 100755 --- a/.ci/verify_function_mesh.sh +++ b/.ci/verify_function_mesh.sh @@ -57,4 +57,12 @@ case ${1} in ci::print_function_log functionmesh-sample-python-function ci::verify_mesh_function ;; + compute_v1alpha1_function_hpa) + ci::verify_function_mesh function-hpa-sample + ci::verify_hpa function-hpa-sample + sleep 60 + ci::print_function_log function-hpa-sample + ci::verify_java_function function-hpa-sample + ci::verify_hpa function-hpa-sample + ;; esac \ No newline at end of file diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml index 2505746a2..29568511d 100644 --- a/.github/workflows/test-integration-kind-samples.yml +++ b/.github/workflows/test-integration-kind-samples.yml @@ -63,6 +63,7 @@ jobs: - name: Verify Function kind - Java Function run: | .ci/verify_function_mesh.sh compute_v1alpha1_function + kubectl delete -f .ci/clusters/compute_v1alpha1_function.yaml - name: Test Function kind - Python Function run: | @@ -72,6 +73,7 @@ jobs: - name: Verify Function kind - Python Function run: | .ci/verify_function_mesh.sh compute_v1alpha1_py_function + kubectl delete -f .ci/clusters/compute_v1alpha1_py_function.yaml - name: Test Function kind - Go Function run: | @@ -81,6 +83,7 @@ jobs: - name: Verify Function kind - Go Function run: | .ci/verify_function_mesh.sh compute_v1alpha1_go_function + kubectl delete -f .ci/clusters/compute_v1alpha1_go_function.yaml - name: Test Mesh kind run: | @@ -90,3 +93,14 @@ jobs: - name: Verify Mesh kind run: | .ci/verify_function_mesh.sh compute_v1alpha1_functionmesh + kubectl delete -f .ci/clusters/compute_v1alpha1_functionmesh.yaml + + - name: Test Function HPA + run: | + kubectl apply -f .ci/clusters/compute_v1alpha1_function_hpa.yaml + kubectl get all + + - name: Verify Function HPA + run: | + .ci/verify_function_mesh.sh compute_v1alpha1_function_hpa + kubectl delete -f .ci/clusters/compute_v1alpha1_function_hpa.yaml diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index f582940bf..79fce139c 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -95,12 +95,18 @@ type PodPolicy struct { // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` - // HPAutoscaler allows user to configure the HPA with k8s.io/api/autoscaling/v2beta2 - // This config will override the maxReplicas in Function/Source/Sink Spec - // If user set HPAutoscaler, then controller will use HPAutoscaler to create HPA - // More info: https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/ + // AutoScalingMetrics contains the specifications for which to use to calculate the + // desired replica count (the maximum replica count across all metrics will + // be used). + // More info: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#metricspec-v2beta2-autoscaling // +optional - HPAutoscaler *autov2beta2.HorizontalPodAutoscalerSpec `json:"hpAutoscaler,omitempty"` + AutoScalingMetrics []autov2beta2.MetricSpec `json:"autoScalingMetrics,omitempty"` + + // AutoScalingBehavior configures the scaling behavior of the target + // in both Up and Down directions (scaleUp and scaleDown fields respectively). + // If not set, the default HPAScalingRules for scale up and scale down are used. + // +optional + AutoScalingBehavior *autov2beta2.HorizontalPodAutoscalerBehavior `json:"autoScalingBehavior,omitempty"` } type Runtime struct { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b68c1509d..6687167ac 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -583,9 +583,16 @@ func (in *PodPolicy) DeepCopyInto(out *PodPolicy) { (*in)[i].DeepCopyInto(&(*out)[i]) } } - if in.HPAutoscaler != nil { - in, out := &in.HPAutoscaler, &out.HPAutoscaler - *out = new(v2beta2.HorizontalPodAutoscalerSpec) + if in.AutoScalingMetrics != nil { + in, out := &in.AutoScalingMetrics, &out.AutoScalingMetrics + *out = make([]v2beta2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.AutoScalingBehavior != nil { + in, out := &in.AutoScalingBehavior, &out.AutoScalingBehavior + *out = new(v2beta2.HorizontalPodAutoscalerBehavior) (*in).DeepCopyInto(*out) } } diff --git a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml index 08d0e71ea..fc4655eab 100644 --- a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml +++ b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml @@ -478,277 +478,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -2922,277 +2898,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - describedObject - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - pods: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - resource: + target: properties: - name: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object required: - - name - - target + - type type: object - type: + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -5246,68 +5198,282 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + properties: + preferredDuringSchedulingIgnoredDuringExecution: + items: + properties: + podAffinityTerm: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object required: - - topologyKey + - name type: object - type: array - type: object - podAntiAffinity: - properties: - preferredDuringSchedulingIgnoredDuringExecution: - items: + target: properties: - podAffinityTerm: - properties: - labelSelector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - weight: + averageUtilization: format: int32 type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - podAffinityTerm - - weight + - type type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - labelSelector: + name: + type: string + selector: properties: matchExpressions: items: @@ -5330,293 +5496,55 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string - required: - - topologyKey - type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - hpAutoscaler: - properties: - behavior: - properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - properties: - external: - properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object - required: - - metric - - target - type: object - object: - properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object required: - - describedObject - - metric - - target + - name type: object - pods: + target: properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - metric - - target + - type type: object - resource: + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: properties: - name: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object required: - - name - - target + - type type: object - type: - type: string required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_functions.yaml b/config/crd/bases/compute.functionmesh.io_functions.yaml index 28ed3134f..947862775 100644 --- a/config/crd/bases/compute.functionmesh.io_functions.yaml +++ b/config/crd/bases/compute.functionmesh.io_functions.yaml @@ -479,277 +479,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sinks.yaml b/config/crd/bases/compute.functionmesh.io_sinks.yaml index c49dc8b8f..83f3a3cd7 100644 --- a/config/crd/bases/compute.functionmesh.io_sinks.yaml +++ b/config/crd/bases/compute.functionmesh.io_sinks.yaml @@ -413,277 +413,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - resource: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sources.yaml b/config/crd/bases/compute.functionmesh.io_sources.yaml index 8dc3aa1e3..54093cba8 100644 --- a/config/crd/bases/compute.functionmesh.io_sources.yaml +++ b/config/crd/bases/compute.functionmesh.io_sources.yaml @@ -390,277 +390,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/controllers/function.go b/controllers/function.go index 6157baa68..977100c71 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -167,7 +167,7 @@ func (r *FunctionReconciler) ApplyFunctionService(ctx context.Context, req ctrl. func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Request, function *v1alpha1.Function) error { - if function.Spec.Pod.HPAutoscaler == nil && function.Spec.MaxReplicas == nil { + if function.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -205,7 +205,7 @@ func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Re func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Request, function *v1alpha1.Function) error { - if function.Spec.Pod.HPAutoscaler == nil && function.Spec.MaxReplicas == nil { + if function.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -219,6 +219,7 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ switch condition.Action { case v1alpha1.Create: hpa := spec.MakeFunctionHPA(function) + r.Log.Info("hpa", "hpa", *hpa) if err := r.Create(ctx, hpa); err != nil { r.Log.Error(err, "failed to create pod autoscaler for function", "name", function.Name) return err diff --git a/controllers/function_controller_test.go b/controllers/function_controller_test.go index 3c25ea1b4..59d0c7f05 100644 --- a/controllers/function_controller_test.go +++ b/controllers/function_controller_test.go @@ -20,6 +20,8 @@ package controllers import ( "context" + autov2beta2 "k8s.io/api/autoscaling/v2beta2" + . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/streamnative/function-mesh/api/v1alpha1" @@ -82,6 +84,33 @@ var _ = Describe("Function Controller (Batcher)", func() { }) }) +var _ = Describe("Function Controller (HPA)", func() { + Context("Simple Function Item with HPA", func() { + configs := makeSamplePulsarConfig() + function := makeFunctionSample(TestFunctionName) + cpuPercentage := int32(80) + function.Spec.Pod.AutoScalingMetrics = []autov2beta2.MetricSpec{ + { + Type: autov2beta2.ResourceMetricSourceType, + Resource: &autov2beta2.ResourceMetricSource{ + Name: v1.ResourceCPU, + Target: autov2beta2.MetricTarget{ + Type: autov2beta2.UtilizationMetricType, + AverageUtilization: &cpuPercentage, + }, + }, + }, + } + + createFunctionConfigMap(configs) + createFunction(function) + createFunctionHPA(function) + deleteFunctionHPA(function) + deleteFunction(function) + deleteFunctionConfigMap(configs) + }) +}) + func createFunction(function *v1alpha1.Function) { if function.Status.Conditions == nil { function.Status.Conditions = make(map[v1alpha1.Component]v1alpha1.ResourceCondition) @@ -93,6 +122,17 @@ func createFunction(function *v1alpha1.Function) { }) } +func createFunctionHPA(function *v1alpha1.Function) { + if function.Status.Conditions == nil { + function.Status.Conditions = make(map[v1alpha1.Component]v1alpha1.ResourceCondition) + } + + It("HPA should be created", func() { + hpa := spec.MakeFunctionHPA(function) + Expect(k8sClient.Create(context.Background(), hpa)).Should(Succeed()) + }) +} + func createFunctionConfigMap(configs *v1.ConfigMap) { It("Should create pulsar configmap successfully", func() { Expect(k8sClient.Create(context.Background(), configs)).Should(Succeed()) @@ -123,3 +163,10 @@ func deleteFunctionSecret(secret *v1.Secret) { Expect(k8sClient.Delete(context.Background(), secret)).Should(Succeed()) }) } + +func deleteFunctionHPA(function *v1alpha1.Function) { + hpa := spec.MakeFunctionHPA(function) + It("Should delete HPA successfully", func() { + Expect(k8sClient.Delete(context.Background(), hpa)).Should(Succeed()) + }) +} diff --git a/controllers/sink.go b/controllers/sink.go index 5c7839a76..2242db35a 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -155,7 +155,7 @@ func (r *SinkReconciler) ApplySinkService(ctx context.Context, req ctrl.Request, } func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, sink *v1alpha1.Sink) error { - if sink.Spec.Pod.HPAutoscaler == nil && sink.Spec.MaxReplicas == nil { + if sink.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -187,7 +187,7 @@ func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, s } func (r *SinkReconciler) ApplySinkHPA(ctx context.Context, req ctrl.Request, sink *v1alpha1.Sink) error { - if sink.Spec.Pod.HPAutoscaler == nil && sink.Spec.MaxReplicas == nil { + if sink.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } diff --git a/controllers/source.go b/controllers/source.go index 21eba8aae..8bb06ba16 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -157,7 +157,7 @@ func (r *SourceReconciler) ApplySourceService(ctx context.Context, req ctrl.Requ } func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Request, source *v1alpha1.Source) error { - if source.Spec.Pod.HPAutoscaler == nil && source.Spec.MaxReplicas == nil { + if source.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } @@ -189,7 +189,7 @@ func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Reques } func (r *SourceReconciler) ApplySourceHPA(ctx context.Context, req ctrl.Request, source *v1alpha1.Source) error { - if source.Spec.Pod.HPAutoscaler == nil && source.Spec.MaxReplicas == nil { + if source.Spec.MaxReplicas == nil { // HPA not enabled, skip further action return nil } diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 62d78e050..0eb2f8817 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -32,10 +32,10 @@ var log = logf.Log.WithName("sink-resource") func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeFunctionObjectMeta(function) - if function.Spec.Pod.HPAutoscaler == nil { - return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Kind) + if !isDefaultHPAEnabled(function.Spec.Replicas, function.Spec.MaxReplicas, function.Spec.Pod) { + return makeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Spec.Pod) } - return makeHPA(objectMeta, function.Spec.Pod.HPAutoscaler, function.Kind) + return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas) } func MakeFunctionService(function *v1alpha1.Function) *corev1.Service { diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index 0b66b8333..43def0e24 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -18,11 +18,16 @@ package spec import ( + "github.com/streamnative/function-mesh/api/v1alpha1" autov2beta2 "k8s.io/api/autoscaling/v2beta2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func isDefaultHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.PodPolicy) bool { + return minReplicas != nil && maxReplicas != nil && podPolicy.AutoScalingBehavior == nil && len(podPolicy.AutoScalingMetrics) == 0 && *maxReplicas > *minReplicas +} + // defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. func defaultHPAMetrics() []autov2beta2.MetricSpec { // TODO: configurable cpu percentage @@ -41,19 +46,18 @@ func defaultHPAMetrics() []autov2beta2.MetricSpec { } } -func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, - kind string) *autov2beta2.HorizontalPodAutoscaler { +func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32) *autov2beta2.HorizontalPodAutoscaler { return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ - Kind: "autoscaling/v1", - APIVersion: "HorizontalPodAutoscaler", + APIVersion: "autoscaling/v2beta2", + Kind: "HorizontalPodAutoscaler", }, ObjectMeta: *objectMeta, Spec: autov2beta2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: kind, + Kind: "StatefulSet", Name: objectMeta.Name, - APIVersion: "compute.functionmesh.io/v1alpha1", + APIVersion: "apps/v1", }, MinReplicas: &minReplicas, MaxReplicas: maxReplicas, @@ -62,18 +66,22 @@ func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int3 } } -func makeHPA(objectMeta *metav1.ObjectMeta, autoscalerSpec *autov2beta2.HorizontalPodAutoscalerSpec, - kind string) *autov2beta2.HorizontalPodAutoscaler { - spec := *autoscalerSpec - spec.ScaleTargetRef = autov2beta2.CrossVersionObjectReference{ - Kind: kind, - Name: objectMeta.Name, - APIVersion: "compute.functionmesh.io/v1alpha1", +func makeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, podPolicy v1alpha1.PodPolicy) *autov2beta2.HorizontalPodAutoscaler { + spec := autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ + Kind: "StatefulSet", + Name: objectMeta.Name, + APIVersion: "apps/v1", + }, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: podPolicy.AutoScalingMetrics, + Behavior: podPolicy.AutoScalingBehavior, } return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ - Kind: "autoscaling/v1", - APIVersion: "HorizontalPodAutoscaler", + APIVersion: "autoscaling/v2beta2", + Kind: "HorizontalPodAutoscaler", }, ObjectMeta: *objectMeta, Spec: spec, diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index 15f483f61..1f27205fc 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -28,10 +28,10 @@ import ( func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) - if sink.Spec.Pod.HPAutoscaler == nil { - return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Kind) + if !isDefaultHPAEnabled(sink.Spec.Replicas, sink.Spec.MaxReplicas, sink.Spec.Pod) { + return makeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Spec.Pod) } - return makeHPA(objectMeta, sink.Spec.Pod.HPAutoscaler, sink.Kind) + return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas) } func MakeSinkService(sink *v1alpha1.Sink) *corev1.Service { diff --git a/controllers/spec/source.go b/controllers/spec/source.go index 820176fae..1a728f9a2 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -28,10 +28,10 @@ import ( func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSourceObjectMeta(source) - if source.Spec.Pod.HPAutoscaler == nil { - return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Kind) + if !isDefaultHPAEnabled(source.Spec.Replicas, source.Spec.MaxReplicas, source.Spec.Pod) { + return makeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Spec.Pod) } - return makeHPA(objectMeta, source.Spec.Pod.HPAutoscaler, source.Kind) + return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas) } func MakeSourceService(source *v1alpha1.Source) *corev1.Service { diff --git a/hack/kind-cluster-build.sh b/hack/kind-cluster-build.sh index 15a9bbcd4..8e00dbefa 100755 --- a/hack/kind-cluster-build.sh +++ b/hack/kind-cluster-build.sh @@ -118,6 +118,8 @@ configFile=${workDir}/kind-config.yaml cat < ${configFile} kind: Cluster apiVersion: kind.x-k8s.io/v1alpha4 +runtimeConfig: + "autoscaling/v2beta2": "true" nodes: - role: control-plane extraPortMappings: @@ -238,6 +240,10 @@ docker pull gcr.io/google-containers/kube-scheduler:${k8sVersion} docker tag gcr.io/google-containers/kube-scheduler:${k8sVersion} mirantis/hypokube:final kind load docker-image --name=${clusterName} mirantis/hypokube:final +echo "install metrics-server" +kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml +kubectl patch deployment metrics-server -n kube-system -p '{"spec":{"template":{"spec":{"containers":[{"name":"metrics-server","args":["--cert-dir=/tmp", "--secure-port=4443", "--kubelet-insecure-tls","--kubelet-preferred-address-types=InternalIP"]}]}}}}' + echo "############# success create cluster:[${clusterName}] #############" echo "To start using your cluster, run:" diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 59ada4b0f..90f080177 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -476,277 +476,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - describedObject - - metric - - target + - type type: object - pods: + required: + - metric + - target + type: object + object: + properties: + describedObject: properties: - metric: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - resource: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -2920,277 +2896,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - object: + required: + - metric + - target + type: object + object: + properties: + describedObject: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -5298,14 +5250,177 @@ spec: format: int32 type: integer required: - - podAffinityTerm - - weight + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + required: + - metric + - target + type: object + object: + properties: + describedObject: properties: - labelSelector: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: properties: matchExpressions: items: @@ -5328,293 +5443,106 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string required: - - topologyKey + - name type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - hpAutoscaler: - properties: - behavior: - properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - properties: - external: + target: properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - metric - - target + - type type: object - object: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object - required: - - describedObject - - metric - - target - type: object - pods: - properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + operator: type: string - type: object + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - resource: + target: properties: - name: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object required: - - name - - target + - type type: object - type: - type: string required: - - type + - metric + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -7927,277 +7855,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - object: + required: + - metric + - target + type: object + object: + properties: + describedObject: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + apiVersion: + type: string + kind: + type: string + name: + type: string required: - - describedObject - - metric - - target + - kind + - name type: object - pods: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -10435,277 +10339,253 @@ spec: additionalProperties: type: string type: object - hpAutoscaler: + autoScalingBehavior: properties: - behavior: + scaleDown: properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: properties: - external: + metric: properties: - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - metric - - target + - name type: object - object: + target: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - describedObject - - metric - - target + - name type: object - pods: + target: properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - metric - - target + - type type: object - resource: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: name: type: string - target: + selector: properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object required: - name - - target type: object - type: + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -12877,14 +12757,177 @@ spec: format: int32 type: integer required: - - podAffinityTerm - - weight + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + metric: properties: - labelSelector: + name: + type: string + selector: properties: matchExpressions: items: @@ -12907,293 +12950,106 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: - type: string required: - - topologyKey + - name type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object - hpAutoscaler: - properties: - behavior: - properties: - scaleDown: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - scaleUp: - properties: - policies: - items: - properties: - periodSeconds: - format: int32 - type: integer - type: - type: string - value: - format: int32 - type: integer - required: - - periodSeconds - - type - - value - type: object - type: array - selectPolicy: - type: string - stabilizationWindowSeconds: - format: int32 - type: integer - type: object - type: object - maxReplicas: - format: int32 - type: integer - metrics: - items: - properties: - external: + target: properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - metric - - target + - type type: object - object: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - describedObject: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - metric: + name: + type: string + selector: properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: + matchExpressions: + items: + properties: + key: type: string - type: object + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type type: object required: - - describedObject - - metric - - target + - name type: object - pods: + target: properties: - metric: - properties: - name: - type: string - selector: - properties: - matchExpressions: - items: - properties: - key: - type: string - operator: - type: string - values: - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - type: object - type: object - required: - - name - type: object - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - metric - - target + - type type: object - resource: + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: properties: - name: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string - target: - properties: - averageUtilization: - format: int32 - type: integer - averageValue: - type: string - type: - type: string - value: - type: string - required: - - type - type: object required: - - name - - target + - type type: object - type: - type: string required: - - type + - name + - target type: object - type: array - minReplicas: - format: int32 - type: integer - scaleTargetRef: - properties: - apiVersion: - type: string - kind: - type: string - name: - type: string - required: - - kind - - name - type: object - required: - - maxReplicas - - scaleTargetRef - type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: From 277f8d5971003843c71f2746d04f5351d66e9523 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 21:31:15 +0800 Subject: [PATCH 08/28] test 1.17.17 --- hack/kind-cluster-build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/kind-cluster-build.sh b/hack/kind-cluster-build.sh index 8e00dbefa..92bdc3481 100755 --- a/hack/kind-cluster-build.sh +++ b/hack/kind-cluster-build.sh @@ -82,7 +82,7 @@ done clusterName=${clusterName:-pulsar-dev} nodeNum=${nodeNum:-6} -k8sVersion=${k8sVersion:-v1.15.12} +k8sVersion=${k8sVersion:-v1.17.17} volumeNum=${volumeNum:-9} echo "clusterName: ${clusterName}" From b8e0243dd21a3ef29b1cd760f9286871bdb0dc16 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 22:08:40 +0800 Subject: [PATCH 09/28] fix CI --- .ci/deploy_pulsar_cluster.sh | 3 +++ .ci/helm.sh | 7 +++++++ hack/kind-cluster-build.sh | 6 +----- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.ci/deploy_pulsar_cluster.sh b/.ci/deploy_pulsar_cluster.sh index 0825dac47..6c4791602 100755 --- a/.ci/deploy_pulsar_cluster.sh +++ b/.ci/deploy_pulsar_cluster.sh @@ -40,6 +40,9 @@ fi # install storage provisioner ci::install_storage_provisioner +# install metrics server +ci::install_metrics_server + # install pulsar chart ci::install_pulsar_charts "$VALUES_FILE" diff --git a/.ci/helm.sh b/.ci/helm.sh index 9c500269e..02c9fc121 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -64,6 +64,13 @@ function ci::install_storage_provisioner() { echo "Successfully installed the local storage provisioner." } +function ci::install_metrics_server() { + echo "install metrics-server" + ${KUBECTL} apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml + ${KUBECTL} patch deployment metrics-server -n kube-system -p '{"spec":{"template":{"spec":{"containers":[{"name":"metrics-server","args":["--cert-dir=/tmp", "--secure-port=4443", "--kubelet-insecure-tls","--kubelet-preferred-address-types=InternalIP"]}]}}}}' + echo "Successfully installed the metrics-server." +} + function ci::install_pulsar_charts() { echo "Installing the pulsar charts ..." values=${1:-".ci/clusters/values.yaml"} diff --git a/hack/kind-cluster-build.sh b/hack/kind-cluster-build.sh index 92bdc3481..c67c1912a 100755 --- a/hack/kind-cluster-build.sh +++ b/hack/kind-cluster-build.sh @@ -82,7 +82,7 @@ done clusterName=${clusterName:-pulsar-dev} nodeNum=${nodeNum:-6} -k8sVersion=${k8sVersion:-v1.17.17} +k8sVersion=${k8sVersion:-v1.15.12} volumeNum=${volumeNum:-9} echo "clusterName: ${clusterName}" @@ -240,10 +240,6 @@ docker pull gcr.io/google-containers/kube-scheduler:${k8sVersion} docker tag gcr.io/google-containers/kube-scheduler:${k8sVersion} mirantis/hypokube:final kind load docker-image --name=${clusterName} mirantis/hypokube:final -echo "install metrics-server" -kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml -kubectl patch deployment metrics-server -n kube-system -p '{"spec":{"template":{"spec":{"containers":[{"name":"metrics-server","args":["--cert-dir=/tmp", "--secure-port=4443", "--kubelet-insecure-tls","--kubelet-preferred-address-types=InternalIP"]}]}}}}' - echo "############# success create cluster:[${clusterName}] #############" echo "To start using your cluster, run:" From 130863cc58df0d73b70a1593c19ae535437185f9 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Tue, 3 Aug 2021 22:17:46 +0800 Subject: [PATCH 10/28] wait metrics server --- .ci/helm.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.ci/helm.sh b/.ci/helm.sh index 02c9fc121..f2e97e7ef 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -69,6 +69,13 @@ function ci::install_metrics_server() { ${KUBECTL} apply -f https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.3.7/components.yaml ${KUBECTL} patch deployment metrics-server -n kube-system -p '{"spec":{"template":{"spec":{"containers":[{"name":"metrics-server","args":["--cert-dir=/tmp", "--secure-port=4443", "--kubelet-insecure-tls","--kubelet-preferred-address-types=InternalIP"]}]}}}}' echo "Successfully installed the metrics-server." + WC=$(${KUBECTL} get pods -n kube-system --field-selector=status.phase=Running | grep metrics-server | wc -l) + while [[ ${WC} -lt 1 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get pods -n kube-system + WC=$(${KUBECTL} get pods -n kube-system --field-selector=status.phase=Running | grep metrics-server | wc -l) + done } function ci::install_pulsar_charts() { From b85a49dd5092eea5f42f3e4d033507f441c45f02 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 08:39:33 +0800 Subject: [PATCH 11/28] add verify hpa --- .ci/helm.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.ci/helm.sh b/.ci/helm.sh index f2e97e7ef..af27199a0 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -143,6 +143,16 @@ function ci::verify_function_mesh() { function ci::verify_hpa() { FUNCTION_NAME=$1 ${KUBECTL} get hpa.v2beta2.autoscaling + ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml + ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o -o jsonpath='{.status.conditions}' | grep False | wc -l) + while [[ ${WC} -lt 0 ]]; do + echo ${WC}; + sleep 20 + ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml + ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o -o jsonpath='{.status.conditions}' | grep False | wc -l) + done } function ci::test_function_runners() { From f34ce0f70b1381c7e8383c61b905ab84a56153a6 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 09:28:25 +0800 Subject: [PATCH 12/28] add hpa verify --- .ci/helm.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index af27199a0..3134e6b69 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -145,13 +145,14 @@ function ci::verify_hpa() { ${KUBECTL} get hpa.v2beta2.autoscaling ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function - WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o -o jsonpath='{.status.conditions}' | grep False | wc -l) + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' | grep False | wc -l) while [[ ${WC} -lt 0 ]]; do echo ${WC}; sleep 20 ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function - WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o -o jsonpath='{.status.conditions}' | grep False | wc -l) + ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' | grep False | wc -l) done } From 87e81bd930b515ca23260e28f8610475f8a03c9a Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 09:38:23 +0800 Subject: [PATCH 13/28] update crds --- ...ompute.functionmesh.io_functionmeshes.yaml | 793 +++++++++++++++++- .../compute.functionmesh.io_functions.yaml | 247 ++++++ .../crds/compute.functionmesh.io_sinks.yaml | 249 +++++- .../crds/compute.functionmesh.io_sources.yaml | 249 +++++- 4 files changed, 1506 insertions(+), 32 deletions(-) diff --git a/charts/function-mesh-operator/crds/compute.functionmesh.io_functionmeshes.yaml b/charts/function-mesh-operator/crds/compute.functionmesh.io_functionmeshes.yaml index 5f561a5df..18cbd7ebd 100644 --- a/charts/function-mesh-operator/crds/compute.functionmesh.io_functionmeshes.yaml +++ b/charts/function-mesh-operator/crds/compute.functionmesh.io_functionmeshes.yaml @@ -478,6 +478,253 @@ spec: additionalProperties: type: string type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -2372,8 +2619,6 @@ spec: jarLocation: type: string type: object - logTopic: - type: string maxMessageRetry: format: int32 type: integer @@ -2598,23 +2843,237 @@ spec: items: type: string type: array - topologyKey: - type: string - required: - - topologyKey + topologyKey: + type: string + required: + - topologyKey + type: object + weight: + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + items: + properties: + labelSelector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + namespaces: + items: + type: string + type: array + topologyKey: + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + annotations: + additionalProperties: + type: string + type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object type: object - weight: + required: + - name + type: object + target: + properties: + averageUtilization: format: int32 type: integer + averageValue: + type: string + type: + type: string + value: + type: string required: - - podAffinityTerm - - weight + - type type: object - type: array - requiredDuringSchedulingIgnoredDuringExecution: - items: + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: properties: - labelSelector: + name: + type: string + selector: properties: matchExpressions: items: @@ -2637,22 +3096,55 @@ spec: type: string type: object type: object - namespaces: - items: - type: string - type: array - topologyKey: + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: type: string required: - - topologyKey + - type type: object - type: array - type: object - type: object - annotations: - additionalProperties: - type: string - type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: @@ -4472,8 +4964,6 @@ spec: jarLocation: type: string type: object - logTopic: - type: string maxReplicas: format: int32 type: integer @@ -4806,6 +5296,253 @@ spec: additionalProperties: type: string type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/charts/function-mesh-operator/crds/compute.functionmesh.io_functions.yaml b/charts/function-mesh-operator/crds/compute.functionmesh.io_functions.yaml index bf21dd5c9..947862775 100644 --- a/charts/function-mesh-operator/crds/compute.functionmesh.io_functions.yaml +++ b/charts/function-mesh-operator/crds/compute.functionmesh.io_functions.yaml @@ -479,6 +479,253 @@ spec: additionalProperties: type: string type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/charts/function-mesh-operator/crds/compute.functionmesh.io_sinks.yaml b/charts/function-mesh-operator/crds/compute.functionmesh.io_sinks.yaml index bf7915149..83f3a3cd7 100644 --- a/charts/function-mesh-operator/crds/compute.functionmesh.io_sinks.yaml +++ b/charts/function-mesh-operator/crds/compute.functionmesh.io_sinks.yaml @@ -134,8 +134,6 @@ spec: jarLocation: type: string type: object - logTopic: - type: string maxMessageRetry: format: int32 type: integer @@ -415,6 +413,253 @@ spec: additionalProperties: type: string type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: diff --git a/charts/function-mesh-operator/crds/compute.functionmesh.io_sources.yaml b/charts/function-mesh-operator/crds/compute.functionmesh.io_sources.yaml index 109615848..1dc29098f 100644 --- a/charts/function-mesh-operator/crds/compute.functionmesh.io_sources.yaml +++ b/charts/function-mesh-operator/crds/compute.functionmesh.io_sources.yaml @@ -56,8 +56,6 @@ spec: jarLocation: type: string type: object - logTopic: - type: string maxReplicas: format: int32 type: integer @@ -390,6 +388,253 @@ spec: additionalProperties: type: string type: object + autoScalingBehavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + autoScalingMetrics: + items: + properties: + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + type: string + type: + type: string + value: + type: string + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array imagePullSecrets: items: properties: From 4ed5bbc805f8b8a42399acf021c204d4c509fe0b Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 11:16:20 +0800 Subject: [PATCH 14/28] fix HPA not work with Kinds --- .ci/helm.sh | 6 +++--- controllers/spec/function.go | 11 ++++++++--- controllers/spec/hpa.go | 30 +++++++++++------------------- controllers/spec/sink.go | 9 +++++++-- controllers/spec/source.go | 9 +++++++-- 5 files changed, 36 insertions(+), 29 deletions(-) diff --git a/.ci/helm.sh b/.ci/helm.sh index 3134e6b69..1c5cda656 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -145,14 +145,14 @@ function ci::verify_hpa() { ${KUBECTL} get hpa.v2beta2.autoscaling ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function - WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' | grep False | wc -l) + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[?(@.type=="AbleToScale")].status}' | grep False | wc -l) while [[ ${WC} -lt 0 ]]; do echo ${WC}; sleep 20 ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function - ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' - WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[*].status}' | grep False | wc -l) + ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[?(@.type=="AbleToScale")].status}' + WC=$(${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o jsonpath='{.status.conditions[?(@.type=="AbleToScale")].status}' | grep False | wc -l) done } diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 0eb2f8817..9c60e4121 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -28,14 +28,19 @@ import ( ) // log is for logging in this package. -var log = logf.Log.WithName("sink-resource") +var log = logf.Log.WithName("function-resource") func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeFunctionObjectMeta(function) + targetRef := autov2beta2.CrossVersionObjectReference{ + Kind: function.Kind, + Name: function.Name, + APIVersion: function.APIVersion, + } if !isDefaultHPAEnabled(function.Spec.Replicas, function.Spec.MaxReplicas, function.Spec.Pod) { - return makeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Spec.Pod) + return makeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Spec.Pod, targetRef) } - return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas) + return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, targetRef) } func MakeFunctionService(function *v1alpha1.Function) *corev1.Service { diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index 43def0e24..031efad0a 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -46,7 +46,7 @@ func defaultHPAMetrics() []autov2beta2.MetricSpec { } } -func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32) *autov2beta2.HorizontalPodAutoscaler { +func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference) *autov2beta2.HorizontalPodAutoscaler { return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling/v2beta2", @@ -54,29 +54,21 @@ func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int3 }, ObjectMeta: *objectMeta, Spec: autov2beta2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: "StatefulSet", - Name: objectMeta.Name, - APIVersion: "apps/v1", - }, - MinReplicas: &minReplicas, - MaxReplicas: maxReplicas, - Metrics: defaultHPAMetrics(), + ScaleTargetRef: targetRef, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: defaultHPAMetrics(), }, } } -func makeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, podPolicy v1alpha1.PodPolicy) *autov2beta2.HorizontalPodAutoscaler { +func makeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, podPolicy v1alpha1.PodPolicy, targetRef autov2beta2.CrossVersionObjectReference) *autov2beta2.HorizontalPodAutoscaler { spec := autov2beta2.HorizontalPodAutoscalerSpec{ - ScaleTargetRef: autov2beta2.CrossVersionObjectReference{ - Kind: "StatefulSet", - Name: objectMeta.Name, - APIVersion: "apps/v1", - }, - MinReplicas: &minReplicas, - MaxReplicas: maxReplicas, - Metrics: podPolicy.AutoScalingMetrics, - Behavior: podPolicy.AutoScalingBehavior, + ScaleTargetRef: targetRef, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: podPolicy.AutoScalingMetrics, + Behavior: podPolicy.AutoScalingBehavior, } return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index 1f27205fc..8948386b9 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -28,10 +28,15 @@ import ( func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSinkObjectMeta(sink) + targetRef := autov2beta2.CrossVersionObjectReference{ + Kind: sink.Kind, + Name: sink.Name, + APIVersion: sink.APIVersion, + } if !isDefaultHPAEnabled(sink.Spec.Replicas, sink.Spec.MaxReplicas, sink.Spec.Pod) { - return makeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Spec.Pod) + return makeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Spec.Pod, targetRef) } - return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas) + return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, targetRef) } func MakeSinkService(sink *v1alpha1.Sink) *corev1.Service { diff --git a/controllers/spec/source.go b/controllers/spec/source.go index 1a728f9a2..85f2aa8af 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -28,10 +28,15 @@ import ( func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler { objectMeta := MakeSourceObjectMeta(source) + targetRef := autov2beta2.CrossVersionObjectReference{ + Kind: source.Kind, + Name: source.Name, + APIVersion: source.APIVersion, + } if !isDefaultHPAEnabled(source.Spec.Replicas, source.Spec.MaxReplicas, source.Spec.Pod) { - return makeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Spec.Pod) + return makeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Spec.Pod, targetRef) } - return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas) + return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, targetRef) } func MakeSourceService(source *v1alpha1.Source) *corev1.Service { From 1085b7e1959dbd4396d2189eec8bec676c7e1139 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 11:38:50 +0800 Subject: [PATCH 15/28] fix CI --- .github/workflows/test-helm-charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-helm-charts.yml b/.github/workflows/test-helm-charts.yml index 0e44c79e7..8ba540710 100644 --- a/.github/workflows/test-helm-charts.yml +++ b/.github/workflows/test-helm-charts.yml @@ -62,7 +62,7 @@ jobs: if: steps.list-changed.outputs.changed == 'true' - name: Create kind cluster - uses: helm/kind-action@v1.2.0 + run: hack/kind-cluster-build.sh --name chart-testing -c 3 -v 10 if: steps.list-changed.outputs.changed == 'true' with: node_image: kindest/node:v1.15.12 From f9ca51c1d93df0106c95664584b60cbd9ed05664 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 11:48:17 +0800 Subject: [PATCH 16/28] remove debug log --- controllers/function.go | 1 - 1 file changed, 1 deletion(-) diff --git a/controllers/function.go b/controllers/function.go index 977100c71..6bdb73460 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -219,7 +219,6 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ switch condition.Action { case v1alpha1.Create: hpa := spec.MakeFunctionHPA(function) - r.Log.Info("hpa", "hpa", *hpa) if err := r.Create(ctx, hpa); err != nil { r.Log.Error(err, "failed to create pod autoscaler for function", "name", function.Name) return err From b2041a4354ea969efc2183ac5e9749b7b5b49877 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 13:27:59 +0800 Subject: [PATCH 17/28] fix tests --- controllers/function_controller_test.go | 135 +++++++++--------------- controllers/suite_test.go | 1 + 2 files changed, 52 insertions(+), 84 deletions(-) diff --git a/controllers/function_controller_test.go b/controllers/function_controller_test.go index 59d0c7f05..0aa2e0799 100644 --- a/controllers/function_controller_test.go +++ b/controllers/function_controller_test.go @@ -19,6 +19,10 @@ package controllers import ( "context" + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/types" autov2beta2 "k8s.io/api/autoscaling/v2beta2" @@ -27,66 +31,39 @@ import ( "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + timeout = time.Second * 10 + interval = time.Millisecond * 250 ) var _ = Describe("Function Controller", func() { Context("Simple Function Item", func() { - configs := makeSamplePulsarConfig() function := makeFunctionSample(TestFunctionName) - createFunctionConfigMap(configs) createFunction(function) - deleteFunction(function) - deleteFunctionConfigMap(configs) }) }) var _ = Describe("Function Controller (E2E)", func() { Context("Function With E2E Crypto Item", func() { - cryptosecrets := &v1.Secret{ - TypeMeta: metav1.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "java-function-crypto-sample-crypto-secret", - Namespace: "default", - UID: "dead-beef-secret", - }, - Data: map[string][]byte{ - "test_ecdsa_privkey.pem": []byte{0x00, 0x01, 0x02}, - "test_ecdsa_pubkey.pem": []byte{0x02, 0x01, 0x00}, - }, - Type: "Opaque", - } - configs := makeSamplePulsarConfig() function := makeFunctionSampleWithCryptoEnabled() - createFunctionConfigMap(configs) - createFunctionSecret(cryptosecrets) createFunction(function) - deleteFunction(function) - deleteFunctionSecret(cryptosecrets) - deleteFunctionConfigMap(configs) }) }) var _ = Describe("Function Controller (Batcher)", func() { Context("Function With Batcher Item", func() { - configs := makeSamplePulsarConfig() function := makeFunctionSampleWithKeyBasedBatcher() - createFunctionConfigMap(configs) createFunction(function) - deleteFunction(function) - deleteFunctionConfigMap(configs) }) }) var _ = Describe("Function Controller (HPA)", func() { Context("Simple Function Item with HPA", func() { - configs := makeSamplePulsarConfig() function := makeFunctionSample(TestFunctionName) cpuPercentage := int32(80) function.Spec.Pod.AutoScalingMetrics = []autov2beta2.MetricSpec{ @@ -102,71 +79,61 @@ var _ = Describe("Function Controller (HPA)", func() { }, } - createFunctionConfigMap(configs) createFunction(function) - createFunctionHPA(function) - deleteFunctionHPA(function) - deleteFunction(function) - deleteFunctionConfigMap(configs) }) }) func createFunction(function *v1alpha1.Function) { - if function.Status.Conditions == nil { - function.Status.Conditions = make(map[v1alpha1.Component]v1alpha1.ResourceCondition) - } - - It("StatefulSet should be created", func() { - statefulSet := spec.MakeFunctionStatefulSet(function) - Expect(k8sClient.Create(context.Background(), statefulSet)).Should(Succeed()) - }) -} - -func createFunctionHPA(function *v1alpha1.Function) { - if function.Status.Conditions == nil { - function.Status.Conditions = make(map[v1alpha1.Component]v1alpha1.ResourceCondition) - } - - It("HPA should be created", func() { - hpa := spec.MakeFunctionHPA(function) - Expect(k8sClient.Create(context.Background(), hpa)).Should(Succeed()) - }) -} -func createFunctionConfigMap(configs *v1.ConfigMap) { - It("Should create pulsar configmap successfully", func() { - Expect(k8sClient.Create(context.Background(), configs)).Should(Succeed()) + It("Function should be created", func() { + Expect(k8sClient.Create(context.Background(), function)).Should(Succeed()) }) -} - -func createFunctionSecret(secret *v1.Secret) { - It("Should create crypto secret successfully", func() { - Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed()) - }) -} -func deleteFunction(function *v1alpha1.Function) { - It("StatefulSet should be deleted", func() { - statefulSet := spec.MakeFunctionStatefulSet(function) - Expect(k8sClient.Delete(context.Background(), statefulSet)).Should(Succeed()) + It("StatefulSet should be created", func() { + statefulSet := &appsv1.StatefulSet{} + + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{ + Namespace: function.Namespace, + Name: spec.MakeFunctionObjectMeta(function).Name, + }, statefulSet) + if err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + + Expect(*statefulSet.Spec.Replicas).Should(Equal(int32(1))) }) -} -func deleteFunctionConfigMap(configs *v1.ConfigMap) { - It("Should create pulsar configmap successfully", func() { - Expect(k8sClient.Delete(context.Background(), configs)).Should(Succeed()) + It("Service should be created", func() { + srv := &v1.Service{} + svcName := spec.MakeHeadlessServiceName(spec.MakeFunctionObjectMeta(function).Name) + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: function.Namespace, + Name: svcName}, srv) + if err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) }) -} -func deleteFunctionSecret(secret *v1.Secret) { - It("Should delete crypto secret successfully", func() { - Expect(k8sClient.Delete(context.Background(), secret)).Should(Succeed()) + It("HPA should be created", func() { + if function.Spec.MaxReplicas != nil { + hpa := &autov2beta2.HorizontalPodAutoscaler{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: function.Namespace, + Name: spec.MakeFunctionObjectMeta(function).Name}, hpa) + if err != nil { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + } }) -} -func deleteFunctionHPA(function *v1alpha1.Function) { - hpa := spec.MakeFunctionHPA(function) - It("Should delete HPA successfully", func() { - Expect(k8sClient.Delete(context.Background(), hpa)).Should(Succeed()) + It("Function should be deleted", func() { + Expect(k8sClient.Delete(context.Background(), function)).Should(Succeed()) }) } diff --git a/controllers/suite_test.go b/controllers/suite_test.go index 7362fe590..b105916d3 100644 --- a/controllers/suite_test.go +++ b/controllers/suite_test.go @@ -67,6 +67,7 @@ var _ = BeforeSuite(func(done Done) { Paths: []string{filepath.Join("..", "config", "crd", "bases")}, }, AttachControlPlaneOutput: true, + ErrorIfCRDPathMissing: true, } } var err error From 7b6d0006b1acc1bfe99053ccf3fa28f284cfe0d2 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 13:37:56 +0800 Subject: [PATCH 18/28] fix style --- controllers/function_controller_test.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/controllers/function_controller_test.go b/controllers/function_controller_test.go index 0aa2e0799..1e15791a8 100644 --- a/controllers/function_controller_test.go +++ b/controllers/function_controller_test.go @@ -97,10 +97,7 @@ func createFunction(function *v1alpha1.Function) { Namespace: function.Namespace, Name: spec.MakeFunctionObjectMeta(function).Name, }, statefulSet) - if err != nil { - return false - } - return true + return err == nil }, timeout, interval).Should(BeTrue()) Expect(*statefulSet.Spec.Replicas).Should(Equal(int32(1))) @@ -112,10 +109,7 @@ func createFunction(function *v1alpha1.Function) { Eventually(func() bool { err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: function.Namespace, Name: svcName}, srv) - if err != nil { - return false - } - return true + return err == nil }, timeout, interval).Should(BeTrue()) }) @@ -125,10 +119,7 @@ func createFunction(function *v1alpha1.Function) { Eventually(func() bool { err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: function.Namespace, Name: spec.MakeFunctionObjectMeta(function).Name}, hpa) - if err != nil { - return false - } - return true + return err == nil }, timeout, interval).Should(BeTrue()) } }) From bc3e11794f3760c041613bdf2aa9888738f09a58 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 14:12:19 +0800 Subject: [PATCH 19/28] observe HPA update --- controllers/function.go | 29 ++++++++++++++++++++++++++++- controllers/sink.go | 27 +++++++++++++++++++++++++++ controllers/source.go | 27 +++++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 1 deletion(-) diff --git a/controllers/function.go b/controllers/function.go index 6bdb73460..16e1b8a0e 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -19,6 +19,7 @@ package controllers import ( "context" + "reflect" "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" @@ -197,6 +198,14 @@ func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Re return err } + if hpa.Spec.MaxReplicas != *function.Spec.MaxReplicas || + !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { + condition.Status = metav1.ConditionFalse + condition.Action = v1alpha1.Update + function.Status.Conditions[v1alpha1.HPA] = condition + return nil + } + condition.Action = v1alpha1.NoAction condition.Status = metav1.ConditionTrue function.Status.Conditions[v1alpha1.HPA] = condition @@ -223,7 +232,25 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ r.Log.Error(err, "failed to create pod autoscaler for function", "name", function.Name) return err } - case v1alpha1.Wait: + case v1alpha1.Update: + hpa := &autov2beta2.HorizontalPodAutoscaler{} + err := r.Get(ctx, types.NamespacedName{Namespace: function.Namespace, + Name: spec.MakeFunctionObjectMeta(function).Name}, hpa) + if err != nil { + r.Log.Error(err, "failed to update pod autoscaler for function, cannot find hpa", "name", function.Name) + return err + } + if hpa.Spec.MaxReplicas != *function.Spec.MaxReplicas { + hpa.Spec.MaxReplicas = *function.Spec.MaxReplicas + } + if !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { + hpa.Spec.Metrics = function.Spec.Pod.AutoScalingMetrics + } + if err := r.Update(ctx, hpa); err != nil { + r.Log.Error(err, "failed to update pod autoscaler for function", "name", function.Name) + return err + } + case v1alpha1.Wait, v1alpha1.NoAction: // do nothing } diff --git a/controllers/sink.go b/controllers/sink.go index 2242db35a..ee24738f1 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -19,6 +19,7 @@ package controllers import ( "context" + "reflect" "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" @@ -182,6 +183,14 @@ func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, s condition.Status = metav1.ConditionTrue } + if hpa.Spec.MaxReplicas != *sink.Spec.MaxReplicas || + !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { + condition.Status = metav1.ConditionFalse + condition.Action = v1alpha1.Update + sink.Status.Conditions[v1alpha1.HPA] = condition + return nil + } + sink.Status.Conditions[v1alpha1.HPA] = condition return nil } @@ -205,6 +214,24 @@ func (r *SinkReconciler) ApplySinkHPA(ctx context.Context, req ctrl.Request, sin r.Log.Error(err, "failed to create pod autoscaler for sink", "name", sink.Name) return err } + case v1alpha1.Update: + hpa := &autov2beta2.HorizontalPodAutoscaler{} + err := r.Get(ctx, types.NamespacedName{Namespace: sink.Namespace, + Name: spec.MakeSinkObjectMeta(sink).Name}, hpa) + if err != nil { + r.Log.Error(err, "failed to update pod autoscaler for sink, cannot find hpa", "name", sink.Name) + return err + } + if hpa.Spec.MaxReplicas != *sink.Spec.MaxReplicas { + hpa.Spec.MaxReplicas = *sink.Spec.MaxReplicas + } + if !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { + hpa.Spec.Metrics = sink.Spec.Pod.AutoScalingMetrics + } + if err := r.Update(ctx, hpa); err != nil { + r.Log.Error(err, "failed to update pod autoscaler for sink", "name", sink.Name) + return err + } case v1alpha1.Wait, v1alpha1.NoAction: // do nothing } diff --git a/controllers/source.go b/controllers/source.go index 8bb06ba16..513220efe 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -19,6 +19,7 @@ package controllers import ( "context" + "reflect" "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" @@ -184,6 +185,14 @@ func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Reques condition.Status = metav1.ConditionTrue } + if hpa.Spec.MaxReplicas != *source.Spec.MaxReplicas || + !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { + condition.Status = metav1.ConditionFalse + condition.Action = v1alpha1.Update + source.Status.Conditions[v1alpha1.HPA] = condition + return nil + } + source.Status.Conditions[v1alpha1.HPA] = condition return nil } @@ -207,6 +216,24 @@ func (r *SourceReconciler) ApplySourceHPA(ctx context.Context, req ctrl.Request, r.Log.Error(err, "failed to create pod autoscaler for source", "name", source.Name) return err } + case v1alpha1.Update: + hpa := &autov2beta2.HorizontalPodAutoscaler{} + err := r.Get(ctx, types.NamespacedName{Namespace: source.Namespace, + Name: spec.MakeSourceObjectMeta(source).Name}, hpa) + if err != nil { + r.Log.Error(err, "failed to update pod autoscaler for source, cannot find hpa", "name", source.Name) + return err + } + if hpa.Spec.MaxReplicas != *source.Spec.MaxReplicas { + hpa.Spec.MaxReplicas = *source.Spec.MaxReplicas + } + if !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { + hpa.Spec.Metrics = source.Spec.Pod.AutoScalingMetrics + } + if err := r.Update(ctx, hpa); err != nil { + r.Log.Error(err, "failed to update pod autoscaler for source", "name", source.Name) + return err + } case v1alpha1.Wait, v1alpha1.NoAction: // do nothing } From ecd3f0a14e2cc21e9dd98bd972637fda9806c034 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 4 Aug 2021 14:16:29 +0800 Subject: [PATCH 20/28] observe HPA update --- controllers/function.go | 8 +++++++- controllers/sink.go | 8 +++++++- controllers/source.go | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/controllers/function.go b/controllers/function.go index 16e1b8a0e..1b33e04a1 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -199,7 +199,10 @@ func (r *FunctionReconciler) ObserveFunctionHPA(ctx context.Context, req ctrl.Re } if hpa.Spec.MaxReplicas != *function.Spec.MaxReplicas || - !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { + !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) || + (function.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior == nil) || + (function.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior != nil && + !reflect.DeepEqual(*hpa.Spec.Behavior, *function.Spec.Pod.AutoScalingBehavior)) { condition.Status = metav1.ConditionFalse condition.Action = v1alpha1.Update function.Status.Conditions[v1alpha1.HPA] = condition @@ -246,6 +249,9 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ if !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = function.Spec.Pod.AutoScalingMetrics } + if function.Spec.Pod.AutoScalingBehavior != nil { + hpa.Spec.Behavior = function.Spec.Pod.AutoScalingBehavior + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for function", "name", function.Name) return err diff --git a/controllers/sink.go b/controllers/sink.go index ee24738f1..f92738dd5 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -184,7 +184,10 @@ func (r *SinkReconciler) ObserveSinkHPA(ctx context.Context, req ctrl.Request, s } if hpa.Spec.MaxReplicas != *sink.Spec.MaxReplicas || - !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { + !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) || + (sink.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior == nil) || + (sink.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior != nil && + !reflect.DeepEqual(*hpa.Spec.Behavior, *sink.Spec.Pod.AutoScalingBehavior)) { condition.Status = metav1.ConditionFalse condition.Action = v1alpha1.Update sink.Status.Conditions[v1alpha1.HPA] = condition @@ -228,6 +231,9 @@ func (r *SinkReconciler) ApplySinkHPA(ctx context.Context, req ctrl.Request, sin if !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = sink.Spec.Pod.AutoScalingMetrics } + if sink.Spec.Pod.AutoScalingBehavior != nil { + hpa.Spec.Behavior = sink.Spec.Pod.AutoScalingBehavior + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for sink", "name", sink.Name) return err diff --git a/controllers/source.go b/controllers/source.go index 513220efe..28e7d7983 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -186,7 +186,10 @@ func (r *SourceReconciler) ObserveSourceHPA(ctx context.Context, req ctrl.Reques } if hpa.Spec.MaxReplicas != *source.Spec.MaxReplicas || - !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { + !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) || + (source.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior == nil) || + (source.Spec.Pod.AutoScalingBehavior != nil && hpa.Spec.Behavior != nil && + !reflect.DeepEqual(*hpa.Spec.Behavior, *source.Spec.Pod.AutoScalingBehavior)) { condition.Status = metav1.ConditionFalse condition.Action = v1alpha1.Update source.Status.Conditions[v1alpha1.HPA] = condition @@ -230,6 +233,9 @@ func (r *SourceReconciler) ApplySourceHPA(ctx context.Context, req ctrl.Request, if !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = source.Spec.Pod.AutoScalingMetrics } + if source.Spec.Pod.AutoScalingBehavior != nil { + hpa.Spec.Behavior = source.Spec.Pod.AutoScalingBehavior + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for source", "name", source.Name) return err From c369a29e380e9951f4061560cdb7a10fbe399eba Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Wed, 11 Aug 2021 10:30:44 +0800 Subject: [PATCH 21/28] add buildin hpa --- api/v1alpha1/common.go | 4 ++++ api/v1alpha1/hpa.go | 10 ++++++++++ 2 files changed, 14 insertions(+) create mode 100644 api/v1alpha1/hpa.go diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index 79fce139c..d13073db0 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -95,6 +95,10 @@ type PodPolicy struct { // +optional ServiceAccountName string `json:"serviceAccountName,omitempty"` + // BuiltinAutoscaler refers to the built-in autoscaling rules + // +optional + BuiltinAutoscaler BuiltinAutoScaler `json:"builtinAutoscaler,omitempty"` + // AutoScalingMetrics contains the specifications for which to use to calculate the // desired replica count (the maximum replica count across all metrics will // be used). diff --git a/api/v1alpha1/hpa.go b/api/v1alpha1/hpa.go new file mode 100644 index 000000000..9691fc7ae --- /dev/null +++ b/api/v1alpha1/hpa.go @@ -0,0 +1,10 @@ +package v1alpha1 + +type BuiltinAutoScaler struct { + Type string `json:"type,omitempty"` + // +kubebuilder:validation:Optional + // +kubebuilder:pruning:PreserveUnknownFields + Config *Config `json:"config,omitempty"` +} + +type From a8e3b051480fa400dd3289bad3ad8c13c25fc2bd Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 09:54:08 +0800 Subject: [PATCH 22/28] add builtin autoscaler rules --- ...compute_v1alpha1_function_builtin_hpa.yaml | 75 ++++++++++++++ .ci/verify_function_mesh.sh | 8 ++ .../test-integration-kind-samples.yml | 10 ++ api/v1alpha1/common.go | 5 +- api/v1alpha1/hpa.go | 17 ++-- controllers/spec/function.go | 4 +- controllers/spec/hpa.go | 99 +++++++++++++++++-- controllers/spec/sink.go | 4 +- controllers/spec/source.go | 4 +- 9 files changed, 209 insertions(+), 17 deletions(-) create mode 100644 .ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml new file mode 100644 index 000000000..271e66845 --- /dev/null +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -0,0 +1,75 @@ +apiVersion: compute.functionmesh.io/v1alpha1 +kind: Function +metadata: + name: function-builtin-hpa-sample + namespace: default +spec: + image: streamnative/pulsar-functions-java-sample:2.8.0.7 + className: org.apache.pulsar.functions.api.examples.ExclamationFunction + forwardSourceMessageProperty: true + MaxPendingAsyncRequests: 1000 + replicas: 1 + maxReplicas: 5 + logTopic: persistent://public/default/logging-function-logs + input: + topics: + - persistent://public/default/input-java-topic + typeClassName: java.lang.String + output: + topic: persistent://public/default/output-java-topic + typeClassName: java.lang.String + resources: + requests: + cpu: "0.1" + memory: 1G + limits: + cpu: "0.2" + memory: 1.1G + # each secret will be loaded ad an env variable from the `path` secret with the `key` in that secret in the name of `name` + secretsMap: + "name": + path: "test-secret" + key: "username" + "pwd": + path: "test-secret" + key: "password" + pulsar: + pulsarConfig: "test-pulsar" + #authConfig: "test-auth" + java: + jar: /pulsar/examples/api-examples.jar + # to be delete & use admission hook + clusterName: test + autoAck: true + builtinAutoscaler: + - "AverageUtilizationCPUPercent80" + - "AverageUtilizationMemoryPercent20" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-pulsar +data: + webServiceURL: http://sn-platform-pulsar-broker.default.svc.cluster.local:8080 + brokerServiceURL: pulsar://sn-platform-pulsar-broker.default.svc.cluster.local:6650 +#--- +#apiVersion: v1 +#kind: ConfigMap +#metadata: +# name: test-auth +#data: +# clientAuthenticationPlugin: "abc" +# clientAuthenticationParameters: "xyz" +# tlsTrustCertsFilePath: "uvw" +# useTls: "true" +# tlsAllowInsecureConnection: "false" +# tlsHostnameVerificationEnable: "true" +--- +apiVersion: v1 +data: + username: YWRtaW4= + password: MWYyZDFlMmU2N2Rm +kind: Secret +metadata: + name: test-secret +type: Opaque diff --git a/.ci/verify_function_mesh.sh b/.ci/verify_function_mesh.sh index af57cd8fa..d09ee5389 100755 --- a/.ci/verify_function_mesh.sh +++ b/.ci/verify_function_mesh.sh @@ -65,4 +65,12 @@ case ${1} in ci::verify_java_function function-hpa-sample ci::verify_hpa function-hpa-sample ;; + compute_v1alpha1_function_builtin_hpa) + ci::verify_function_mesh function-builtin-hpa-sample + ci::verify_hpa function-builtin-hpa-sample + sleep 60 + ci::print_function_log function-builtin-hpa-sample + ci::verify_java_function function-builtin-hpa-sample + ci::verify_hpa function-builtin-hpa-sample + ;; esac \ No newline at end of file diff --git a/.github/workflows/test-integration-kind-samples.yml b/.github/workflows/test-integration-kind-samples.yml index 29568511d..b87945cfc 100644 --- a/.github/workflows/test-integration-kind-samples.yml +++ b/.github/workflows/test-integration-kind-samples.yml @@ -104,3 +104,13 @@ jobs: run: | .ci/verify_function_mesh.sh compute_v1alpha1_function_hpa kubectl delete -f .ci/clusters/compute_v1alpha1_function_hpa.yaml + + - name: Test Function Builtin HPA + run: | + kubectl apply -f .ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml + kubectl get all + + - name: Verify Function Builtin HPA + run: | + .ci/verify_function_mesh.sh compute_v1alpha1_function_builtin_hpa + kubectl delete -f .ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml diff --git a/api/v1alpha1/common.go b/api/v1alpha1/common.go index d13073db0..f9e542bce 100644 --- a/api/v1alpha1/common.go +++ b/api/v1alpha1/common.go @@ -96,8 +96,11 @@ type PodPolicy struct { ServiceAccountName string `json:"serviceAccountName,omitempty"` // BuiltinAutoscaler refers to the built-in autoscaling rules + // Available values: AverageUtilizationCPUPercent80, AverageUtilizationCPUPercent50, AverageUtilizationCPUPercent20 + // AverageUtilizationMemoryPercent80, AverageUtilizationMemoryPercent50, AverageUtilizationMemoryPercent20 // +optional - BuiltinAutoscaler BuiltinAutoScaler `json:"builtinAutoscaler,omitempty"` + // TODO: validate the rules, user may provide duplicate rules, should check with webhook + BuiltinAutoscaler []BuiltinHPARule `json:"builtinAutoscaler,omitempty"` // AutoScalingMetrics contains the specifications for which to use to calculate the // desired replica count (the maximum replica count across all metrics will diff --git a/api/v1alpha1/hpa.go b/api/v1alpha1/hpa.go index 9691fc7ae..312658fe1 100644 --- a/api/v1alpha1/hpa.go +++ b/api/v1alpha1/hpa.go @@ -1,10 +1,13 @@ package v1alpha1 -type BuiltinAutoScaler struct { - Type string `json:"type,omitempty"` - // +kubebuilder:validation:Optional - // +kubebuilder:pruning:PreserveUnknownFields - Config *Config `json:"config,omitempty"` -} +type BuiltinHPARule string -type +const ( + AverageUtilizationCPUPercent80 BuiltinHPARule = "AverageUtilizationCPUPercent80" + AverageUtilizationCPUPercent50 BuiltinHPARule = "AverageUtilizationCPUPercent50" + AverageUtilizationCPUPercent20 BuiltinHPARule = "AverageUtilizationCPUPercent20" + + AverageUtilizationMemoryPercent80 BuiltinHPARule = "AverageUtilizationMemoryPercent80" + AverageUtilizationMemoryPercent50 BuiltinHPARule = "AverageUtilizationMemoryPercent50" + AverageUtilizationMemoryPercent20 BuiltinHPARule = "AverageUtilizationMemoryPercent20" +) diff --git a/controllers/spec/function.go b/controllers/spec/function.go index 9c60e4121..6a4737feb 100644 --- a/controllers/spec/function.go +++ b/controllers/spec/function.go @@ -37,7 +37,9 @@ func MakeFunctionHPA(function *v1alpha1.Function) *autov2beta2.HorizontalPodAuto Name: function.Name, APIVersion: function.APIVersion, } - if !isDefaultHPAEnabled(function.Spec.Replicas, function.Spec.MaxReplicas, function.Spec.Pod) { + if isBuiltinHPAEnabled(function.Spec.Replicas, function.Spec.MaxReplicas, function.Spec.Pod) { + return makeBuiltinHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, targetRef, function.Spec.Pod.BuiltinAutoscaler) + } else if !isDefaultHPAEnabled(function.Spec.Replicas, function.Spec.MaxReplicas, function.Spec.Pod) { return makeHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, function.Spec.Pod, targetRef) } return makeDefaultHPA(objectMeta, *function.Spec.Replicas, *function.Spec.MaxReplicas, targetRef) diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index 031efad0a..4c7678374 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -24,14 +24,42 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +type BuiltinAutoScaler interface { + metrics() []autov2beta2.MetricSpec +} + func isDefaultHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.PodPolicy) bool { - return minReplicas != nil && maxReplicas != nil && podPolicy.AutoScalingBehavior == nil && len(podPolicy.AutoScalingMetrics) == 0 && *maxReplicas > *minReplicas + return minReplicas != nil && maxReplicas != nil && podPolicy.AutoScalingBehavior == nil && len(podPolicy.AutoScalingMetrics) == 0 && len(podPolicy.BuiltinAutoscaler) == 0 && *maxReplicas > *minReplicas } -// defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. -func defaultHPAMetrics() []autov2beta2.MetricSpec { - // TODO: configurable cpu percentage - cpuPercentage := int32(80) +func isBuiltinHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.PodPolicy) bool { + return minReplicas != nil && maxReplicas != nil && podPolicy.AutoScalingBehavior == nil && len(podPolicy.AutoScalingMetrics) == 0 && len(podPolicy.BuiltinAutoscaler) > 0 && *maxReplicas > *minReplicas +} + +type HPARuleAverageUtilizationCPUPercent struct { + cpuPercentage int32 +} + +type HPARuleAverageUtilizationResourceMemoryPercent struct { + memoryPercentage int32 +} + +func (H *HPARuleAverageUtilizationResourceMemoryPercent) metrics() []autov2beta2.MetricSpec { + return []autov2beta2.MetricSpec{ + { + Type: autov2beta2.ResourceMetricSourceType, + Resource: &autov2beta2.ResourceMetricSource{ + Name: corev1.ResourceMemory, + Target: autov2beta2.MetricTarget{ + Type: autov2beta2.UtilizationMetricType, + AverageUtilization: &H.memoryPercentage, + }, + }, + }, + } +} + +func (H *HPARuleAverageUtilizationCPUPercent) metrics() []autov2beta2.MetricSpec { return []autov2beta2.MetricSpec{ { Type: autov2beta2.ResourceMetricSourceType, @@ -39,13 +67,49 @@ func defaultHPAMetrics() []autov2beta2.MetricSpec { Name: corev1.ResourceCPU, Target: autov2beta2.MetricTarget{ Type: autov2beta2.UtilizationMetricType, - AverageUtilization: &cpuPercentage, + AverageUtilization: &H.cpuPercentage, }, }, }, } } +func NewHPARuleAverageUtilizationCPUPercent(cpuPercentage int32) BuiltinAutoScaler { + return &HPARuleAverageUtilizationCPUPercent{ + cpuPercentage: cpuPercentage, + } +} + +func NewHPARuleAverageUtilizationMemoryPercent(memoryPercentage int32) BuiltinAutoScaler { + return &HPARuleAverageUtilizationResourceMemoryPercent{ + memoryPercentage: memoryPercentage, + } +} + +func GetBuiltinAutoScaler(builtinRule v1alpha1.BuiltinHPARule) BuiltinAutoScaler { + switch builtinRule { + case v1alpha1.AverageUtilizationCPUPercent80: + return NewHPARuleAverageUtilizationCPUPercent(80) + case v1alpha1.AverageUtilizationCPUPercent50: + return NewHPARuleAverageUtilizationCPUPercent(50) + case v1alpha1.AverageUtilizationCPUPercent20: + return NewHPARuleAverageUtilizationCPUPercent(20) + case v1alpha1.AverageUtilizationMemoryPercent80: + return NewHPARuleAverageUtilizationMemoryPercent(80) + case v1alpha1.AverageUtilizationMemoryPercent50: + return NewHPARuleAverageUtilizationMemoryPercent(50) + case v1alpha1.AverageUtilizationMemoryPercent20: + return NewHPARuleAverageUtilizationMemoryPercent(20) + default: + return nil + } +} + +// defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. +func defaultHPAMetrics() []autov2beta2.MetricSpec { + return NewHPARuleAverageUtilizationCPUPercent(80).metrics() +} + func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference) *autov2beta2.HorizontalPodAutoscaler { return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ @@ -62,6 +126,29 @@ func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int3 } } +func makeBuiltinHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference, builtinRules []v1alpha1.BuiltinHPARule) *autov2beta2.HorizontalPodAutoscaler { + metrics := []autov2beta2.MetricSpec{} + for _, r := range builtinRules { + s := GetBuiltinAutoScaler(r) + if s != nil { + metrics = append(metrics, s.metrics()...) + } + } + return &autov2beta2.HorizontalPodAutoscaler{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "autoscaling/v2beta2", + Kind: "HorizontalPodAutoscaler", + }, + ObjectMeta: *objectMeta, + Spec: autov2beta2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: targetRef, + MinReplicas: &minReplicas, + MaxReplicas: maxReplicas, + Metrics: metrics, + }, + } +} + func makeHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, podPolicy v1alpha1.PodPolicy, targetRef autov2beta2.CrossVersionObjectReference) *autov2beta2.HorizontalPodAutoscaler { spec := autov2beta2.HorizontalPodAutoscalerSpec{ ScaleTargetRef: targetRef, diff --git a/controllers/spec/sink.go b/controllers/spec/sink.go index 8948386b9..aaa1f7adb 100644 --- a/controllers/spec/sink.go +++ b/controllers/spec/sink.go @@ -33,7 +33,9 @@ func MakeSinkHPA(sink *v1alpha1.Sink) *autov2beta2.HorizontalPodAutoscaler { Name: sink.Name, APIVersion: sink.APIVersion, } - if !isDefaultHPAEnabled(sink.Spec.Replicas, sink.Spec.MaxReplicas, sink.Spec.Pod) { + if isBuiltinHPAEnabled(sink.Spec.Replicas, sink.Spec.MaxReplicas, sink.Spec.Pod) { + return makeBuiltinHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, targetRef, sink.Spec.Pod.BuiltinAutoscaler) + } else if !isDefaultHPAEnabled(sink.Spec.Replicas, sink.Spec.MaxReplicas, sink.Spec.Pod) { return makeHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, sink.Spec.Pod, targetRef) } return makeDefaultHPA(objectMeta, *sink.Spec.Replicas, *sink.Spec.MaxReplicas, targetRef) diff --git a/controllers/spec/source.go b/controllers/spec/source.go index 85f2aa8af..bc363bade 100644 --- a/controllers/spec/source.go +++ b/controllers/spec/source.go @@ -33,7 +33,9 @@ func MakeSourceHPA(source *v1alpha1.Source) *autov2beta2.HorizontalPodAutoscaler Name: source.Name, APIVersion: source.APIVersion, } - if !isDefaultHPAEnabled(source.Spec.Replicas, source.Spec.MaxReplicas, source.Spec.Pod) { + if isBuiltinHPAEnabled(source.Spec.Replicas, source.Spec.MaxReplicas, source.Spec.Pod) { + return makeBuiltinHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, targetRef, source.Spec.Pod.BuiltinAutoscaler) + } else if !isDefaultHPAEnabled(source.Spec.Replicas, source.Spec.MaxReplicas, source.Spec.Pod) { return makeHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, source.Spec.Pod, targetRef) } return makeDefaultHPA(objectMeta, *source.Spec.Replicas, *source.Spec.MaxReplicas, targetRef) From da5e25f9778ac677c1781e0b791be8f41009555a Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 09:58:22 +0800 Subject: [PATCH 23/28] fix CI --- api/v1alpha1/hpa.go | 20 ++++++++++++++++ api/v1alpha1/zz_generated.deepcopy.go | 5 ++++ ...ompute.functionmesh.io_functionmeshes.yaml | 12 ++++++++++ .../compute.functionmesh.io_functions.yaml | 4 ++++ .../bases/compute.functionmesh.io_sinks.yaml | 4 ++++ .../compute.functionmesh.io_sources.yaml | 4 ++++ manifests/crd.yaml | 24 +++++++++++++++++++ 7 files changed, 73 insertions(+) diff --git a/api/v1alpha1/hpa.go b/api/v1alpha1/hpa.go index 312658fe1..07382f7e8 100644 --- a/api/v1alpha1/hpa.go +++ b/api/v1alpha1/hpa.go @@ -1,3 +1,23 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package v1alpha1 contains API Schema definitions for the cloud v1alpha1 API group +// +kubebuilder:object:generate=true +// +groupName=compute.functionmesh.io package v1alpha1 type BuiltinHPARule string diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 6687167ac..f8bcc964d 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -583,6 +583,11 @@ func (in *PodPolicy) DeepCopyInto(out *PodPolicy) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.BuiltinAutoscaler != nil { + in, out := &in.BuiltinAutoscaler, &out.BuiltinAutoscaler + *out = make([]BuiltinHPARule, len(*in)) + copy(*out, *in) + } if in.AutoScalingMetrics != nil { in, out := &in.AutoScalingMetrics, &out.AutoScalingMetrics *out = make([]v2beta2.MetricSpec, len(*in)) diff --git a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml index fc4655eab..b5e263b8d 100644 --- a/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml +++ b/config/crd/bases/compute.functionmesh.io_functionmeshes.yaml @@ -725,6 +725,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -3145,6 +3149,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -5545,6 +5553,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_functions.yaml b/config/crd/bases/compute.functionmesh.io_functions.yaml index 947862775..e7fc91d48 100644 --- a/config/crd/bases/compute.functionmesh.io_functions.yaml +++ b/config/crd/bases/compute.functionmesh.io_functions.yaml @@ -726,6 +726,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sinks.yaml b/config/crd/bases/compute.functionmesh.io_sinks.yaml index 83f3a3cd7..c64d10a3f 100644 --- a/config/crd/bases/compute.functionmesh.io_sinks.yaml +++ b/config/crd/bases/compute.functionmesh.io_sinks.yaml @@ -660,6 +660,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: diff --git a/config/crd/bases/compute.functionmesh.io_sources.yaml b/config/crd/bases/compute.functionmesh.io_sources.yaml index 54093cba8..b3b22344f 100644 --- a/config/crd/bases/compute.functionmesh.io_sources.yaml +++ b/config/crd/bases/compute.functionmesh.io_sources.yaml @@ -637,6 +637,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 90f080177..4ac2a19c0 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -723,6 +723,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -3143,6 +3147,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -5543,6 +5551,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -8102,6 +8114,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -10586,6 +10602,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: @@ -13050,6 +13070,10 @@ spec: - type type: object type: array + builtinAutoscaler: + items: + type: string + type: array imagePullSecrets: items: properties: From 838965d37cc3a982d8a6806fbdf3fd7483ff3b66 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 11:00:22 +0800 Subject: [PATCH 24/28] fix CI --- .../compute_v1alpha1_function_builtin_hpa.yaml | 7 ++++--- .ci/clusters/compute_v1alpha1_function_hpa.yaml | 15 ++++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml index 271e66845..124983884 100644 --- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -41,9 +41,10 @@ spec: # to be delete & use admission hook clusterName: test autoAck: true - builtinAutoscaler: - - "AverageUtilizationCPUPercent80" - - "AverageUtilizationMemoryPercent20" + pod: + builtinAutoscaler: + - "AverageUtilizationCPUPercent80" + - "AverageUtilizationMemoryPercent20" --- apiVersion: v1 kind: ConfigMap diff --git a/.ci/clusters/compute_v1alpha1_function_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_hpa.yaml index e32a12973..2ce4af9a4 100644 --- a/.ci/clusters/compute_v1alpha1_function_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_hpa.yaml @@ -41,13 +41,14 @@ spec: # to be delete & use admission hook clusterName: test autoAck: true - autoScalingMetrics: - - type: Resource - resource: - name: cpu - target: - type: Utilization - averageUtilization: 50 + pod: + autoScalingMetrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 50 --- apiVersion: v1 kind: ConfigMap From a6c0b57306a7b3cf593eb7096b1704e62f960488 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 12:38:38 +0800 Subject: [PATCH 25/28] validate builtin --- .ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml index 124983884..6c914426a 100644 --- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -43,7 +43,7 @@ spec: autoAck: true pod: builtinAutoscaler: - - "AverageUtilizationCPUPercent80" + - "AverageUtilizationCPUPercent20" - "AverageUtilizationMemoryPercent20" --- apiVersion: v1 From 562ff153612153a9b604f2bfadc548f53f855b9f Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 13:35:01 +0800 Subject: [PATCH 26/28] verify builtin --- .ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml | 4 ++-- .ci/helm.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml index 6c914426a..dcd2685e3 100644 --- a/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml +++ b/.ci/clusters/compute_v1alpha1_function_builtin_hpa.yaml @@ -43,8 +43,8 @@ spec: autoAck: true pod: builtinAutoscaler: - - "AverageUtilizationCPUPercent20" - - "AverageUtilizationMemoryPercent20" + - AverageUtilizationCPUPercent20 + - AverageUtilizationMemoryPercent20 --- apiVersion: v1 kind: ConfigMap diff --git a/.ci/helm.sh b/.ci/helm.sh index 1c5cda656..797973b27 100644 --- a/.ci/helm.sh +++ b/.ci/helm.sh @@ -142,6 +142,8 @@ function ci::verify_function_mesh() { function ci::verify_hpa() { FUNCTION_NAME=$1 + ${KUBECTL} get function + ${KUBECTL} get function ${FUNCTION_NAME} -o yaml ${KUBECTL} get hpa.v2beta2.autoscaling ${KUBECTL} get hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function -o yaml ${KUBECTL} describe hpa.v2beta2.autoscaling ${FUNCTION_NAME}-function From c55ad52b7776b74c4c4e8e21db9a0d07d5e643d3 Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 17:08:24 +0800 Subject: [PATCH 27/28] validate HPA --- controllers/function.go | 11 ++- controllers/function_controller_test.go | 99 ++++++++++++++++++++++++- controllers/sink.go | 8 +- controllers/source.go | 8 +- controllers/spec/hpa.go | 21 ++++-- controllers/test_utils_test.go | 13 +++- 6 files changed, 142 insertions(+), 18 deletions(-) diff --git a/controllers/function.go b/controllers/function.go index 1b33e04a1..e89d44b4a 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -20,6 +20,7 @@ package controllers import ( "context" "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/streamnative/function-mesh/api/v1alpha1" "github.com/streamnative/function-mesh/controllers/spec" @@ -235,6 +236,8 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ r.Log.Error(err, "failed to create pod autoscaler for function", "name", function.Name) return err } + key, _ := client.ObjectKeyFromObject(hpa) + r.Get(context.Background(), key, hpa) case v1alpha1.Update: hpa := &autov2beta2.HorizontalPodAutoscaler{} err := r.Get(ctx, types.NamespacedName{Namespace: function.Namespace, @@ -246,12 +249,18 @@ func (r *FunctionReconciler) ApplyFunctionHPA(ctx context.Context, req ctrl.Requ if hpa.Spec.MaxReplicas != *function.Spec.MaxReplicas { hpa.Spec.MaxReplicas = *function.Spec.MaxReplicas } - if !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { + if len(function.Spec.Pod.AutoScalingMetrics) > 0 && !reflect.DeepEqual(hpa.Spec.Metrics, function.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = function.Spec.Pod.AutoScalingMetrics } if function.Spec.Pod.AutoScalingBehavior != nil { hpa.Spec.Behavior = function.Spec.Pod.AutoScalingBehavior } + if len(function.Spec.Pod.BuiltinAutoscaler) > 0 { + metrics := spec.MakeMetricsFromBuiltinHPARules(function.Spec.Pod.BuiltinAutoscaler) + if !reflect.DeepEqual(hpa.Spec.Metrics, metrics) { + hpa.Spec.Metrics = metrics + } + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for function", "name", function.Name) return err diff --git a/controllers/function_controller_test.go b/controllers/function_controller_test.go index 1e15791a8..ac9e8d7c8 100644 --- a/controllers/function_controller_test.go +++ b/controllers/function_controller_test.go @@ -21,6 +21,9 @@ import ( "context" "time" + "sigs.k8s.io/controller-runtime/pkg/client" + logf "sigs.k8s.io/controller-runtime/pkg/log" + appsv1 "k8s.io/api/apps/v1" "k8s.io/apimachinery/pkg/types" @@ -33,6 +36,8 @@ import ( v1 "k8s.io/api/core/v1" ) +var log = logf.Log.WithName("function-resource-test") + const ( timeout = time.Second * 10 interval = time.Millisecond * 250 @@ -64,8 +69,8 @@ var _ = Describe("Function Controller (Batcher)", func() { var _ = Describe("Function Controller (HPA)", func() { Context("Simple Function Item with HPA", func() { - function := makeFunctionSample(TestFunctionName) - cpuPercentage := int32(80) + function := makeFunctionSample(TestFunctionHPAName) + cpuPercentage := int32(20) function.Spec.Pod.AutoScalingMetrics = []autov2beta2.MetricSpec{ { Type: autov2beta2.ResourceMetricSourceType, @@ -77,6 +82,30 @@ var _ = Describe("Function Controller (HPA)", func() { }, }, }, + { + Type: autov2beta2.ResourceMetricSourceType, + Resource: &autov2beta2.ResourceMetricSource{ + Name: v1.ResourceMemory, + Target: autov2beta2.MetricTarget{ + Type: autov2beta2.UtilizationMetricType, + AverageUtilization: &cpuPercentage, + }, + }, + }, + } + + createFunction(function) + }) +}) + +var _ = Describe("Function Controller (builtin HPA)", func() { + Context("Simple Function Item with builtin HPA", func() { + r := int32(100) + function := makeFunctionSample(TestFunctionBuiltinHPAName) + function.Spec.MaxReplicas = &r + function.Spec.Pod.BuiltinAutoscaler = []v1alpha1.BuiltinHPARule{ + v1alpha1.AverageUtilizationCPUPercent20, + v1alpha1.AverageUtilizationMemoryPercent20, } createFunction(function) @@ -86,7 +115,10 @@ var _ = Describe("Function Controller (HPA)", func() { func createFunction(function *v1alpha1.Function) { It("Function should be created", func() { - Expect(k8sClient.Create(context.Background(), function)).Should(Succeed()) + Eventually(func() bool { + err := k8sClient.Create(context.Background(), function) + return err == nil + }, timeout, interval).Should(BeTrue()) }) It("StatefulSet should be created", func() { @@ -100,6 +132,7 @@ func createFunction(function *v1alpha1.Function) { return err == nil }, timeout, interval).Should(BeTrue()) + Expect(statefulSet.Name).Should(Equal(spec.MakeFunctionObjectMeta(function).Name)) Expect(*statefulSet.Spec.Replicas).Should(Equal(int32(1))) }) @@ -121,10 +154,68 @@ func createFunction(function *v1alpha1.Function) { Name: spec.MakeFunctionObjectMeta(function).Name}, hpa) return err == nil }, timeout, interval).Should(BeTrue()) + + log.Info("HPA should be created", "hpa", hpa, "name", spec.MakeFunctionObjectMeta(function).Name) + + if len(function.Spec.Pod.AutoScalingMetrics) > 0 { + Expect(len(hpa.Spec.Metrics)).Should(Equal(len(function.Spec.Pod.AutoScalingMetrics))) + for _, metric := range function.Spec.Pod.AutoScalingMetrics { + Expect(hpa.Spec.Metrics).Should(ContainElement(metric)) + } + } + + if len(function.Spec.Pod.BuiltinAutoscaler) > 0 { + Expect(len(hpa.Spec.Metrics)).Should(Equal(len(function.Spec.Pod.BuiltinAutoscaler))) + for _, rule := range function.Spec.Pod.BuiltinAutoscaler { + autoscaler := spec.GetBuiltinAutoScaler(rule) + Expect(autoscaler).Should(Not(BeNil())) + Expect(hpa.Spec.Metrics).Should(ContainElement(autoscaler.Metrics()[0])) + } + } } }) It("Function should be deleted", func() { - Expect(k8sClient.Delete(context.Background(), function)).Should(Succeed()) + key, err := client.ObjectKeyFromObject(function) + Expect(err).To(Succeed()) + + log.Info("deleting resource", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + Expect(k8sClient.Delete(context.Background(), function)).To(Succeed()) + + log.Info("waiting for resource to disappear", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + Eventually(func() error { + return k8sClient.Get(context.Background(), key, function) + }, timeout, interval).Should(HaveOccurred()) + log.Info("deleted resource", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + + statefulsets := new(appsv1.StatefulSetList) + err = k8sClient.List(context.Background(), statefulsets, client.InNamespace(function.Namespace)) + Expect(err).Should(BeNil()) + for _, item := range statefulsets.Items { + log.Info("deleting statefulset resource", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + Expect(k8sClient.Delete(context.Background(), &item)).To(Succeed()) + } + log.Info("waiting for StatefulSet resource to disappear", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + Eventually(func() bool { + err := k8sClient.List(context.Background(), statefulsets, client.InNamespace(function.Namespace)) + return err == nil && len(statefulsets.Items) == 0 + }, timeout, interval).Should(BeTrue()) + log.Info("StatefulSet resource deleted", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + + hpas := new(autov2beta2.HorizontalPodAutoscalerList) + err = k8sClient.List(context.Background(), hpas, client.InNamespace(function.Namespace)) + Expect(err).Should(BeNil()) + for _, item := range hpas.Items { + log.Info("deleting HPA resource", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + log.Info("deleting HPA", "item", item) + Expect(k8sClient.Delete(context.Background(), &item)).To(Succeed()) + } + log.Info("waiting for HPA resource to disappear", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + Eventually(func() bool { + err := k8sClient.List(context.Background(), hpas, client.InNamespace(function.Namespace)) + return err == nil && len(hpas.Items) == 0 + }, timeout, interval).Should(BeTrue()) + log.Info("HPA resource deleted", "namespace", key.Namespace, "name", key.Name, "test", CurrentGinkgoTestDescription().FullTestText) + }) } diff --git a/controllers/sink.go b/controllers/sink.go index f92738dd5..ea5499b38 100644 --- a/controllers/sink.go +++ b/controllers/sink.go @@ -228,12 +228,18 @@ func (r *SinkReconciler) ApplySinkHPA(ctx context.Context, req ctrl.Request, sin if hpa.Spec.MaxReplicas != *sink.Spec.MaxReplicas { hpa.Spec.MaxReplicas = *sink.Spec.MaxReplicas } - if !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { + if len(sink.Spec.Pod.AutoScalingMetrics) > 0 && !reflect.DeepEqual(hpa.Spec.Metrics, sink.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = sink.Spec.Pod.AutoScalingMetrics } if sink.Spec.Pod.AutoScalingBehavior != nil { hpa.Spec.Behavior = sink.Spec.Pod.AutoScalingBehavior } + if len(sink.Spec.Pod.BuiltinAutoscaler) > 0 { + metrics := spec.MakeMetricsFromBuiltinHPARules(sink.Spec.Pod.BuiltinAutoscaler) + if !reflect.DeepEqual(hpa.Spec.Metrics, metrics) { + hpa.Spec.Metrics = metrics + } + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for sink", "name", sink.Name) return err diff --git a/controllers/source.go b/controllers/source.go index 28e7d7983..2068004f2 100644 --- a/controllers/source.go +++ b/controllers/source.go @@ -230,12 +230,18 @@ func (r *SourceReconciler) ApplySourceHPA(ctx context.Context, req ctrl.Request, if hpa.Spec.MaxReplicas != *source.Spec.MaxReplicas { hpa.Spec.MaxReplicas = *source.Spec.MaxReplicas } - if !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { + if len(source.Spec.Pod.AutoScalingMetrics) > 0 && !reflect.DeepEqual(hpa.Spec.Metrics, source.Spec.Pod.AutoScalingMetrics) { hpa.Spec.Metrics = source.Spec.Pod.AutoScalingMetrics } if source.Spec.Pod.AutoScalingBehavior != nil { hpa.Spec.Behavior = source.Spec.Pod.AutoScalingBehavior } + if len(source.Spec.Pod.BuiltinAutoscaler) > 0 { + metrics := spec.MakeMetricsFromBuiltinHPARules(source.Spec.Pod.BuiltinAutoscaler) + if !reflect.DeepEqual(hpa.Spec.Metrics, metrics) { + hpa.Spec.Metrics = metrics + } + } if err := r.Update(ctx, hpa); err != nil { r.Log.Error(err, "failed to update pod autoscaler for source", "name", source.Name) return err diff --git a/controllers/spec/hpa.go b/controllers/spec/hpa.go index 4c7678374..3c401da66 100644 --- a/controllers/spec/hpa.go +++ b/controllers/spec/hpa.go @@ -25,7 +25,7 @@ import ( ) type BuiltinAutoScaler interface { - metrics() []autov2beta2.MetricSpec + Metrics() []autov2beta2.MetricSpec } func isDefaultHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.PodPolicy) bool { @@ -33,7 +33,7 @@ func isDefaultHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.Pod } func isBuiltinHPAEnabled(minReplicas, maxReplicas *int32, podPolicy v1alpha1.PodPolicy) bool { - return minReplicas != nil && maxReplicas != nil && podPolicy.AutoScalingBehavior == nil && len(podPolicy.AutoScalingMetrics) == 0 && len(podPolicy.BuiltinAutoscaler) > 0 && *maxReplicas > *minReplicas + return minReplicas != nil && maxReplicas != nil && len(podPolicy.BuiltinAutoscaler) > 0 && *maxReplicas > *minReplicas } type HPARuleAverageUtilizationCPUPercent struct { @@ -44,7 +44,7 @@ type HPARuleAverageUtilizationResourceMemoryPercent struct { memoryPercentage int32 } -func (H *HPARuleAverageUtilizationResourceMemoryPercent) metrics() []autov2beta2.MetricSpec { +func (H *HPARuleAverageUtilizationResourceMemoryPercent) Metrics() []autov2beta2.MetricSpec { return []autov2beta2.MetricSpec{ { Type: autov2beta2.ResourceMetricSourceType, @@ -59,7 +59,7 @@ func (H *HPARuleAverageUtilizationResourceMemoryPercent) metrics() []autov2beta2 } } -func (H *HPARuleAverageUtilizationCPUPercent) metrics() []autov2beta2.MetricSpec { +func (H *HPARuleAverageUtilizationCPUPercent) Metrics() []autov2beta2.MetricSpec { return []autov2beta2.MetricSpec{ { Type: autov2beta2.ResourceMetricSourceType, @@ -105,9 +105,9 @@ func GetBuiltinAutoScaler(builtinRule v1alpha1.BuiltinHPARule) BuiltinAutoScaler } } -// defaultHPAMetrics generates a default HPA metrics settings based on CPU usage and utilized on 80%. +// defaultHPAMetrics generates a default HPA Metrics settings based on CPU usage and utilized on 80%. func defaultHPAMetrics() []autov2beta2.MetricSpec { - return NewHPARuleAverageUtilizationCPUPercent(80).metrics() + return NewHPARuleAverageUtilizationCPUPercent(80).Metrics() } func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference) *autov2beta2.HorizontalPodAutoscaler { @@ -126,14 +126,19 @@ func makeDefaultHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int3 } } -func makeBuiltinHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference, builtinRules []v1alpha1.BuiltinHPARule) *autov2beta2.HorizontalPodAutoscaler { +func MakeMetricsFromBuiltinHPARules(builtinRules []v1alpha1.BuiltinHPARule) []autov2beta2.MetricSpec { metrics := []autov2beta2.MetricSpec{} for _, r := range builtinRules { s := GetBuiltinAutoScaler(r) if s != nil { - metrics = append(metrics, s.metrics()...) + metrics = append(metrics, s.Metrics()...) } } + return metrics +} + +func makeBuiltinHPA(objectMeta *metav1.ObjectMeta, minReplicas, maxReplicas int32, targetRef autov2beta2.CrossVersionObjectReference, builtinRules []v1alpha1.BuiltinHPARule) *autov2beta2.HorizontalPodAutoscaler { + metrics := MakeMetricsFromBuiltinHPARules(builtinRules) return &autov2beta2.HorizontalPodAutoscaler{ TypeMeta: metav1.TypeMeta{ APIVersion: "autoscaling/v2beta2", diff --git a/controllers/test_utils_test.go b/controllers/test_utils_test.go index e19266b80..a1f2e7eb6 100644 --- a/controllers/test_utils_test.go +++ b/controllers/test_utils_test.go @@ -18,13 +18,20 @@ package controllers import ( + "fmt" + "github.com/streamnative/function-mesh/api/v1alpha1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) const TestClusterName string = "test-pulsar" const TestFunctionName string = "test-function" +const TestFunctionE2EName string = "test-function-e2e" +const TestFunctionKeyBatcherName string = "test-function-key-batcher" +const TestFunctionHPAName string = "test-function-hpa" +const TestFunctionBuiltinHPAName string = "test-function-builtin-hpa" const TestFunctionMeshName = "test-function-mesh" const TestSinkName string = "test-sink" const TestSourceName string = "test-source" @@ -34,7 +41,7 @@ func makeSampleObjectMeta(name string) *metav1.ObjectMeta { return &metav1.ObjectMeta{ Name: name, Namespace: TestNameSpace, - UID: "dead-beef", // uid not generate automatically with fake k8s + UID: types.UID(fmt.Sprintf("dead-beef-%s", name)), // uid not generate automatically with fake k8s } } @@ -102,7 +109,7 @@ func makeFunctionSample(functionName string) *v1alpha1.Function { } func makeFunctionSampleWithCryptoEnabled() *v1alpha1.Function { - function := makeFunctionSample(TestFunctionName) + function := makeFunctionSample(TestFunctionE2EName) function.Spec.Input = v1alpha1.InputConf{ Topics: []string{ "persistent://public/default/java-function-input-topic", @@ -137,7 +144,7 @@ func makeFunctionSampleWithCryptoEnabled() *v1alpha1.Function { } func makeFunctionSampleWithKeyBasedBatcher() *v1alpha1.Function { - function := makeFunctionSample(TestFunctionName) + function := makeFunctionSample(TestFunctionKeyBatcherName) function.Spec.Output = v1alpha1.OutputConf{ Topic: "persistent://public/default/java-function-output-topic", ProducerConf: &v1alpha1.ProducerConfig{ From f9b5a2ea59feaca38aed642278d4d10070e0948f Mon Sep 17 00:00:00 2001 From: Rui Fu Date: Thu, 12 Aug 2021 18:23:54 +0800 Subject: [PATCH 28/28] fix style --- controllers/function.go | 1 + 1 file changed, 1 insertion(+) diff --git a/controllers/function.go b/controllers/function.go index e89d44b4a..9d12f3331 100644 --- a/controllers/function.go +++ b/controllers/function.go @@ -20,6 +20,7 @@ package controllers import ( "context" "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/streamnative/function-mesh/api/v1alpha1"