Skip to content

Add Druid deployment lifecycle tracking, pipeline status semantics, and rollout metrics #16

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
razinbouzar wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add Druid deployment lifecycle tracking, pipeline status semantics, and rollout metrics #16

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 60 additions & 12 deletions apis/druid/v1alpha1/druid_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,18 @@ type DruidSpec struct {
// +kubebuilder:default:="IfNotPresent"
ImagePullPolicy v1.PullPolicy `json:"imagePullPolicy,omitempty"`

// ForceRedeployToken forces a rollout when changed, even if the image tag is unchanged.
// This is primarily intended for mutable-tag redeploys driven by external automation.
// +optional
ForceRedeployToken string `json:"forceRedeployToken,omitempty"`

// ExpectedBuildRevision is the Druid runtime build identifier that must be observed
// across live servers before a deployment lifecycle is considered complete.
// The operator verifies this against sys.servers.build_revision when available and
// falls back to sys.servers.version for Druid versions that do not expose build_revision.
// +optional
ExpectedBuildRevision string `json:"expectedBuildRevision,omitempty"`

// Env Environment variables for druid containers.
// +optional
Env []v1.EnvVar `json:"env,omitempty"`
Expand Down Expand Up @@ -570,20 +582,56 @@ type DruidNodeTypeStatus struct {
Reason string `json:"reason,omitempty"`
}

type DeploymentLifecyclePhase string

const (
DeploymentLifecyclePending DeploymentLifecyclePhase = "Pending"
DeploymentLifecycleInProgress DeploymentLifecyclePhase = "InProgress"
DeploymentLifecycleSucceeded DeploymentLifecyclePhase = "Succeeded"
DeploymentLifecycleFailed DeploymentLifecyclePhase = "Failed"
)

type DeploymentLifecycleTrigger string

const (
DeploymentTriggerSpecChange DeploymentLifecycleTrigger = "SpecChange"
DeploymentTriggerImageChange DeploymentLifecycleTrigger = "ImageChange"
DeploymentTriggerManualRollout DeploymentLifecycleTrigger = "ManualRollout"
)

type DeploymentLifecycleStatus struct {
Revision string `json:"revision,omitempty"`
// +kubebuilder:validation:Enum=Pending;InProgress;Succeeded;Failed
Phase DeploymentLifecyclePhase `json:"phase,omitempty"`
Reason string `json:"reason,omitempty"`
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
StartedAt *metav1.Time `json:"startedAt,omitempty"`
CompletedAt *metav1.Time `json:"completedAt,omitempty"`

LastSuccessfulImage string `json:"lastSuccessfulImage,omitempty"`
LastSuccessfulImageInputsHash string `json:"lastSuccessfulImageInputsHash,omitempty"`
LastSuccessfulForceRedeployToken string `json:"lastSuccessfulForceRedeployToken,omitempty"`

// +kubebuilder:validation:Enum=SpecChange;ImageChange;ManualRollout
Trigger DeploymentLifecycleTrigger `json:"trigger,omitempty"`

ExpectedBuildRevision string `json:"expectedBuildRevision,omitempty"`
ObservedBuildRevisions []string `json:"observedBuildRevisions,omitempty"`
}

// DruidClusterStatus Defines the observed state of Druid.
type DruidClusterStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
DruidNodeStatus DruidNodeTypeStatus `json:"druidNodeStatus,omitempty"`
StatefulSets []string `json:"statefulSets,omitempty"`
Deployments []string `json:"deployments,omitempty"`
Services []string `json:"services,omitempty"`
ConfigMaps []string `json:"configMaps,omitempty"`
PodDisruptionBudgets []string `json:"podDisruptionBudgets,omitempty"`
Ingress []string `json:"ingress,omitempty"`
HPAutoScalers []string `json:"hpAutoscalers,omitempty"`
Pods []string `json:"pods,omitempty"`
PersistentVolumeClaims []string `json:"persistentVolumeClaims,omitempty"`
DeploymentLifecycle DeploymentLifecycleStatus `json:"deploymentLifecycle,omitempty"`
DruidNodeStatus DruidNodeTypeStatus `json:"druidNodeStatus,omitempty"`
StatefulSets []string `json:"statefulSets,omitempty"`
Deployments []string `json:"deployments,omitempty"`
Services []string `json:"services,omitempty"`
ConfigMaps []string `json:"configMaps,omitempty"`
PodDisruptionBudgets []string `json:"podDisruptionBudgets,omitempty"`
Ingress []string `json:"ingress,omitempty"`
HPAutoScalers []string `json:"hpAutoscalers,omitempty"`
Pods []string `json:"pods,omitempty"`
PersistentVolumeClaims []string `json:"persistentVolumeClaims,omitempty"`
}

// Druid is the Schema for the druids API.
Expand Down
29 changes: 29 additions & 0 deletions apis/druid/v1alpha1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

56 changes: 53 additions & 3 deletions chart/crds/druid.apache.org_druids.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1712,6 +1712,13 @@ spec:
x-kubernetes-map-type: atomic
type: object
type: array
expectedBuildRevision:
description: |-
ExpectedBuildRevision is the Druid runtime build identifier that must be observed
across live servers before a deployment lifecycle is considered complete.
The operator verifies this against sys.servers.build_revision when available and
falls back to sys.servers.version for Druid versions that do not expose build_revision.
type: string
extraCommonConfig:
description: |-
ExtraCommonConfig References to ConfigMaps holding more configuration files to mount to the
Expand Down Expand Up @@ -1768,6 +1775,11 @@ spec:
issue: https://github.com/kubernetes/kubernetes/issues/67250
doc: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#forced-rollback
type: boolean
forceRedeployToken:
description: |-
ForceRedeployToken forces a rollout when changed, even if the image tag is unchanged.
This is primarily intended for mutable-tag redeploys driven by external automation.
type: string
hdfs-site.xml:
description: HdfsSite Contents of `hdfs-site.xml`.
type: string
Expand Down Expand Up @@ -11851,14 +11863,52 @@ spec:
items:
type: string
type: array
deploymentLifecycle:
properties:
completedAt:
format: date-time
type: string
expectedBuildRevision:
type: string
lastSuccessfulForceRedeployToken:
type: string
lastSuccessfulImage:
type: string
lastSuccessfulImageInputsHash:
type: string
observedBuildRevisions:
items:
type: string
type: array
observedGeneration:
format: int64
type: integer
phase:
enum:
- Pending
- InProgress
- Succeeded
- Failed
type: string
reason:
type: string
revision:
type: string
startedAt:
format: date-time
type: string
trigger:
enum:
- SpecChange
- ImageChange
- ManualRollout
type: string
type: object
deployments:
items:
type: string
type: array
druidNodeStatus:
description: |-
INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
Important: Run "make" to regenerate code after modifying this file
properties:
druidNode:
type: string
Expand Down
56 changes: 53 additions & 3 deletions config/crd/bases/druid.apache.org_druids.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1712,6 +1712,13 @@ spec:
x-kubernetes-map-type: atomic
type: object
type: array
expectedBuildRevision:
description: |-
ExpectedBuildRevision is the Druid runtime build identifier that must be observed
across live servers before a deployment lifecycle is considered complete.
The operator verifies this against sys.servers.build_revision when available and
falls back to sys.servers.version for Druid versions that do not expose build_revision.
type: string
extraCommonConfig:
description: |-
ExtraCommonConfig References to ConfigMaps holding more configuration files to mount to the
Expand Down Expand Up @@ -1768,6 +1775,11 @@ spec:
issue: https://github.com/kubernetes/kubernetes/issues/67250
doc: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#forced-rollback
type: boolean
forceRedeployToken:
description: |-
ForceRedeployToken forces a rollout when changed, even if the image tag is unchanged.
This is primarily intended for mutable-tag redeploys driven by external automation.
type: string
hdfs-site.xml:
description: HdfsSite Contents of `hdfs-site.xml`.
type: string
Expand Down Expand Up @@ -11851,14 +11863,52 @@ spec:
items:
type: string
type: array
deploymentLifecycle:
properties:
completedAt:
format: date-time
type: string
expectedBuildRevision:
type: string
lastSuccessfulForceRedeployToken:
type: string
lastSuccessfulImage:
type: string
lastSuccessfulImageInputsHash:
type: string
observedBuildRevisions:
items:
type: string
type: array
observedGeneration:
format: int64
type: integer
phase:
enum:
- Pending
- InProgress
- Succeeded
- Failed
type: string
reason:
type: string
revision:
type: string
startedAt:
format: date-time
type: string
trigger:
enum:
- SpecChange
- ImageChange
- ManualRollout
type: string
type: object
deployments:
items:
type: string
type: array
druidNodeStatus:
description: |-
INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
Important: Run "make" to regenerate code after modifying this file
properties:
druidNode:
type: string
Expand Down
47 changes: 41 additions & 6 deletions controllers/druid/druid_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,11 @@ package druid

import (
"context"
"errors"
"os"
"time"

"k8s.io/apimachinery/pkg/api/errors"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/tools/record"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -70,14 +71,16 @@ func NewDruidReconciler(mgr ctrl.Manager) *DruidReconciler {
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch

func (r *DruidReconciler) Reconcile(ctx context.Context, request reconcile.Request) (ctrl.Result, error) {
func (r *DruidReconciler) Reconcile(ctx context.Context, request reconcile.Request) (result ctrl.Result, err error) {
_ = r.Log.WithValues("druid", request.NamespacedName)

// Fetch the Druid instance
instance := &druidv1alpha1.Druid{}
err := r.Get(ctx, request.NamespacedName, instance)
err = r.Get(ctx, request.NamespacedName, instance)
if err != nil {
if errors.IsNotFound(err) {
if k8serrors.IsNotFound(err) {
defaultDeploymentLifecycleMetrics.deleteCluster(request.Namespace, request.Name)
defaultDruidRolloutMetrics.deleteCluster(request.Namespace, request.Name)
// Request object not found, could have been deleted after reconcile request.
// Owned objects are automatically garbage collected. For additional cleanup logic use finalizers.
// Return and don't requeue
Expand All @@ -87,16 +90,48 @@ func (r *DruidReconciler) Reconcile(ctx context.Context, request reconcile.Reque
return ctrl.Result{}, err
}

defer func() {
if metricsErr := defaultDruidRolloutMetrics.sync(ctx, r.Client, instance); metricsErr != nil {
r.Log.Error(metricsErr, "failed to sync rollout metrics", "name", instance.Name, "namespace", instance.Namespace)
}
}()

// Initialize Emit Events
var emitEvent EventEmitter = EmitEventFuncs{r.Recorder}

if err := ensureDeploymentLifecycleStarted(ctx, r.Client, instance, emitEvent); err != nil {
return ctrl.Result{}, err
}

runLifecycleStep := func(step func() error) error {
if err := step(); err != nil {
if shouldPersistDeploymentLifecycleFailure(err) {
if patchErr := markDeploymentLifecycleFailed(ctx, r.Client, instance, emitEvent, err); patchErr != nil {
return errors.Join(err, patchErr)
}
}
return err
}
return nil
}

// Deploy Druid Cluster
if err := deployDruidCluster(ctx, r.Client, instance, emitEvent); err != nil {
if err := runLifecycleStep(func() error {
return deployDruidCluster(ctx, r.Client, instance, emitEvent)
}); err != nil {
return ctrl.Result{}, err
}

// Update Druid Dynamic Configs
if err := updateDruidDynamicConfigs(ctx, r.Client, instance, emitEvent); err != nil {
if err := runLifecycleStep(func() error {
return updateDruidDynamicConfigs(ctx, r.Client, instance, emitEvent)
}); err != nil {
return ctrl.Result{}, err
}

if err := runLifecycleStep(func() error {
return reconcileDeploymentLifecycle(ctx, r.Client, instance, emitEvent)
}); err != nil {
return ctrl.Result{}, err
}

Expand Down
Loading
Loading