From 9965c287b422c442f794b63371d9899114a143f0 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Fri, 3 Jun 2022 07:48:48 +0200 Subject: [PATCH 1/2] Tune leader election --- api/v1alpha1/zz_generated.deepcopy.go | 1 + main.go | 23 ++++++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 8cb92b6a..b89689b3 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -1,3 +1,4 @@ +//go:build !ignore_autogenerated // +build !ignore_autogenerated /* diff --git a/main.go b/main.go index e8270c94..e29c0808 100644 --- a/main.go +++ b/main.go @@ -75,8 +75,9 @@ const ( ) var ( - scheme = runtime.NewScheme() - shutDownperiod = time.Second * 5 + scheme = runtime.NewScheme() + shutDownperiod = time.Second * 5 + leaderRetryDuration = 5 * time.Second ) func init() { @@ -118,12 +119,15 @@ func main() { } }() - var err error var configFile string flag.StringVar(&configFile, "config", "", "The controller will load its initial configuration from this file. "+ "Omit this flag to use the default configuration values. "+ "Command-line flags override configuration from this file.") + + var leaderRenewSeconds uint + flag.UintVar(&leaderRenewSeconds, "leader-renew-seconds", 10, "Leader renewal frequency") + flag.Parse() verbosity := 3 // TODO make this configurable 1-5 (5 is a security risk) @@ -144,6 +148,8 @@ func main() { logger := ctrl.Log.WithName("portal_manager") setupLogger := logger.WithName("setup") + var err error + currentNS := os.Getenv(podNamespace) if len(currentNS) == 0 { err = errors.New("current namespace not found") @@ -214,9 +220,16 @@ func main() { } ctrlConfig.IOTCore = iotConfig + renewDeadline := time.Duration(leaderRenewSeconds) * time.Second + leaseDuration := time.Duration(int(1.2*float64(leaderRenewSeconds))) * time.Second + options := ctrl.Options{ - Scheme: scheme, - LeaderElectionID: "storageos-portal-manager-leader", + Scheme: scheme, + LeaderElectionID: "storageos-portal-manager-leader", + LeaderElectionResourceLock: "leases", + RenewDeadline: &renewDeadline, + LeaseDuration: &leaseDuration, + RetryPeriod: &leaderRetryDuration, } if configFile != "" { options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&ctrlConfig)) From 8a9af94db6e70dd0ab8860df5ca94213df7b3b8c Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Tue, 7 Jun 2022 10:44:52 +0200 Subject: [PATCH 2/2] Review fix --- main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index e29c0808..9da7804a 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "go.uber.org/zap/zapcore" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/tools/leaderelection/resourcelock" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -226,7 +227,7 @@ func main() { options := ctrl.Options{ Scheme: scheme, LeaderElectionID: "storageos-portal-manager-leader", - LeaderElectionResourceLock: "leases", + LeaderElectionResourceLock: resourcelock.LeasesResourceLock, RenewDeadline: &renewDeadline, LeaseDuration: &leaseDuration, RetryPeriod: &leaderRetryDuration,