Tune leader election

main.go

@@ -75,8 +75,9 @@ const (
 )
 
 var (
-	scheme         = runtime.NewScheme()
-	shutDownperiod = time.Second * 5
+	scheme              = runtime.NewScheme()
+	shutDownperiod      = time.Second * 5
+	leaderRetryDuration = 5 * time.Second
 )
 
 func init() {
@@ -118,12 +119,15 @@ func main() {
 		}
 	}()
 
-	var err error
 	var configFile string
 	flag.StringVar(&configFile, "config", "",
 		"The controller will load its initial configuration from this file. "+
 			"Omit this flag to use the default configuration values. "+
 			"Command-line flags override configuration from this file.")
+
+	var leaderRenewSeconds uint
+	flag.UintVar(&leaderRenewSeconds, "leader-renew-seconds", 10, "Leader renewal frequency")
+
 	flag.Parse()
 
 	verbosity := 3 // TODO make this configurable 1-5 (5 is a security risk)
@@ -144,6 +148,8 @@ func main() {
 	logger := ctrl.Log.WithName("portal_manager")
 	setupLogger := logger.WithName("setup")
 
+	var err error
+
 	currentNS := os.Getenv(podNamespace)
 	if len(currentNS) == 0 {
 		err = errors.New("current namespace not found")
@@ -214,9 +220,16 @@ func main() {
 	}
 	ctrlConfig.IOTCore = iotConfig
 
+	renewDeadline := time.Duration(leaderRenewSeconds) * time.Second
+	leaseDuration := time.Duration(int(1.2*float64(leaderRenewSeconds))) * time.Second
+
 	options := ctrl.Options{
-		Scheme:           scheme,
-		LeaderElectionID: "storageos-portal-manager-leader",
+		Scheme:                     scheme,
+		LeaderElectionID:           "storageos-portal-manager-leader",
+		LeaderElectionResourceLock: "leases",
+		RenewDeadline:              &renewDeadline,
+		LeaseDuration:              &leaseDuration,
+		RetryPeriod:                &leaderRetryDuration,
 	}
 	if configFile != "" {
 		options, err = options.AndFrom(ctrl.ConfigFile().AtPath(configFile).OfKind(&ctrlConfig))