Register cleanup for antrea and disable secretgen and vsphere-az1/2

Antrea fails to start after Windows nodes reboot because OVS bridge and
HnsNetwork previously created by Antrea is not cleaned, a cleanup script
is provided in the new version of Antrea, this change registers the
script as a callback on system's shutdown, the cleanup script will be
invoked before Windows shutdown.

secretgen does not support Windows cluster, just disable it.

Disable vsphere-az1/2 on Windows cluster because this is not tested on
Windows clusters.

Author: hxietkg

pkg/v1/providers/infrastructure-vsphere/ytt/vsphere-overlay.yaml

@@ -11,7 +11,7 @@
 
 #@ bomDataForK8sVersion = get_bom_data_for_tkr_name()
 
-#@ if data.values.CLUSTER_PLAN == "prod":
+#@ if data.values.CLUSTER_PLAN == "prod" and not data.values.IS_WINDOWS_WORKLOAD_CLUSTER:
 #@overlay/match by=overlay.subset({"kind":"VSphereCluster"})
 ---
 apiVersion: infrastructure.cluster.x-k8s.io/v1beta1

pkg/v1/providers/tests/unit/windows_test.go

@@ -30,6 +30,7 @@ var _ = Describe("Windows Ytt Templating", func() {
 			filepath.Join(yamlRoot, "ytt", "02_addons", "cpi", "cpi_addon_data.lib.yaml"),
 			filepath.Join(yamlRoot, "ytt", "03_customizations", "02_avi", "ako-deployment.lib.yaml"),
 			//filepath.Join(YAML_ROOT, "provider-bundle", "providers", "ytt", "02_addons", "cpi", "cpi_addon_data.lib.yaml"),
+			filepath.Join(yamlRoot, "ytt", "03_customizations", "03_windows"),
 			filepath.Join(yamlRoot, "ytt"), // lib/helpers.star, lib/config_variable_association.star, lib/validate.star
 		}
 	})

pkg/v1/providers/ytt/02_addons/secretgen-controller/add_secretgen-controller.yaml

@@ -4,7 +4,7 @@
 #@ load("/lib/helpers.star", "ValuesFormatStr")
 #@ load("secretgen-controller_addon_data.lib.yaml", "secretgencontrollerdatavalues")
 
-#@ if data.values.PROVIDER_TYPE != "tkg-service-vsphere" and data.values.SECRETGEN_CONTROLLER_ENABLE:
+#@ if data.values.PROVIDER_TYPE != "tkg-service-vsphere" and data.values.SECRETGEN_CONTROLLER_ENABLE and not data.values.IS_WINDOWS_WORKLOAD_CLUSTER:
 ---
 apiVersion: v1
 kind: Secret

pkg/v1/providers/ytt/03_customizations/03_windows/prevent_windows_updates.yaml

@@ -1,15 +1,17 @@
 #@ load("@ytt:overlay", "overlay")
 #@ load("@ytt:data", "data")
 #@ if data.values.IS_WINDOWS_WORKLOAD_CLUSTER:
-#@overlay/match by=overlay.subset({"kind":"KubeadmConfigTemplate"})
+#@overlay/match by=overlay.subset({"kind":"KubeadmConfigTemplate"}), expects="1+"
 ---
 spec:
   template:
     spec:
       files:
+      #@overlay/append
       - path: c:\k\prevent_windows_updates.ps1
         content: |
           Set-Service -Name "wuauserv" -StartupType Disabled -Status Stopped
       postKubeadmCommands:
+      #@overlay/append
       - powershell c:/k/prevent_windows_updates.ps1 -ExecutionPolicy Bypass
 #@ end

pkg/v1/providers/ytt/03_customizations/03_windows/register_antrea_cleanup.yaml

@@ -0,0 +1,57 @@
+#@ load("@ytt:overlay", "overlay")
+#@ load("@ytt:data", "data")
+#@ if data.values.IS_WINDOWS_WORKLOAD_CLUSTER:
+#@ if data.values.CNI == "antrea":
+#@overlay/match by=overlay.subset({"kind":"KubeadmConfigTemplate"}), expects="1+"
+---
+spec:
+  template:
+    spec:
+      files:
+      #@overlay/append
+      - path: C:\k\register_antrea_cleanup.ps1
+        content: |
+          $methodScript = "C:\k\antrea\Clean-AntreaNetwork.ps1"
+          if (Test-Path "$methodScript") {
+              $method = "Shutdown"
+              $RegPath = "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Group Policy"
+              $RegScriptsPath = "$RegPath\Scripts\$method\0"
+              $RegSmScriptsPath = "$RegPath\State\Machine\Scripts\$method\0"
+              # Create the path if not exist
+              $gpoPath = "$ENV:systemRoot\System32\GroupPolicy\Machine"
+              $methodPath = "$gpoPath\Scripts\$method"
+              if (-not (Test-Path $methodPath)) {
+                  New-Item -path $methodPath -itemType Directory
+              }
+              # Create sub-path
+              $items = @("$RegScriptsPath\0", "$RegSmScriptsPath\0")
+              foreach ($item in $items) {
+                  if (-not (Test-Path $item)) {
+                      New-Item -path $item -force
+                  }
+              }
+              # Register callback script to GPO
+              $items = @("$RegScriptsPath", "$RegSmScriptsPath")
+              foreach ($item in $items) {
+                  New-ItemProperty -path "$item" -name DisplayName -propertyType String -value "Local Group Policy" -force
+                  New-ItemProperty -path "$item" -name FileSysPath -propertyType String -value "$gpoPath" -force
+                  New-ItemProperty -path "$item" -name GPO-ID -propertyType String -value "LocalGPO" -force
+                  New-ItemProperty -path "$item" -name GPOName -propertyType String -value "Local Group Policy" -force
+                  New-ItemProperty -path "$item" -name PSScriptOrder -propertyType DWord -value 2 -force
+                  New-ItemProperty -path "$item" -name SOM-ID -propertyType String -value "Local" -force
+              }
+              $BinaryString = "00,00,00,00,00,00,00,00,00,00,00,00,00,00,00,00"
+              $ExecTime = $BinaryString.Split(',') | ForEach-Object {"0x$_"}
+              $items = @("$RegScriptsPath\0", "$RegSmScriptsPath\0")
+              foreach ($item in $items) {
+                  New-ItemProperty -path "$item" -name Script -propertyType String -value $methodScript -force
+                  New-ItemProperty -path "$item" -name Parameters -propertyType String -value $method -force
+                  New-ItemProperty -path "$item" -name IsPowershell -propertyType DWord -value 1 -force
+                  New-ItemProperty -path "$item" -name ExecTime -propertyType Binary -value ([byte[]]$ExecTime) -force
+              }
+          }
+      postKubeadmCommands:
+      #@overlay/append
+      - powershell C:/k/register_antrea_cleanup.ps1 -ExecutionPolicy Bypass
+#@ end
+#@ end

pkg/v1/tkg/client/client.go

@@ -54,6 +54,7 @@ type CreateClusterOptions struct {
 	SkipValidation              bool
 	ClusterType                 TKGClusterType
 	Edition                     string
+	IsWindowsWorkloadCluster    bool
 }
 
 // InitRegionOptions contains options supported by InitRegion

pkg/v1/tkg/client/client_suite_test.go

@@ -1368,7 +1368,7 @@ var _ = Describe("DistributeMachineDeploymentWorkers", func() {
 	JustBeforeEach(func() {
 		tkgClient, err = CreateTKGClient(tkgConfigPath, testingDir, defaultTKGBoMFileForTesting, 2*time.Second)
 		Expect(err).NotTo(HaveOccurred())
-		workerCounts, err = tkgClient.DistributeMachineDeploymentWorkers(workerMachineCount, isProdConfig, isManagementCluster, infraProviderName)
+		workerCounts, err = tkgClient.DistributeMachineDeploymentWorkers(workerMachineCount, isProdConfig, isManagementCluster, infraProviderName, false)
 	})
 
 	Context("when not aws and azure", func() {

pkg/v1/tkg/client/cluster.go

@@ -130,7 +130,7 @@ func (c *TkgClient) CreateCluster(options *CreateClusterOptions, waitForCluster
 	if err != nil {
 		return err
 	}
-	bytes, err = c.getClusterConfiguration(&options.ClusterConfigOptions, isManagementCluster, infraProviderName)
+	bytes, err = c.getClusterConfiguration(&options.ClusterConfigOptions, isManagementCluster, infraProviderName, options.IsWindowsWorkloadCluster)
 	if err != nil {
 		return errors.Wrap(err, "unable to get cluster configuration")
 	}

pkg/v1/tkg/client/config.go

@@ -44,7 +44,7 @@ func (c *TkgClient) GetClusterConfiguration(options *CreateClusterOptions) ([]by
 			if err := c.configureAndValidateConfiguration(options, nil, true); err != nil {
 				return nil, err
 			}
-			return c.getClusterConfiguration(&options.ClusterConfigOptions, false, provider)
+			return c.getClusterConfiguration(&options.ClusterConfigOptions, false, provider, options.IsWindowsWorkloadCluster)
 		}
 	}
 	currentRegion, err := c.GetCurrentRegionContext()
@@ -87,7 +87,7 @@ func (c *TkgClient) GetClusterConfiguration(options *CreateClusterOptions) ([]by
 		return nil, err
 	}
 
-	return c.getClusterConfiguration(&options.ClusterConfigOptions, false, infraProviderName)
+	return c.getClusterConfiguration(&options.ClusterConfigOptions, false, infraProviderName, options.IsWindowsWorkloadCluster)
 }
 
 func (c *TkgClient) configureAndValidateConfiguration(options *CreateClusterOptions, regionalClusterClient clusterclient.Client, skipValidation bool) error {
@@ -102,7 +102,7 @@ func (c *TkgClient) configureAndValidateConfiguration(options *CreateClusterOpti
 	return nil
 }
 
-func (c *TkgClient) getClusterConfiguration(options *ClusterConfigOptions, isManagementCluster bool, infraProvider string) ([]byte, error) {
+func (c *TkgClient) getClusterConfiguration(options *ClusterConfigOptions, isManagementCluster bool, infraProvider string, isWindowsWorkloadCluster bool) ([]byte, error) {
 	// Set CLUSTER_PLAN to viper configuration
 	c.SetPlan(options.ProviderRepositorySource.Flavor)
 
@@ -112,7 +112,7 @@ func (c *TkgClient) getClusterConfiguration(options *ClusterConfigOptions, isMan
 	}
 
 	// need to provide clusterctl the worker count for md0 and not the full worker-machine-count value.
-	workerCounts, err := c.DistributeMachineDeploymentWorkers(*options.WorkerMachineCount, options.ProviderRepositorySource.Flavor == constants.PlanProd, isManagementCluster, infraProviderName)
+	workerCounts, err := c.DistributeMachineDeploymentWorkers(*options.WorkerMachineCount, options.ProviderRepositorySource.Flavor == constants.PlanProd, isManagementCluster, infraProviderName, isWindowsWorkloadCluster)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to distribute machine deployments")
 	}

pkg/v1/tkg/client/init.go

@@ -574,7 +574,7 @@ func (c *TkgClient) BuildRegionalClusterConfiguration(options *InitRegionOptions
 		clusterConfigOptions.YamlProcessor = yamlprocessor.NewYttProcessorWithConfigDir(c.tkgConfigDir)
 	}
 
-	bytes, err = c.getClusterConfiguration(&clusterConfigOptions, true, clusterConfigOptions.ProviderRepositorySource.InfrastructureProvider)
+	bytes, err = c.getClusterConfiguration(&clusterConfigOptions, true, clusterConfigOptions.ProviderRepositorySource.InfrastructureProvider, false)
 
 	return bytes, options.ClusterName, err
 }