Skip to content
This repository was archived by the owner on Oct 10, 2023. It is now read-only.

Commit 1b41090

Browse files
saji-pivotalsaji-pivotal
committed
Trying to deploy clusters on multiple-azs in vSphere. Build succeeds, no regressions
Signed-off-by: Sudarshan <asudarshan@vmware.com>
1 parent 543a835 commit 1b41090

File tree

5 files changed

+241
-9
lines changed

5 files changed

+241
-9
lines changed

‎pkg/v1/providers/config_default.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,10 @@ VSPHERE_PASSWORD:
111111
VSPHERE_REGION:
112112
VSPHERE_ZONE:
113113

114+
VSPHERE_AZ_0:
115+
VSPHERE_AZ_1:
116+
VSPHERE_AZ_2:
117+
114118
#! FQDN or IP address to vCenter
115119
VSPHERE_SERVER:
116120

‎pkg/v1/providers/infrastructure-vsphere/v1.0.1/ytt/overlay.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,11 @@ spec:
302302
infrastructureRef:
303303
name: #@ "{}-worker".format(data.values.CLUSTER_NAME)
304304
version: #@ data.values.KUBERNETES_VERSION
305+
#@overlay/match missing_ok=True
306+
#@ if data.values.VSPHERE_AZ_0:
307+
failureDomain: #@ data.values.VSPHERE_AZ_0
308+
#@ end
309+
305310
#@overlay/match by=overlay.subset({"kind": "Secret", "metadata":{"name": "${ CLUSTER_NAME }"}})
306311
---
307312
apiVersion: v1

‎pkg/v1/providers/infrastructure-vsphere/ytt/vsphere-overlay.yaml‎

Lines changed: 228 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,228 @@
1-
#! Please add any overlays specific to vSphere provider under this file.
1+
#! Please add any overlays specific to vSphere provider under this file.
2+
#@ load("@ytt:overlay", "overlay")
3+
#@ load("@ytt:data", "data")
4+
5+
6+
#@ load("lib/helpers.star", "get_bom_data_for_tkr_name", "get_default_tkg_bom_data", "kubeadm_image_repo", "get_image_repo_for_component", "get_vsphere_thumbprint")
7+
8+
#@ load("lib/validate.star", "validate_configuration")
9+
#@ load("@ytt:yaml", "yaml")
10+
#@ validate_configuration("vsphere")
11+
12+
#@ bomDataForK8sVersion = get_bom_data_for_tkr_name()
13+
14+
#@ if data.values.CLUSTER_PLAN == "prod":
15+
#@overlay/match by=overlay.subset({"kind":"VSphereCluster"})
16+
---
17+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
18+
kind: VSphereCluster
19+
metadata:
20+
name: #@ data.values.CLUSTER_NAME
21+
spec:
22+
#@ if not data.values.AVI_CONTROL_PLANE_HA_PROVIDER:
23+
controlPlaneEndpoint:
24+
host: #@ data.values.VSPHERE_CONTROL_PLANE_ENDPOINT
25+
#@ else:
26+
#@overlay/remove
27+
controlPlaneEndpoint:
28+
#@ end
29+
thumbprint: #@ get_vsphere_thumbprint()
30+
server: #@ data.values.VSPHERE_SERVER
31+
identityRef:
32+
kind: Secret
33+
name: #@ data.values.CLUSTER_NAME
34+
---
35+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
36+
kind: VSphereMachineTemplate
37+
metadata:
38+
name: #@ "{}-md-1".format(data.values.CLUSTER_NAME)
39+
spec:
40+
template:
41+
spec:
42+
cloneMode: #@ data.values.VSPHERE_CLONE_MODE
43+
datacenter: #@ data.values.VSPHERE_DATACENTER
44+
datastore: #@ data.values.VSPHERE_DATASTORE
45+
storagePolicyName: #@ data.values.VSPHERE_STORAGE_POLICY_ID
46+
diskGiB: #@ data.values.VSPHERE_WORKER_DISK_GIB
47+
folder: #@ data.values.VSPHERE_FOLDER
48+
memoryMiB: #@ data.values.VSPHERE_WORKER_MEM_MIB
49+
network:
50+
devices:
51+
#@overlay/match by=overlay.index(0)
52+
#@overlay/replace
53+
- networkName: #@ data.values.VSPHERE_NETWORK
54+
#@ if data.values.WORKER_NODE_NAMESERVERS:
55+
nameservers: #@ data.values.WORKER_NODE_NAMESERVERS.replace(" ", "").split(",")
56+
#@ end
57+
#@ if data.values.TKG_IP_FAMILY == "ipv6":
58+
dhcp6: true
59+
#@ elif data.values.TKG_IP_FAMILY in ["ipv4,ipv6", "ipv6,ipv4"]:
60+
dhcp4: true
61+
dhcp6: true
62+
#@ else:
63+
dhcp4: true
64+
#@ end
65+
numCPUs: #@ data.values.VSPHERE_WORKER_NUM_CPUS
66+
resourcePool: #@ data.values.VSPHERE_RESOURCE_POOL
67+
server: #@ data.values.VSPHERE_SERVER
68+
template: #@ data.values.VSPHERE_TEMPLATE
69+
---
70+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
71+
kind: VSphereMachineTemplate
72+
metadata:
73+
name: #@ "{}-md-2".format(data.values.CLUSTER_NAME)
74+
spec:
75+
template:
76+
spec:
77+
cloneMode: #@ data.values.VSPHERE_CLONE_MODE
78+
datacenter: #@ data.values.VSPHERE_DATACENTER
79+
datastore: #@ data.values.VSPHERE_DATASTORE
80+
storagePolicyName: #@ data.values.VSPHERE_STORAGE_POLICY_ID
81+
diskGiB: #@ data.values.VSPHERE_WORKER_DISK_GIB
82+
folder: #@ data.values.VSPHERE_FOLDER
83+
memoryMiB: #@ data.values.VSPHERE_WORKER_MEM_MIB
84+
network:
85+
devices:
86+
#@overlay/match by=overlay.index(0)
87+
#@overlay/replace
88+
- networkName: #@ data.values.VSPHERE_NETWORK
89+
#@ if data.values.WORKER_NODE_NAMESERVERS:
90+
nameservers: #@ data.values.WORKER_NODE_NAMESERVERS.replace(" ", "").split(",")
91+
#@ end
92+
#@ if data.values.TKG_IP_FAMILY == "ipv6":
93+
dhcp6: true
94+
#@ elif data.values.TKG_IP_FAMILY in ["ipv4,ipv6", "ipv6,ipv4"]:
95+
dhcp4: true
96+
dhcp6: true
97+
#@ else:
98+
dhcp4: true
99+
#@ end
100+
numCPUs: #@ data.values.VSPHERE_WORKER_NUM_CPUS
101+
resourcePool: #@ data.values.VSPHERE_RESOURCE_POOL
102+
server: #@ data.values.VSPHERE_SERVER
103+
template: #@ data.values.VSPHERE_TEMPLATE
104+
---
105+
apiVersion: cluster.x-k8s.io/v1beta1
106+
kind: MachineDeployment
107+
metadata:
108+
labels:
109+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
110+
name: #@ "{}-md-1".format(data.values.CLUSTER_NAME)
111+
spec:
112+
clusterName: #@ data.values.CLUSTER_NAME
113+
replicas: #@ data.values.WORKER_MACHINE_COUNT_1
114+
selector:
115+
matchLabels:
116+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
117+
template:
118+
metadata:
119+
labels:
120+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
121+
node-pool: #@ "{}-worker-pool".format(data.values.CLUSTER_NAME)
122+
spec:
123+
bootstrap:
124+
configRef:
125+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
126+
kind: KubeadmConfigTemplate
127+
name: #@ "{}-md-1".format(data.values.CLUSTER_NAME)
128+
clusterName: #@ data.values.CLUSTER_NAME
129+
infrastructureRef:
130+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
131+
kind: VSphereMachineTemplate
132+
name: #@ "{}-md-1".format(data.values.CLUSTER_NAME)
133+
version: #@ data.values.KUBERNETES_VERSION
134+
#@ if data.values.VSPHERE_AZ_1:
135+
failureDomain: #@ data.values.VSPHERE_AZ_1
136+
#@ end
137+
---
138+
apiVersion: cluster.x-k8s.io/v1beta1
139+
kind: MachineDeployment
140+
metadata:
141+
labels:
142+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
143+
name: #@ "{}-md-2".format(data.values.CLUSTER_NAME)
144+
spec:
145+
clusterName: #@ data.values.CLUSTER_NAME
146+
replicas: #@ data.values.WORKER_MACHINE_COUNT_2
147+
selector:
148+
matchLabels:
149+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
150+
template:
151+
metadata:
152+
labels:
153+
cluster.x-k8s.io/cluster-name: #@ data.values.CLUSTER_NAME
154+
node-pool: #@ "{}-worker-pool".format(data.values.CLUSTER_NAME)
155+
spec:
156+
bootstrap:
157+
configRef:
158+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
159+
kind: KubeadmConfigTemplate
160+
name: #@ "{}-md-2".format(data.values.CLUSTER_NAME)
161+
clusterName: #@ data.values.CLUSTER_NAME
162+
infrastructureRef:
163+
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
164+
kind: VSphereMachineTemplate
165+
name: #@ "{}-md-2".format(data.values.CLUSTER_NAME)
166+
version: #@ data.values.KUBERNETES_VERSION
167+
#@ if data.values.VSPHERE_AZ_2:
168+
failureDomain: #@ data.values.VSPHERE_AZ_2
169+
#@ end
170+
---
171+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
172+
kind: KubeadmConfigTemplate
173+
metadata:
174+
name: #@ "{}-md-1".format(data.values.CLUSTER_NAME)
175+
namespace: '${ NAMESPACE }'
176+
spec:
177+
template:
178+
spec:
179+
useExperimentalRetryJoin: true
180+
joinConfiguration:
181+
nodeRegistration:
182+
criSocket: /var/run/containerd/containerd.sock
183+
kubeletExtraArgs:
184+
cloud-provider: external
185+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
186+
name: '{{ ds.meta_data.hostname }}'
187+
preKubeadmCommands:
188+
- hostname "{{ ds.meta_data.hostname }}"
189+
- echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts
190+
- echo "127.0.0.1 localhost" >>/etc/hosts
191+
- echo "127.0.0.1 {{ ds.meta_data.hostname }}" >>/etc/hosts
192+
- echo "{{ ds.meta_data.hostname }}" >/etc/hostname
193+
files: []
194+
users:
195+
- name: capv
196+
sshAuthorizedKeys:
197+
- '${ VSPHERE_SSH_AUTHORIZED_KEY }'
198+
sudo: ALL=(ALL) NOPASSWD:ALL
199+
---
200+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
201+
kind: KubeadmConfigTemplate
202+
metadata:
203+
name: #@ "{}-md-2".format(data.values.CLUSTER_NAME)
204+
namespace: '${ NAMESPACE }'
205+
spec:
206+
template:
207+
spec:
208+
useExperimentalRetryJoin: true
209+
joinConfiguration:
210+
nodeRegistration:
211+
criSocket: /var/run/containerd/containerd.sock
212+
kubeletExtraArgs:
213+
cloud-provider: external
214+
tls-cipher-suites: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
215+
name: '{{ ds.meta_data.hostname }}'
216+
preKubeadmCommands:
217+
- hostname "{{ ds.meta_data.hostname }}"
218+
- echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts
219+
- echo "127.0.0.1 localhost" >>/etc/hosts
220+
- echo "127.0.0.1 {{ ds.meta_data.hostname }}" >>/etc/hosts
221+
- echo "{{ ds.meta_data.hostname }}" >/etc/hostname
222+
files: []
223+
users:
224+
- name: capv
225+
sshAuthorizedKeys:
226+
- '${ VSPHERE_SSH_AUTHORIZED_KEY }'
227+
sudo: ALL=(ALL) NOPASSWD:ALL
228+
#@ end

‎pkg/v1/tkg/client/client_suite_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1378,11 +1378,11 @@ var _ = Describe("DistributeMachineDeploymentWorkers", func() {
13781378
isManagementCluster = false
13791379
infraProviderName = constants.InfrastructureProviderVSphere
13801380
})
1381-
It("should put all workers in first MD", func() {
1381+
It("should distribute evenly", func() {
13821382
Expect(err).To(Not(HaveOccurred()))
1383-
Expect(workerCounts[0]).To(Equal(3))
1384-
Expect(workerCounts[1]).To(Equal(0))
1385-
Expect(workerCounts[2]).To(Equal(0))
1383+
Expect(workerCounts[0]).To(Equal(1))
1384+
Expect(workerCounts[1]).To(Equal(1))
1385+
Expect(workerCounts[2]).To(Equal(1))
13861386
})
13871387
})
13881388

‎pkg/v1/tkg/client/validate.go‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1351,10 +1351,6 @@ func (c *TkgClient) ConfigureAndValidateCNIType(cniType string) error {
13511351
// DistributeMachineDeploymentWorkers distributes machine deployment for worker nodes
13521352
func (c *TkgClient) DistributeMachineDeploymentWorkers(workerMachineCount int64, isProdConfig, isManagementCluster bool, infraProviderName string) ([]int, error) { // nolint:gocyclo
13531353
workerCounts := make([]int, 3)
1354-
if infraProviderName != AWSProviderName && infraProviderName != AzureProviderName {
1355-
workerCounts[0] = int(workerMachineCount)
1356-
return workerCounts, nil
1357-
}
13581354
workerCount1Str, err1 := c.TKGConfigReaderWriter().Get(constants.ConfigVariableWorkerMachineCount0)
13591355
workerCount2Str, err2 := c.TKGConfigReaderWriter().Get(constants.ConfigVariableWorkerMachineCount1)
13601356
workerCount3Str, err3 := c.TKGConfigReaderWriter().Get(constants.ConfigVariableWorkerMachineCount2)

0 commit comments

Comments
 (0)