Skip to content

Commit 5ef82b5

Browse files
fix: Add Tilt configs for local development (#53)
* fix: Add Tilt configs for local development and add missing RBAC annotations that have been added to the helm chart. * fix: Add a local minio and a more functional default setup * fix: refactor local overrides, add sample config file update README.md * fix: need to specify the chart until this gets released to stable
1 parent ba0e85f commit 5ef82b5

10 files changed

Lines changed: 374 additions & 14 deletions

File tree

.gitignore

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,4 +27,6 @@ Dockerfile.cross
2727

2828
/charts
2929
/cdk8s
30-
/git
30+
/git
31+
32+
tilt-settings.json

README.md

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,51 @@
22

33
## Development
44

5-
- `make install run` - runs the operator outside of the cluster. This is in a
6-
non production mode, so the console will fail to load operator logs because
7-
its not inside the cluster
5+
### Prerequisites
6+
7+
- A Kubernetes cluster (e.g. [kind](https://kind.sigs.k8s.io/))
8+
- [Tilt](https://tilt.dev/)
9+
10+
#### Install Kind
11+
12+
A kubernetes cluster is required to run the operator. [kind](https://kind.sigs.k8s.io/) is recommended for local development. Install `kind` and
13+
create a cluster:
14+
15+
```bash
16+
brew install kind
17+
kind create cluster
18+
```
19+
20+
This will create a new kind cluster with the name `kind`. The kubernetes context will be called `kind-kind`.
21+
22+
#### Install Tilt
23+
24+
[Tilt](https://tilt.dev/) is a tool for local development of Kubernetes applications. Install `tilt`:
25+
26+
```bash
27+
brew install tilt
28+
```
29+
30+
### Configuring and Running Tilt
31+
32+
#### Tilt Settings
33+
There are settings for Tilt that can be configured using a `tilt-settings.json` file. The settings file is not checked
34+
into source control. A sample settings file is provided in `tilt-settings.sample.json`. To use the sample settings file,
35+
copy it to `tilt-settings.json`
36+
37+
By default, Tilt is configured to only allow connections to the following Kubernetes contexts:
38+
39+
- `docker-desktop`
40+
- `kind-kind`
41+
- `minikube`
42+
43+
Please add any additional contexts to the `allowedContexts` list in your `tilt-settings.json` file.
44+
45+
#### Running Tilt
46+
```bash
47+
tilt up
48+
```
49+
850

951
## Testing
1052

Tiltfile

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
# default values
2+
settings = {
3+
"allowedContexts": [
4+
"docker-desktop",
5+
"minikube",
6+
"kind-kind",
7+
],
8+
"installMinio": True,
9+
"installWandb": True,
10+
"wandbCRD": "default",
11+
}
12+
13+
# global settings
14+
settings.update(read_json(
15+
"tilt-settings.json",
16+
default = {},
17+
))
18+
19+
allow_k8s_contexts(settings.get("allowed_k8s_contexts"))
20+
21+
os.putenv('PATH', './bin:' + os.getenv('PATH'))
22+
23+
load('ext://restart_process', 'docker_build_with_restart')
24+
25+
DOCKERFILE = '''
26+
FROM registry.access.redhat.com/ubi9/ubi
27+
28+
ADD tilt_bin/manager /manager
29+
30+
ENV HELM_CACHE_HOME=/helm/.cache/helm
31+
ENV HELM_CONFIG_HOME=/helm/.config/helm
32+
ENV HELM_DATA_HOME=/helm/.local/share/helm
33+
'''
34+
DOMAIN = "wandb.com"
35+
GROUP = "apps"
36+
VERSION = "v1"
37+
KIND = "wandb"
38+
IMG = 'controller:latest'
39+
CONTROLLERGEN = 'rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases;'
40+
DISABLE_SECURITY_CONTEXT = True
41+
42+
def yaml():
43+
data = local('cd config/manager; kustomize edit set image controller=' + IMG + '; cd ../..; kustomize build config/manager')
44+
if DISABLE_SECURITY_CONTEXT:
45+
decoded = decode_yaml_stream(data)
46+
if decoded:
47+
for d in decoded:
48+
# Live update conflicts with SecurityContext, until a better solution, just remove it
49+
if d["kind"] == "Deployment":
50+
if "securityContext" in d['spec']['template']['spec']:
51+
d['spec']['template']['spec'].pop('securityContext')
52+
for c in d['spec']['template']['spec']['containers']:
53+
if "securityContext" in c:
54+
c.pop('securityContext')
55+
56+
return encode_yaml_stream(decoded)
57+
return data
58+
59+
def manifests():
60+
return 'controller-gen ' + CONTROLLERGEN
61+
62+
def generate():
63+
return 'controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./...";'
64+
65+
def vetfmt():
66+
return 'go vet ./...; go fmt ./...'
67+
68+
# build to tilt_bin beause kubebuilder has a dockerignore for bin/
69+
def binary():
70+
return 'CGO_ENABLED=0 GOOS=linux GO111MODULE=on go build -o tilt_bin/manager main.go'
71+
72+
installed = local("which kubebuilder")
73+
print("kubebuilder is present:", installed)
74+
75+
DIRNAME = os.path.basename(os. getcwd())
76+
77+
local(manifests() + generate())
78+
79+
if settings.get("installMinio"):
80+
local_resource('Minio', 'kubectl apply -f ./hack/testing-manifests/minio/minio.yaml')
81+
82+
local_resource('CRD', manifests() + 'kustomize build config/crd | kubectl apply -f -', deps=["api"])
83+
84+
local_resource('RBAC', 'kustomize build config/rbac | kubectl apply -f -', deps=["config/rbac"])
85+
86+
k8s_yaml(yaml())
87+
88+
deps = ['controllers', 'pkg', 'main.go']
89+
deps.append('api')
90+
91+
local_resource('Watch&Compile', generate() + binary(), deps=deps, ignore=['*/*/zz_generated.deepcopy.go'])
92+
93+
if settings.get("installWandb"):
94+
local_resource('Sample YAML', 'kubectl apply -f ./hack/testing-manifests/wandb/' + settings.get('wandbCRD') + '.yaml', deps=["./hack/testing-manifests/wandb/default.yaml"], resource_deps=["controller-manager"])
95+
96+
docker_build_with_restart(IMG, '.',
97+
dockerfile_contents=DOCKERFILE,
98+
entrypoint='/manager',
99+
only=['./tilt_bin/manager'],
100+
live_update=[
101+
sync('./tilt_bin/manager', '/manager'),
102+
]
103+
)

config/manager/kustomization.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
44
kind: Kustomization
55
images:
66
- name: controller
7-
newName: wandb/controller
7+
newName: controller
88
newTag: latest

config/rbac/role.yaml

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ kind: ClusterRole
44
metadata:
55
name: manager-role
66
rules:
7+
- nonResourceURLs:
8+
- /metrics
9+
verbs:
10+
- get
711
- apiGroups:
812
- ""
913
resources:
@@ -18,20 +22,31 @@ rules:
1822
- delete
1923
- get
2024
- list
25+
- patch
2126
- update
2227
- watch
2328
- apiGroups:
2429
- ""
2530
resources:
31+
- endpoints
32+
- ingresses
2633
- namespaces
34+
- namespaces/status
2735
- nodes
36+
- nodes/metrics
37+
- nodes/proxy
38+
- nodes/spec
39+
- nodes/stats
2840
- pods
2941
- pods/log
30-
- serviceaccounts
31-
- services
42+
- pods/status
43+
- replicationcontrollers
44+
- replicationcontrollers/status
45+
- resourcequotas
3246
verbs:
3347
- get
3448
- list
49+
- watch
3550
- apiGroups:
3651
- apps
3752
resources:
@@ -45,6 +60,7 @@ rules:
4560
- delete
4661
- get
4762
- list
63+
- patch
4864
- update
4965
- watch
5066
- apiGroups:
@@ -91,29 +107,87 @@ rules:
91107
- delete
92108
- get
93109
- list
110+
- patch
94111
- update
95112
- watch
113+
- apiGroups:
114+
- batch
115+
resources:
116+
- cronjobs
117+
- jobs
118+
verbs:
119+
- get
120+
- list
121+
- watch
122+
- apiGroups:
123+
- cloud.google.com
124+
resources:
125+
- backendconfigs
126+
verbs:
127+
- create
128+
- delete
129+
- get
130+
- list
131+
- patch
132+
- update
133+
- watch
134+
- apiGroups:
135+
- events.k8s.io
136+
resources:
137+
- events
138+
verbs:
139+
- list
140+
- watch
141+
- apiGroups:
142+
- extensions
143+
resources:
144+
- daemonsets
145+
- deployments
146+
- ingresses
147+
- ingresses/status
148+
- replicasets
149+
verbs:
150+
- get
151+
- list
152+
- watch
96153
- apiGroups:
97154
- networking.k8s.io
98155
resources:
99156
- ingresses
157+
- ingresses/status
100158
- networkpolicies
101159
verbs:
102160
- create
103161
- delete
104162
- get
105163
- list
164+
- patch
165+
- update
166+
- watch
167+
- apiGroups:
168+
- policy
169+
resources:
170+
- poddisruptionbudgets
171+
verbs:
172+
- create
173+
- delete
174+
- get
175+
- list
176+
- patch
106177
- update
107178
- watch
108179
- apiGroups:
109180
- rbac.authorization.k8s.io
110181
resources:
182+
- clusterrolebindings
183+
- clusterroles
111184
- rolebindings
112185
- roles
113186
verbs:
114187
- create
115188
- delete
116189
- get
117190
- list
191+
- patch
118192
- update
119193
- watch

config/samples/wandb.com_v1_weightsandbiases.yaml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
---
12
apiVersion: apps.wandb.com/v1
23
kind: WeightsAndBiases
34
metadata:
@@ -9,4 +10,7 @@ metadata:
910
app.kubernetes.io/created-by: operator
1011
name: weightsandbiases-sample
1112
spec:
12-
# TODO(user): Add fields here
13+
values:
14+
ingress:
15+
install: false
16+
create: false

controllers/weightsandbiases_controller.go

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,13 +64,21 @@ type WeightsAndBiasesReconciler struct {
6464
//+kubebuilder:rbac:groups=apps.wandb.com,resources=weightsandbiases,verbs=get;list;watch;create;update;patch;delete
6565
//+kubebuilder:rbac:groups=apps.wandb.com,resources=weightsandbiases/status,verbs=get;update;patch
6666
//+kubebuilder:rbac:groups=apps.wandb.com,resources=weightsandbiases/finalizers,verbs=update
67-
//+kubebuilder:rbac:groups="",resources=configmaps;events;persistentvolumeclaims;secrets;serviceaccounts;services,verbs=update;delete;get;list;create;watch
68-
//+kubebuilder:rbac:groups="",resources=nodes;namespaces;pods;pods/log;serviceaccounts;services,verbs=get;list
69-
//+kubebuilder:rbac:groups=apps,resources=deployments;controllerrevisions;daemonsets;replicasets;statefulsets,verbs=update;delete;get;list;create;watch
67+
//+kubebuilder:rbac:groups="",resources=configmaps;events;persistentvolumeclaims;secrets;serviceaccounts;services,verbs=update;delete;get;list;create;patch;watch
68+
//+kubebuilder:rbac:groups="",resources=endpoints;ingresses;nodes;nodes/spec;nodes/stats;nodes/metrics;nodes/proxy;namespaces;namespaces/status;replicationcontrollers;replicationcontrollers/status;resourcequotas;pods;pods/log;pods/status,verbs=get;list;watch
69+
//+kubebuilder:rbac:groups=apps,resources=deployments;controllerrevisions;daemonsets;replicasets;statefulsets,verbs=update;delete;get;list;create;patch;watch
7070
//+kubebuilder:rbac:groups=apps,resources=deployments/status;daemonsets/status;replicasets/status;statefulsets/status,verbs=get
71-
//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=update;delete;get;list;create;watch
72-
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses;networkpolicies,verbs=update;delete;get;list;create;watch
73-
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=update;delete;get;list;create;watch
71+
//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=update;delete;get;list;patch;create;watch
72+
//+kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;watch
73+
//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=list;watch
74+
//+kubebuilder:rbac:groups=cloud.google.com,resources=backendconfigs,verbs=update;delete;get;list;patch;create;watch
75+
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses;ingresses/status;networkpolicies,verbs=update;delete;get;list;create;patch;watch
76+
//+kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=update;delete;get;list;patch;create;watch
77+
//+kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings;clusterroles;clusterrolebindings,verbs=update;delete;get;list;patch;create;watch
78+
//+kubebuilder:rbac:urls=/metrics,verbs=get
79+
80+
// Deprecated/Erroneously required RBAC rules
81+
//+kubebuilder:rbac:groups=extensions,resources=daemonsets;deployments;replicasets;ingresses;ingresses/status,verbs=get;list;watch
7482

7583
// Reconcile is part of the main kubernetes reconciliation loop which aims to
7684
// move the current state of the cluster closer to the desired state.

0 commit comments

Comments
 (0)