Skip to content

Commit 441a705

Browse files
gman0gman0
committed
Split driver and automount; added cmd/automount-runner
automount process now runs inside its own container; it responsible for autofs and CVMFS mounts instead of the nodeplugin container, which is now just lean CSI server.
1 parent b929bff commit 441a705

File tree

17 files changed

+518
-266
lines changed

17 files changed

+518
-266
lines changed

‎Makefile‎

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,9 @@
1616
# Based on the helm Makefile:
1717
# https://github.com/helm/helm/blob/master/Makefile
1818

19-
BINDIR := $(CURDIR)/bin
20-
DIST_DIRS := find * -type d -exec
21-
TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64le windows/amd64
22-
BINNAME ?= csi-cvmfsplugin
19+
BINDIR := $(CURDIR)/bin
20+
DIST_DIRS := find * -type d -exec
21+
TARGETS ?= darwin/amd64 linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64le windows/amd64
2322
IMAGE_BUILD_TOOL ?= docker
2423

2524
GOPATH = $(shell go env GOPATH)
@@ -69,10 +68,13 @@ all: build
6968
# build
7069

7170
.PHONY: build
72-
build: $(BINDIR)/$(BINNAME)
71+
build: $(BINDIR)/csi-cvmfsplugin $(BINDIR)/automount-runner
7372

74-
$(BINDIR)/$(BINNAME): $(SRC)
75-
go build $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $(BINDIR)/$(BINNAME) ./cmd/csi-cvmfsplugin
73+
$(BINDIR)/csi-cvmfsplugin: $(SRC)
74+
go build $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $@ ./cmd/csi-cvmfsplugin
75+
76+
$(BINDIR)/automount-runner: $(SRC)
77+
go build $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' -o $@ ./cmd/automount-runner
7678

7779
# ------------------------------------------------------------------------------
7880
# test
@@ -123,7 +125,8 @@ $(GOIMPORTS):
123125
.PHONY: build-cross
124126
build-cross: LDFLAGS += -extldflags "-static"
125127
build-cross: $(GOX)
126-
CGO_ENABLED=0 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/$(BINNAME)_{{.OS}}_{{.Arch}}" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' ./cmd/csi-cvmfsplugin
128+
CGO_ENABLED=0 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/csi-cvmfsplugin_{{.OS}}_{{.Arch}}" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' ./cmd/csi-cvmfsplugin
129+
CGO_ENABLED=0 $(GOX) -parallel=3 -output="_dist/{{.OS}}-{{.Arch}}/automount-runner_{{.OS}}_{{.Arch}}" -osarch='$(TARGETS)' $(GOFLAGS) -tags '$(TAGS)' -ldflags '$(LDFLAGS)' ./cmd/automount-runner
127130

128131
.PHONY: dist
129132
dist:
@@ -156,6 +159,7 @@ endif
156159
image: build-cross
157160
mkdir -p bin
158161
cp _dist/linux-amd64/csi-cvmfsplugin_linux_amd64 bin/csi-cvmfsplugin
162+
cp _dist/linux-amd64/automount-runner_linux_amd64 bin/automount-runner
159163
sudo $(IMAGE_BUILD_TOOL) build -t cvmfs-csi:${DOCKER_TAG} -f deployments/docker/Dockerfile .
160164

161165
# ------------------------------------------------------------------------------

‎cmd/automount-runner/main.go‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
// Copyright CERN.
2+
//
3+
//
4+
// Licensed under the Apache License, Version 2.0 (the "License");
5+
// you may not use this file except in compliance with the License.
6+
// You may obtain a copy of the License at
7+
//
8+
// http://www.apache.org/licenses/LICENSE-2.0
9+
//
10+
// Unless required by applicable law or agreed to in writing, software
11+
// distributed under the License is distributed on an "AS IS" BASIS,
12+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
// See the License for the specific language governing permissions and
14+
// limitations under the License.
15+
//
16+
17+
package main
18+
19+
import (
20+
"flag"
21+
"fmt"
22+
"os"
23+
24+
"github.com/cernops/cvmfs-csi/internal/cvmfs/automount"
25+
"github.com/cernops/cvmfs-csi/internal/log"
26+
cvmfsversion "github.com/cernops/cvmfs-csi/internal/version"
27+
28+
"k8s.io/klog/v2"
29+
)
30+
31+
var (
32+
version = flag.Bool("version", false, "Print driver version and exit.")
33+
34+
hasAlienCache = flag.Bool("has-alien-cache", false, "CVMFS client is using alien cache volume")
35+
36+
automountDaemonUnmountAfterIdleSeconds = flag.Int("automount-unmount-timeout", 300, "number of seconds of idle time after which an autofs-managed CVMFS mount will be unmounted. '0' means never unmount, '-1' leaves automount default option.")
37+
)
38+
39+
func main() {
40+
// Handle flags and initialize logging.
41+
42+
klog.InitFlags(nil)
43+
if err := flag.Set("logtostderr", "true"); err != nil {
44+
klog.Exitf("failed to set logtostderr flag: %v", err)
45+
}
46+
flag.Parse()
47+
48+
if *version {
49+
fmt.Println("automount-runner for CVMFS CSI plugin version", cvmfsversion.FullVersion())
50+
os.Exit(0)
51+
}
52+
53+
// Initialize and run automount-runner.
54+
55+
log.Infof("automount-runner for CVMFS CSI plugin version %s", cvmfsversion.FullVersion())
56+
log.Infof("Command line arguments %v", os.Args)
57+
58+
err := automount.Init(&automount.Opts{
59+
UnmountTimeoutSeconds: *automountDaemonUnmountAfterIdleSeconds,
60+
HasAlienCache: *hasAlienCache,
61+
})
62+
63+
if err != nil {
64+
log.Fatalf("Failed to initialize automount-runner: %v", err)
65+
}
66+
67+
if err = automount.RunBlocking(); err != nil {
68+
log.Fatalf("Failed to run automount-runner: %v", err)
69+
}
70+
71+
os.Exit(0)
72+
}

‎cmd/csi-cvmfsplugin/main.go‎

Lines changed: 9 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -62,20 +62,13 @@ var (
6262
version = flag.Bool("version", false, "Print driver version and exit.")
6363
roles rolesFlag
6464

65-
hasAlienCache = flag.Bool("has-alien-cache", false, "CVMFS client is using alien cache volume")
66-
startAutomountDaemon = flag.Bool("start-automount-daemon", true, "start automount daemon when initializing CVMFS CSI driver")
65+
hasAlienCache = flag.Bool("has-alien-cache", false, "(DEPRECATED: use automount-runner --has-alien-cache) CVMFS client is using alien cache volume")
66+
startAutomountDaemon = flag.Bool("start-automount-daemon", true, "(DEPRECATED: use automount-runner) start automount daemon when initializing CVMFS CSI driver")
6767

68-
automountDaemonStartupTimeoutSeconds = flag.Int("automount-startup-timeout", 5, "number of seconds to wait for automount daemon to start up before exiting")
69-
automountDaemonUnmountAfterIdleSeconds = flag.Int("automount-unmount-timeout", -1, "number of seconds of idle time after which an autofs-managed CVMFS mount will be unmounted. '0' means never unmount, '-1' leaves automount default option.")
68+
automountDaemonStartupTimeoutSeconds = flag.Int("automount-startup-timeout", 10, "number of seconds to wait for automount daemon to start up before giving up and exiting. '0' means wait forever")
69+
automountDaemonUnmountAfterIdleSeconds = flag.Int("automount-unmount-timeout", 300, "(DEPRECATED: use automount-runner --unmount-timeout) number of seconds of idle time after which an autofs-managed CVMFS mount will be unmounted. '0' means never unmount, '-1' leaves automount default option.")
7070
)
7171

72-
func printVersion() {
73-
fmt.Printf(
74-
"CVMFS CSI plugin version %s (commit: %s; build time: %s; metadata: %s)\n",
75-
cvmfsversion.Version(), cvmfsversion.Commit(), cvmfsversion.BuildTime(), cvmfsversion.Metadata(),
76-
)
77-
}
78-
7972
func main() {
8073
// Handle flags and initialize logging.
8174

@@ -88,14 +81,15 @@ func main() {
8881
flag.Parse()
8982

9083
if *version {
91-
printVersion()
84+
fmt.Println("CVMFS CSI plugin version", cvmfsversion.FullVersion())
9285
os.Exit(0)
9386
}
9487

95-
log.Infof("Running CVMFS CSI plugin with %v", os.Args)
96-
9788
// Initialize and run the driver.
9889

90+
log.Infof("CVMFS CSI plugin version %s", cvmfsversion.FullVersion())
91+
log.Infof("Command line arguments %v", os.Args)
92+
9993
driverRoles := make(map[driver.ServiceRole]bool, len(roles))
10094
for _, role := range roles {
10195
driverRoles[role] = true
@@ -107,11 +101,7 @@ func main() {
107101
NodeID: *nodeId,
108102
Roles: driverRoles,
109103

110-
StartAutomountDaemon: *startAutomountDaemon,
111-
HasAlienCache: *hasAlienCache,
112-
113-
AutomountDaemonStartupTimeoutSeconds: *automountDaemonStartupTimeoutSeconds,
114-
AutomountDaemonUnmountAfterIdleSeconds: *automountDaemonUnmountAfterIdleSeconds,
104+
AutomountDaemonStartupTimeoutSeconds: *automountDaemonStartupTimeoutSeconds,
115105
})
116106

117107
if err != nil {

‎deployments/docker/Dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ RUN yum install -y http://ecsft.cern.ch/dist/cvmfs/cvmfs-release/cvmfs-release-l
66
yum install -y cvmfs && yum clean all && rm -rf /var/cache/yum
77

88
COPY bin/csi-cvmfsplugin /csi-cvmfsplugin
9-
RUN chmod +x /csi-cvmfsplugin
10-
ENTRYPOINT ["/csi-cvmfsplugin"]
9+
COPY bin/automount-runner /automount-runner
10+
11+
RUN chmod +x /csi-cvmfsplugin /automount-runner

‎deployments/helm/README.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,14 @@ Alternatively, a YAML file that specifies the values of the parameters can be pr
6262
| `nodeplugin.plugin.image.tag` | Container image tag for CVMFS CSI node plugin. |
6363
| `nodeplugin.plugin.image.pullPolicy` | Pull policy for CVMFS CSI node plugin image. |
6464
| `nodeplugin.plugin.image.resources` | Resource constraints for the `nodeplugin` container. |
65+
| `nodeplugin.automount.image.repository` | Container image repository for CVMFS CSI node plugin (automount-runner). |
66+
| `nodeplugin.automount.image.tag` | Container image tag for CVMFS CSI node plugin (automount-runner). |
67+
| `nodeplugin.automount.image.pullPolicy` | Pull policy for CVMFS CSI node plugin image (automount-runner). |
68+
| `nodeplugin.automount.image.resources` | Resource constraints for the `automount` container. |
69+
| `nodeplugin.registrar.image.repository` | Container image repository for csi-node-driver-registrar. |
70+
| `nodeplugin.registrar.image.tag` | Container image tag for csi-node-driver-registrar. |
71+
| `nodeplugin.registrar.image.pullPolicy` | Pull policy for csi-node-driver-registrar image. |
72+
| `nodeplugin.registrar.image.resources` | Resource constraints for the `registrar` container. |
6573
| `nodeplugin.registrar.image.repository` | Container image repository for csi-node-driver-registrar. |
6674
| `nodeplugin.registrar.image.tag` | Container image tag for csi-node-driver-registrar. |
6775
| `nodeplugin.registrar.image.pullPolicy` | Pull policy for csi-node-driver-registrar image. |

‎deployments/helm/cvmfs-csi/templates/controllerplugin-deployment.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ spec:
3636
- name: controllerplugin
3737
image: {{ .Values.controllerplugin.plugin.image.repository }}:{{ .Values.controllerplugin.plugin.image.tag }}
3838
imagePullPolicy: {{ .Values.controllerplugin.plugin.image.pullPolicy }}
39+
command: [/csi-cvmfsplugin]
3940
args:
4041
- -v={{ .Values.logVerbosityLevel }}
4142
- --nodeid=$(NODE_ID)

‎deployments/helm/cvmfs-csi/templates/nodeplugin-daemonset.yaml‎

Lines changed: 45 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,21 +42,14 @@ spec:
4242
{{- end }}
4343
- name: nodeplugin
4444
image: {{ .Values.nodeplugin.plugin.image.repository }}:{{ .Values.nodeplugin.plugin.image.tag }}
45-
command:
46-
- /bin/bash
47-
- -c
48-
- |
49-
ulimit -n {{ .Values._rlimit_nofile }}
50-
/csi-cvmfsplugin \
51-
-v={{ .Values.logVerbosityLevel }} \
52-
--nodeid=$(NODE_ID) \
53-
--endpoint=$(CSI_ENDPOINT) \
54-
--drivername=$(CSI_DRIVERNAME) \
55-
--start-automount-daemon={{ .Values.startAutomountDaemon }} \
56-
--automount-startup-timeout={{ .Values.automountDaemonStartupTimeout }} \
57-
--automount-unmount-timeout={{ .Values.automountDaemonUnmountTimeout }} \
58-
--role=identity,node \
59-
--has-alien-cache={{ .Values.cache.alien.enabled }}
45+
command: [/csi-cvmfsplugin]
46+
args:
47+
- -v={{ .Values.logVerbosityLevel }}
48+
- --nodeid=$(NODE_ID)
49+
- --endpoint=$(CSI_ENDPOINT)
50+
- --drivername=$(CSI_DRIVERNAME)
51+
- --role=identity,node
52+
- --automount-startup-timeout={{ .Values.automountDaemonStartupTimeout }}
6053
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
6154
securityContext:
6255
privileged: true
@@ -85,13 +78,47 @@ spec:
8578
readOnly: true
8679
- name: host-dev
8780
mountPath: /dev
81+
- name: autofs-cvmfs
82+
mountPath: /cvmfs
83+
mountPropagation: Bidirectional
84+
{{- with .Values.nodeplugin.plugin.resources }}
85+
resources: {{ toYaml . | nindent 12 }}
86+
{{- end }}
87+
- name: automount
88+
image: {{ .Values.nodeplugin.automount.image.repository }}:{{ .Values.nodeplugin.automount.image.tag }}
89+
command:
90+
- /bin/bash
91+
- -c
92+
- |
93+
ulimit -n {{ .Values._rlimit_nofile }}
94+
/automount-runner \
95+
-v={{ .Values.logVerbosityLevel }} \
96+
--unmount-timeout={{ .Values.automountDaemonUnmountTimeout }} \
97+
--has-alien-cache={{ .Values.cache.alien.enabled }}
98+
imagePullPolicy: {{ .Values.nodeplugin.plugin.image.pullPolicy }}
99+
securityContext:
100+
privileged: true
101+
capabilities:
102+
add: ["SYS_ADMIN"]
103+
allowPrivilegeEscalation: true
104+
volumeMounts:
105+
- mountPath: /sys
106+
name: host-sys
107+
- name: lib-modules
108+
mountPath: /lib/modules
109+
readOnly: true
110+
- name: host-dev
111+
mountPath: /dev
112+
- name: autofs-cvmfs
113+
mountPath: /cvmfs
114+
mountPropagation: Bidirectional
88115
- name: cvmfs-localcache
89116
mountPath: /cvmfs-localcache
90117
{{- if .Values.cache.alien.enabled }}
91118
- name: cvmfs-aliencache
92119
mountPath: /cvmfs-aliencache
93120
{{- end }}
94-
{{- with .Values.nodeplugin.plugin.extraVolumeMounts }}
121+
{{- with .Values.nodeplugin.automount.extraVolumeMounts }}
95122
{{- toYaml . | nindent 12 }}
96123
{{- end }}
97124
{{- with .Values.nodeplugin.plugin.resources }}
@@ -119,6 +146,8 @@ spec:
119146
- name: host-dev
120147
hostPath:
121148
path: /dev
149+
- name: autofs-cvmfs
150+
emptyDir: {}
122151
- name: cvmfs-localcache
123152
{{- toYaml .Values.cache.local.volumeSpec | nindent 10 }}
124153
{{- if .Values.cache.alien.enabled }}

‎deployments/helm/cvmfs-csi/values.yaml‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,15 @@ nodeplugin:
6363
plugin:
6464
image:
6565
repository: registry.cern.ch/magnum/cvmfs-csi
66-
tag: v2.1.1
66+
tag: latest
67+
pullPolicy: IfNotPresent
68+
resources: {}
69+
70+
# CVMFS CSI image and container resources specs.
71+
automount:
72+
image:
73+
repository: registry.cern.ch/magnum/cvmfs-csi
74+
tag: latest
6775
pullPolicy: IfNotPresent
6876
resources: {}
6977
# Extra volume mounts to append to nodeplugin's
@@ -142,7 +150,7 @@ controllerplugin:
142150
plugin:
143151
image:
144152
repository: registry.cern.ch/magnum/cvmfs-csi
145-
tag: v2.1.1
153+
tag: latest
146154
pullPolicy: IfNotPresent
147155
resources: {}
148156
extraVolumeMounts: []
@@ -208,17 +216,11 @@ kubeletDirectory: /var/lib/kubelet
208216
# <kubeletPluginDirectory>/plugins/<csiDriverName>/<cvmfsCSIPluginSocketFile>.
209217
cvmfsCSIPluginSocketFile: csi.sock
210218

211-
# Whether CVMFS CSI nodeplugin Pod should run automount daemon. This is required
212-
# for automounts to work. If however worker nodes are already running automount
213-
# daemon (e.g. as a systemd service), you may disable running yet another instance
214-
# of the daemon using this switch.
215-
startAutomountDaemon: true
216-
217219
# Number of seconds to wait for automount daemon to start up before exiting.
218-
automountDaemonStartupTimeout: 5
220+
automountDaemonStartupTimeout: 10
219221
# Number of seconds of idle time after which an autofs-managed CVMFS mount will
220222
# be unmounted. '0' means never unmount, '-1' leaves automount default option.
221-
automountDaemonUnmountTimeout: -1
223+
automountDaemonUnmountTimeout: 300
222224

223225
# Chart name overrides.
224226
nameOverride: ""
@@ -232,4 +234,4 @@ extraMetaLabels: {}
232234
# See https://github.com/cvmfs-contrib/cvmfs-csi/issues/57 for details.
233235
# NOTE: this value will be deprecated once the issue is fixed.
234236
# Empty value "" disables the workaround.
235-
_rlimit_nofile: ""
237+
_rlimit_nofile: "1000000"

‎deployments/kubernetes/controllerplugin-deployment.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ spec:
3535
- name: socket-dir
3636
mountPath: /csi
3737
- name: controllerplugin
38-
image: registry.cern.ch/magnum/cvmfs-csi:v2.1.1
38+
image: registry.cern.ch/magnum/cvmfs-csi:latest
3939
imagePullPolicy: IfNotPresent
40+
command: [/csi-cvmfsplugin]
4041
args:
4142
- -v=5
4243
- --nodeid=$(NODE_ID)

‎deployments/kubernetes/controllerplugin-rbac.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
apiVersion: v1
22
kind: ServiceAccount
33
metadata:
4-
name: cvmfs-controllerplugin
4+
name: cvmfs-csi-controllerplugin
55
labels:
66
app: cvmfs-csi
77
component: controllerplugin
@@ -45,7 +45,7 @@ metadata:
4545
component: controllerplugin
4646
subjects:
4747
- kind: ServiceAccount
48-
name: cvmfs-controllerplugin
48+
name: cvmfs-csi-controllerplugin
4949
namespace: default
5050
roleRef:
5151
kind: ClusterRole

0 commit comments

Comments
 (0)