From 4d3029fd80fcf032fcd7bf678febaafcc3f0dfdb Mon Sep 17 00:00:00 2001 From: Praveen Kumar Date: Tue, 3 Sep 2024 12:54:42 +0530 Subject: [PATCH 1/5] Add flannel resource to optional assets --- assets/optional/flannel/00-namespace.yaml | 15 +++ .../optional/flannel/01-service-account.yaml | 5 + assets/optional/flannel/02-cluster-role.yaml | 116 +++++++++++++++++ .../flannel/03-cluster-role-binding.yaml | 12 ++ assets/optional/flannel/04-configmap.yaml | 38 ++++++ assets/optional/flannel/05-daemonset.yaml | 117 ++++++++++++++++++ .../flannel/kustomization.aarch64.yaml | 8 ++ .../flannel/kustomization.x86_64.yaml | 8 ++ assets/optional/flannel/kustomization.yaml | 9 ++ .../flannel/release-flannel-aarch64.json | 9 ++ .../flannel/release-flannel-x86_64.json | 9 ++ scripts/auto-rebase/assets.yaml | 16 +++ 12 files changed, 362 insertions(+) create mode 100644 assets/optional/flannel/00-namespace.yaml create mode 100644 assets/optional/flannel/01-service-account.yaml create mode 100644 assets/optional/flannel/02-cluster-role.yaml create mode 100644 assets/optional/flannel/03-cluster-role-binding.yaml create mode 100644 assets/optional/flannel/04-configmap.yaml create mode 100644 assets/optional/flannel/05-daemonset.yaml create mode 100644 assets/optional/flannel/kustomization.aarch64.yaml create mode 100644 assets/optional/flannel/kustomization.x86_64.yaml create mode 100644 assets/optional/flannel/kustomization.yaml create mode 100644 assets/optional/flannel/release-flannel-aarch64.json create mode 100644 assets/optional/flannel/release-flannel-x86_64.json diff --git a/assets/optional/flannel/00-namespace.yaml b/assets/optional/flannel/00-namespace.yaml new file mode 100644 index 0000000000..98e3ef029f --- /dev/null +++ b/assets/optional/flannel/00-namespace.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-flannel + labels: + name: kube-flannel + openshift.io/run-level: "0" + openshift.io/cluster-monitoring: "true" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + openshift.io/node-selector: "" + openshift.io/description: "flannel Kubernetes components" + workload.openshift.io/allowed: "management" diff --git a/assets/optional/flannel/01-service-account.yaml b/assets/optional/flannel/01-service-account.yaml new file mode 100644 index 0000000000..98ecb76d31 --- /dev/null +++ b/assets/optional/flannel/01-service-account.yaml @@ -0,0 +1,5 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flannel + namespace: kube-flannel \ No newline at end of file diff --git a/assets/optional/flannel/02-cluster-role.yaml b/assets/optional/flannel/02-cluster-role.yaml new file mode 100644 index 0000000000..a47a3f6ab2 --- /dev/null +++ b/assets/optional/flannel/02-cluster-role.yaml @@ -0,0 +1,116 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flannel +rules: + - apiGroups: [""] + resources: + - namespaces + - nodes + - pods + verbs: + - get + - list + - patch + - watch + - update + - apiGroups: [""] + resources: + - pods + verbs: + - get + - list + - patch + - watch + - delete + - apiGroups: [""] + resources: + - configmaps + verbs: + - get + - create + - update + - patch + - apiGroups: [""] + resources: + - services + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch + - apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: + - get + - list + - watch + - apiGroups: ["", "events.k8s.io"] + resources: + - events + verbs: + - create + - patch + - update + - apiGroups: ["security.openshift.io"] + resources: + - securitycontextconstraints + verbs: + - use + resourceNames: + - privileged + - apiGroups: [""] + resources: + - "nodes/status" + verbs: + - patch + - update + - apiGroups: ["k8s.ovn.org"] + resources: + - egressfirewalls + - egressips + - egressqoses + - adminpolicybasedexternalroutes + verbs: + - get + - list + - watch + - update + - patch + - apiGroups: + - k8s.ovn.org + resources: + - adminpolicybasedexternalroutes/status + verbs: [ "update"] + - apiGroups: ["cloud.network.openshift.io"] + resources: + - cloudprivateipconfigs + verbs: + - create + - patch + - update + - delete + - get + - list + - watch + - apiGroups: ["apiextensions.k8s.io"] + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - apiGroups: ['authentication.k8s.io'] + resources: ['tokenreviews'] + verbs: ['create'] + - apiGroups: ['authorization.k8s.io'] + resources: ['subjectaccessreviews'] + verbs: ['create'] \ No newline at end of file diff --git a/assets/optional/flannel/03-cluster-role-binding.yaml b/assets/optional/flannel/03-cluster-role-binding.yaml new file mode 100644 index 0000000000..a484da31a4 --- /dev/null +++ b/assets/optional/flannel/03-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flannel +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flannel +subjects: + - kind: ServiceAccount + name: flannel + namespace: kube-flannel \ No newline at end of file diff --git a/assets/optional/flannel/04-configmap.yaml b/assets/optional/flannel/04-configmap.yaml new file mode 100644 index 0000000000..30708ed449 --- /dev/null +++ b/assets/optional/flannel/04-configmap.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +data: + cni-conf.json: | + { + "name": "ovn-kubernetes", + "cniVersion": "0.4.0", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] + } + net-conf.json: | + { + "Network": "10.42.0.0/24", + "EnableNFTables": false, + "Backend": { + "Type": "vxlan" + } + } +kind: ConfigMap +metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + name: kube-flannel-cfg + namespace: kube-flannel \ No newline at end of file diff --git a/assets/optional/flannel/05-daemonset.yaml b/assets/optional/flannel/05-daemonset.yaml new file mode 100644 index 0000000000..39c0de8378 --- /dev/null +++ b/assets/optional/flannel/05-daemonset.yaml @@ -0,0 +1,117 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + name: kube-flannel-ds + namespace: kube-flannel +spec: + selector: + matchLabels: + app: flannel + k8s-app: flannel + template: + metadata: + labels: + app: flannel + k8s-app: flannel + tier: node + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + containers: + - args: + - --ip-masq + - --kube-subnet-mgr + command: + - /opt/bin/flanneld + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: EVENT_QUEUE_DEPTH + value: "5000" + image: flannel + imagePullPolicy: IfNotPresent + name: kube-flannel + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + capabilities: + add: + - NET_ADMIN + - NET_RAW + privileged: false + volumeMounts: + - mountPath: /run/flannel + name: run + - mountPath: /etc/kube-flannel/ + name: flannel-cfg + - mountPath: /run/xtables.lock + name: xtables-lock + hostNetwork: true + initContainers: + - args: + - -f + - /flannel + - /usr/libexec/cni + command: + - cp + image: flannel-plugin + imagePullPolicy: IfNotPresent + name: install-cni-plugin + volumeMounts: + - mountPath: /usr/libexec/cni + name: cni-plugin + - args: + - -f + - /etc/kube-flannel/cni-conf.json + - /etc/cni/net.d/10-flannel.conflist + command: + - cp + image: flannel + imagePullPolicy: IfNotPresent + name: install-cni + volumeMounts: + - mountPath: /etc/cni/net.d + name: cni + - mountPath: /etc/kube-flannel/ + name: flannel-cfg + priorityClassName: system-node-critical + serviceAccountName: flannel + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - hostPath: + path: /run/flannel + name: run + - hostPath: + path: /usr/libexec/cni + name: cni-plugin + - hostPath: + path: /etc/cni/net.d + name: cni + - configMap: + name: kube-flannel-cfg + name: flannel-cfg + - hostPath: + path: /run/xtables.lock + type: FileOrCreate + name: xtables-lock \ No newline at end of file diff --git a/assets/optional/flannel/kustomization.aarch64.yaml b/assets/optional/flannel/kustomization.aarch64.yaml new file mode 100644 index 0000000000..781f3357aa --- /dev/null +++ b/assets/optional/flannel/kustomization.aarch64.yaml @@ -0,0 +1,8 @@ + +images: + - name: flannel + newName: docker.io/flannel/flannel + digest: sha256:16fc8a9bc02163d9c02056ec6ac649d47f6f9a5e3adb5c1f2e93b547dba1315f + - name: flannel-plugin + newName: docker.io/flannel/flannel-cni-plugin + digest: sha256:2a6ed38c6f14402aec3433ca0613a3a0e54993d3fa59124687dd94cb39fb59cb \ No newline at end of file diff --git a/assets/optional/flannel/kustomization.x86_64.yaml b/assets/optional/flannel/kustomization.x86_64.yaml new file mode 100644 index 0000000000..e0f1d5ce66 --- /dev/null +++ b/assets/optional/flannel/kustomization.x86_64.yaml @@ -0,0 +1,8 @@ + +images: + - name: flannel + newName: docker.io/flannel/flannel + digest: sha256:9e0e9170ed1f82029ccc4cf20fb20f6325456ab7a00f0544dc23275f11f8bb1c + - name: flannel-plugin + newName: docker.io/flannel/flannel-cni-plugin + digest: sha256:6e700d30d2c9316b4f54f9d0b1423a2d15290bf082f5d1fbd87f9f25173a374c \ No newline at end of file diff --git a/assets/optional/flannel/kustomization.yaml b/assets/optional/flannel/kustomization.yaml new file mode 100644 index 0000000000..67a36b8090 --- /dev/null +++ b/assets/optional/flannel/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 02-cluster-role.yaml + - 03-cluster-role-binding.yaml + - 04-configmap.yaml + - 05-daemonset.yaml diff --git a/assets/optional/flannel/release-flannel-aarch64.json b/assets/optional/flannel/release-flannel-aarch64.json new file mode 100644 index 0000000000..8f04e8f68c --- /dev/null +++ b/assets/optional/flannel/release-flannel-aarch64.json @@ -0,0 +1,9 @@ +{ + "release": { + "base": "4.18.0-0.nightly-arm64-2024-08-29-120159" + }, + "images": { + "flannel": "docker.io/flannel/flannel@sha256:16fc8a9bc02163d9c02056ec6ac649d47f6f9a5e3adb5c1f2e93b547dba1315f", + "flannel-plugin": "docker.io/flannel/flannel-cni-plugin@sha256:2a6ed38c6f14402aec3433ca0613a3a0e54993d3fa59124687dd94cb39fb59cb" + } +} diff --git a/assets/optional/flannel/release-flannel-x86_64.json b/assets/optional/flannel/release-flannel-x86_64.json new file mode 100644 index 0000000000..eb6df07684 --- /dev/null +++ b/assets/optional/flannel/release-flannel-x86_64.json @@ -0,0 +1,9 @@ +{ + "release": { + "base": "4.18.0-0.nightly-2024-08-29-020346" + }, + "images": { + "flannel": "docker.io/flannel/flannel@sha256:9e0e9170ed1f82029ccc4cf20fb20f6325456ab7a00f0544dc23275f11f8bb1c", + "flannel-plugin": "docker.io/flannel/flannel-cni-plugin@sha256:6e700d30d2c9316b4f54f9d0b1423a2d15290bf082f5d1fbd87f9f25173a374c" + } +} diff --git a/scripts/auto-rebase/assets.yaml b/scripts/auto-rebase/assets.yaml index 24a10e59f2..e941561696 100644 --- a/scripts/auto-rebase/assets.yaml +++ b/scripts/auto-rebase/assets.yaml @@ -270,3 +270,19 @@ assets: - file: 07-daemonset-dhcp.yaml - file: release-multus-aarch64.json - file: release-multus-x86_64.json + + - dir: optional/flannel/ + ignore: "they don't exist in upstream repository - only in microshift" + files: + - file: kustomization.yaml + - file: kustomization.aarch64.yaml + - file: kustomization.x86_64.yaml + - file: 00-namespace.yaml + - file: 01-service-account.yaml + - file: 02-cluster-role.yaml + - file: 03-cluster-role-binding.yaml + - file: 04-configmap.yaml + - file: 05-daemonset.yaml + - file: release-flannel-aarch64.json + - file: release-flannel-x86_64.json + From e20f60bf1a96d6bec6e9fcf1c6abef7af90ff797 Mon Sep 17 00:00:00 2001 From: Praveen Kumar Date: Tue, 3 Sep 2024 14:55:16 +0530 Subject: [PATCH 2/5] Add kube-proxy resource to assets/optional --- assets/optional/kube-proxy/00-namespace.yaml | 15 ++++++ .../kube-proxy/01-service-account.yaml | 7 +++ .../optional/kube-proxy/02-cluster-role.yaml | 23 ++++++++ .../kube-proxy/03-cluster-role-binding.yaml | 12 +++++ assets/optional/kube-proxy/04-configmap.yaml | 22 ++++++++ assets/optional/kube-proxy/05-daemonset.yaml | 52 +++++++++++++++++++ .../kube-proxy/kustomization.aarch64.yaml | 5 ++ .../kube-proxy/kustomization.x86_64.yaml | 5 ++ assets/optional/kube-proxy/kustomization.yaml | 9 ++++ .../release-kube-proxy-aarch64.json | 8 +++ .../kube-proxy/release-kube-proxy-x86_64.json | 8 +++ scripts/auto-rebase/assets.yaml | 15 ++++++ 12 files changed, 181 insertions(+) create mode 100644 assets/optional/kube-proxy/00-namespace.yaml create mode 100644 assets/optional/kube-proxy/01-service-account.yaml create mode 100644 assets/optional/kube-proxy/02-cluster-role.yaml create mode 100644 assets/optional/kube-proxy/03-cluster-role-binding.yaml create mode 100644 assets/optional/kube-proxy/04-configmap.yaml create mode 100644 assets/optional/kube-proxy/05-daemonset.yaml create mode 100644 assets/optional/kube-proxy/kustomization.aarch64.yaml create mode 100644 assets/optional/kube-proxy/kustomization.x86_64.yaml create mode 100644 assets/optional/kube-proxy/kustomization.yaml create mode 100644 assets/optional/kube-proxy/release-kube-proxy-aarch64.json create mode 100644 assets/optional/kube-proxy/release-kube-proxy-x86_64.json diff --git a/assets/optional/kube-proxy/00-namespace.yaml b/assets/optional/kube-proxy/00-namespace.yaml new file mode 100644 index 0000000000..99263ff302 --- /dev/null +++ b/assets/optional/kube-proxy/00-namespace.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kube-proxy + labels: + name: kube-flannel + openshift.io/run-level: "0" + openshift.io/cluster-monitoring: "true" + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged + annotations: + openshift.io/node-selector: "" + openshift.io/description: "kube-proxy Kubernetes components" + workload.openshift.io/allowed: "management" \ No newline at end of file diff --git a/assets/optional/kube-proxy/01-service-account.yaml b/assets/optional/kube-proxy/01-service-account.yaml new file mode 100644 index 0000000000..9b118afada --- /dev/null +++ b/assets/optional/kube-proxy/01-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/assets/optional/kube-proxy/02-cluster-role.yaml b/assets/optional/kube-proxy/02-cluster-role.yaml new file mode 100644 index 0000000000..ae9c2bb2fc --- /dev/null +++ b/assets/optional/kube-proxy/02-cluster-role.yaml @@ -0,0 +1,23 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: system:kube-proxy +rules: + - apiGroups: + - "" + resources: + - services + - endpoints + - nodes + - configmaps + verbs: + - list + - watch + - get + - apiGroups: ["discovery.k8s.io"] + resources: + - endpointslices + verbs: + - list + - watch + - get \ No newline at end of file diff --git a/assets/optional/kube-proxy/03-cluster-role-binding.yaml b/assets/optional/kube-proxy/03-cluster-role-binding.yaml new file mode 100644 index 0000000000..2edc2f198c --- /dev/null +++ b/assets/optional/kube-proxy/03-cluster-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: system:kube-proxy +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:kube-proxy +subjects: + - kind: ServiceAccount + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/assets/optional/kube-proxy/04-configmap.yaml b/assets/optional/kube-proxy/04-configmap.yaml new file mode 100644 index 0000000000..0ef2a1b683 --- /dev/null +++ b/assets/optional/kube-proxy/04-configmap.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +data: + config.conf: | + apiVersion: kubeproxy.config.k8s.io/v1alpha1 + kind: KubeProxyConfiguration + clusterCIDR: 10.42.0.0/16 + mode: iptables + clientConnection: + kubeconfig: /var/lib/kubeconfig + iptables: + masqueradeAll: true + conntrack: + maxPerCore: 0 + featureGates: + AllAlpha: false +kind: ConfigMap +metadata: + labels: + app: kube-proxy + k8s-app: kube-proxy + name: kube-proxy + namespace: kube-proxy \ No newline at end of file diff --git a/assets/optional/kube-proxy/05-daemonset.yaml b/assets/optional/kube-proxy/05-daemonset.yaml new file mode 100644 index 0000000000..cf8bdaa0e1 --- /dev/null +++ b/assets/optional/kube-proxy/05-daemonset.yaml @@ -0,0 +1,52 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-proxy + namespace: kube-proxy +spec: + selector: + matchLabels: + k8s-app: kube-proxy + template: + metadata: + labels: + k8s-app: kube-proxy + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: In + values: + - linux + serviceAccountName: kube-proxy # Reference the Service Account here + containers: + - name: kube-proxy + image: kube-proxy + command: + - /usr/bin/kube-proxy + - --config=/var/lib/kube-proxy/config.conf + volumeMounts: + - name: config + mountPath: /var/lib/kube-proxy/ + readOnly: true + - name: kubeconfig + mountPath: /var/lib/kubeconfig + readOnly: true + securityContext: + privileged: true + hostNetwork: true # Allows the pod to use the host network + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: config + configMap: + name: kube-proxy + - hostPath: + path: /var/lib/microshift/resources/kubeadmin/kubeconfig + type: FileOrCreate + name: kubeconfig \ No newline at end of file diff --git a/assets/optional/kube-proxy/kustomization.aarch64.yaml b/assets/optional/kube-proxy/kustomization.aarch64.yaml new file mode 100644 index 0000000000..d344a75559 --- /dev/null +++ b/assets/optional/kube-proxy/kustomization.aarch64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kube-proxy + newName: quay.io/openshift-release-dev/ocp-v4.0-art-dev + digest: sha256:d905190fda655103fda2ebbf983178e44825ea03897178df61ce3ef3f1b2acba \ No newline at end of file diff --git a/assets/optional/kube-proxy/kustomization.x86_64.yaml b/assets/optional/kube-proxy/kustomization.x86_64.yaml new file mode 100644 index 0000000000..5b5badbec2 --- /dev/null +++ b/assets/optional/kube-proxy/kustomization.x86_64.yaml @@ -0,0 +1,5 @@ + +images: + - name: kube-proxy + newName: quay.io/openshift-release-dev/ocp-v4.0-art-dev + digest: sha256:f49a7ed4d131447243e717e250b0a148b4aabea75a6ded5b011db4fde8a714d2 \ No newline at end of file diff --git a/assets/optional/kube-proxy/kustomization.yaml b/assets/optional/kube-proxy/kustomization.yaml new file mode 100644 index 0000000000..67a36b8090 --- /dev/null +++ b/assets/optional/kube-proxy/kustomization.yaml @@ -0,0 +1,9 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 02-cluster-role.yaml + - 03-cluster-role-binding.yaml + - 04-configmap.yaml + - 05-daemonset.yaml diff --git a/assets/optional/kube-proxy/release-kube-proxy-aarch64.json b/assets/optional/kube-proxy/release-kube-proxy-aarch64.json new file mode 100644 index 0000000000..7147f3ac74 --- /dev/null +++ b/assets/optional/kube-proxy/release-kube-proxy-aarch64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "4.18.0-0.nightly-arm64-2024-08-29-120159" + }, + "images": { + "kube-proxy": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:d905190fda655103fda2ebbf983178e44825ea03897178df61ce3ef3f1b2acba" + } +} \ No newline at end of file diff --git a/assets/optional/kube-proxy/release-kube-proxy-x86_64.json b/assets/optional/kube-proxy/release-kube-proxy-x86_64.json new file mode 100644 index 0000000000..182c1c9beb --- /dev/null +++ b/assets/optional/kube-proxy/release-kube-proxy-x86_64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "4.18.0-0.nightly-arm64-2024-08-29-120159" + }, + "images": { + "kube-proxy": "quay.io/openshift-release-dev/ocp-v4.0-art-dev@sha256:f49a7ed4d131447243e717e250b0a148b4aabea75a6ded5b011db4fde8a714d2" + } +} diff --git a/scripts/auto-rebase/assets.yaml b/scripts/auto-rebase/assets.yaml index e941561696..84aae6e211 100644 --- a/scripts/auto-rebase/assets.yaml +++ b/scripts/auto-rebase/assets.yaml @@ -286,3 +286,18 @@ assets: - file: release-flannel-aarch64.json - file: release-flannel-x86_64.json + - dir: optional/kube-proxy/ + ignore: "they don't exist in upstream repository - only in microshift" + files: + - file: kustomization.yaml + - file: kustomization.aarch64.yaml + - file: kustomization.x86_64.yaml + - file: 00-namespace.yaml + - file: 01-service-account.yaml + - file: 02-cluster-role.yaml + - file: 03-cluster-role-binding.yaml + - file: 04-configmap.yaml + - file: 05-daemonset.yaml + - file: release-kube-proxy-aarch64.json + - file: release-kube-proxy-x86_64.json + From db89c7314d9791468002273d95677c620334f120 Mon Sep 17 00:00:00 2001 From: Praveen Kumar Date: Thu, 5 Sep 2024 14:10:57 +0530 Subject: [PATCH 3/5] Add cniPlugin option in network config This option is used to disable default OVN-K plugin and default value for this would be empty string. Also add warning in case this it is disabled. --- .../config/config-openapi-spec.json | 9 +++ docs/user/howto_config.md | 2 + packaging/microshift/config.yaml | 6 ++ pkg/components/networking.go | 4 + pkg/config/config.go | 8 ++ pkg/config/network.go | 38 +++++++++ pkg/config/network_test.go | 81 +++++++++++++++++++ 7 files changed, 148 insertions(+) create mode 100644 pkg/config/network_test.go diff --git a/cmd/generate-config/config/config-openapi-spec.json b/cmd/generate-config/config/config-openapi-spec.json index e8d8c21621..ea02d2f88b 100755 --- a/cmd/generate-config/config/config-openapi-spec.json +++ b/cmd/generate-config/config/config-openapi-spec.json @@ -229,6 +229,15 @@ "type": "string" } }, + "cniPlugin": { + "description": "CNIPlugin is a user defined string value matching one of the above CNI values. MicroShift uses this\nvalue to decide whether to deploy the OVN-K as default CNI. An unset field defaults to \"\" during yaml parsing, and thus\ncould mean that the cluster has been upgraded. In order to support the existing out-of-box behavior, MicroShift\nassumes an empty string to mean the OVN-K should be deployed.\nAllowed values are: unset or one of [\"\", \"ovnk\", \"none\"]", + "type": "string", + "enum": [ + "", + "none", + "ovnk" + ] + }, "serviceNetwork": { "description": "IP address pool for services.\nCurrently, we only support a single entry here.\nThis field is immutable after installation.", "type": "array", diff --git a/docs/user/howto_config.md b/docs/user/howto_config.md index 6d17c7dee3..adad6787b7 100644 --- a/docs/user/howto_config.md +++ b/docs/user/howto_config.md @@ -43,6 +43,7 @@ manifests: network: clusterNetwork: - "" + cniPlugin: "" serviceNetwork: - "" serviceNodePortRange: "" @@ -107,6 +108,7 @@ manifests: network: clusterNetwork: - 10.42.0.0/16 + cniPlugin: "" serviceNetwork: - 10.43.0.0/16 serviceNodePortRange: 30000-32767 diff --git a/packaging/microshift/config.yaml b/packaging/microshift/config.yaml index 6298c3fb9a..44d3cb8144 100644 --- a/packaging/microshift/config.yaml +++ b/packaging/microshift/config.yaml @@ -93,6 +93,12 @@ network: # This field is immutable after installation. clusterNetwork: - 10.42.0.0/16 + # CNIPlugin is a user defined string value matching one of the above CNI values. MicroShift uses this + # value to decide whether to deploy the OVN-K as default CNI. An unset field defaults to "" during yaml parsing, and thus + # could mean that the cluster has been upgraded. In order to support the existing out-of-box behavior, MicroShift + # assumes an empty string to mean the OVN-K should be deployed. + # Allowed values are: unset or one of ["", "ovnk", "none"] + cniPlugin: "" # IP address pool for services. # Currently, we only support a single entry here. # This field is immutable after installation. diff --git a/pkg/components/networking.go b/pkg/components/networking.go index b3b7e87131..02fd72d075 100644 --- a/pkg/components/networking.go +++ b/pkg/components/networking.go @@ -12,6 +12,10 @@ import ( ) func startCNIPlugin(ctx context.Context, cfg *config.Config, kubeconfigPath string) error { + if !cfg.Network.IsEnabled() { + klog.Warningf("CNI deployment disabled, OVN-K will not be available") + return nil + } var ( ns = []string{ "components/ovn/common/namespace.yaml", diff --git a/pkg/config/config.go b/pkg/config/config.go index 221dd29a67..f0438c7131 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -164,6 +164,10 @@ func (c *Config) incorporateUserSettings(u *Config) { c.DNS.BaseDomain = u.DNS.BaseDomain } + if u.Network.CNIPlugin != "" { + c.Network.CNIPlugin = u.Network.CNIPlugin + } + if len(u.Network.ClusterNetwork) != 0 { c.Network.ClusterNetwork = u.Network.ClusterNetwork } @@ -368,6 +372,10 @@ func (c *Config) validate() error { return fmt.Errorf("error validating networks: %w", err) } + if !c.Network.validCNIPlugin() { + return fmt.Errorf("invalid cni plugin for network configuration %q", c.Network.CNIPlugin) + } + //nolint:nestif // extracting the nested ifs will just increase the complexity of the if expressions as validation expands if len(c.ApiServer.SubjectAltNames) > 0 { // Any entry in SubjectAltNames will be included in the external access certificates. diff --git a/pkg/config/network.go b/pkg/config/network.go index 7749fd3ea3..cfde3f91df 100644 --- a/pkg/config/network.go +++ b/pkg/config/network.go @@ -5,9 +5,36 @@ import ( "net" "github.com/apparentlymart/go-cidr/cidr" + "k8s.io/apimachinery/pkg/util/sets" +) + +// CNIPlugin is an enum value that determines whether MicroShift deploys OVNK. +// +kubebuilder:validation:Enum:="";none;ovnk +type CNIPlugin string + +const ( + // CniPluginUnset exists to support backwards compatibility with existing MicroShift clusters. When .network.cniPlugin is + // "", MicroShift will default to deploying OVNK. This preserves the current deployment behavior of existing + // clusters. + CniPluginUnset CNIPlugin = "" + // CniPluginNone signals MicroShift to not deploy the LVMS components. Setting the value for a cluster that has already + // deployed LVMS will not cause LVMS to be deleted. Otherwise, volumes already deployed on the cluster would be + // orphaned once their workloads stop or restart. + CniPluginNone CNIPlugin = "none" + // CniPluginOVNK is equivalent to CniPluginUnset, and explicitly tells MicroShift to deploy OVNK. This option exists to + // provide a differentiation between OVNK and potential future CNI options. + CniPluginOVNK CNIPlugin = "ovnk" ) type Network struct { + // CNIPlugin is a user defined string value matching one of the above CNI values. MicroShift uses this + // value to decide whether to deploy the OVN-K as default CNI. An unset field defaults to "" during yaml parsing, and thus + // could mean that the cluster has been upgraded. In order to support the existing out-of-box behavior, MicroShift + // assumes an empty string to mean the OVN-K should be deployed. + // Allowed values are: unset or one of ["", "ovnk", "none"] + // +kubebuilder:validation:Optional + CNIPlugin CNIPlugin `json:"cniPlugin,omitempty"` + // IP address pool to use for pod IPs. // This field is immutable after installation. // +kubebuilder:default={"10.42.0.0/16"} @@ -63,3 +90,14 @@ func isValidIPAddress(ipAddress string) bool { ip := net.ParseIP(ipAddress) return ip != nil } + +func (n Network) validCNIPlugin() (isSupported bool) { + return sets.New[CNIPlugin](CniPluginUnset, CniPluginOVNK, CniPluginNone).Has(n.CNIPlugin) +} + +// IsEnabled returns false only when .network.cniPlugin: "none". An empty value is considered "enabled" +// for backwards compatibility. Otherwise, the meaning of the config would silently change after an +// upgrade from enabled-by-default to disabled-by-default. +func (n Network) IsEnabled() bool { + return n.CNIPlugin != CniPluginNone +} diff --git a/pkg/config/network_test.go b/pkg/config/network_test.go new file mode 100644 index 0000000000..44d489bbef --- /dev/null +++ b/pkg/config/network_test.go @@ -0,0 +1,81 @@ +package config + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestCNIPlugin_IsEnabled(t *testing.T) { + type fields struct { + CNIPlugin CNIPlugin + } + tests := []struct { + name string + fields fields + want bool + }{ + { + name: "is disabled when plugin is none", + fields: fields{ + CNIPlugin: CniPluginNone, + }, + want: false, + }, + { + name: "is enabled when plugin is ovnk", + fields: fields{ + CNIPlugin: CniPluginOVNK, + }, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := Network{ + CNIPlugin: tt.fields.CNIPlugin, + } + assert.Equalf(t, tt.want, n.IsEnabled(), "IsEnabled()") + }) + } +} + +func TestNetwork_cniPluginIsValid(t *testing.T) { + type fields struct { + CNIPlugin CNIPlugin + } + tests := []struct { + name string + fields fields + wantIsSupported bool + }{ + { + name: "is valid when value matches one of predefined drivers", + fields: fields{ + CNIPlugin: CniPluginOVNK, + }, + wantIsSupported: true, + }, + { + name: "is valid when value is an empty string", + fields: fields{ + CNIPlugin: "", + }, + wantIsSupported: true, + }, + { + name: "is invalid when value does not match one of predefined drivers", + fields: fields{ + CNIPlugin: "foobar", + }, + wantIsSupported: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := Network{ + CNIPlugin: tt.fields.CNIPlugin, + } + assert.Equalf(t, tt.wantIsSupported, n.validCNIPlugin(), "validCNIPlugin()") + }) + } +} From 954833dbe9836ce89cf2f08e59d272021806458e Mon Sep 17 00:00:00 2001 From: Praveen Kumar Date: Thu, 5 Sep 2024 14:21:56 +0530 Subject: [PATCH 4/5] microshift.spec: Add flannel as subpackage for microshift --- packaging/flannel/00-disableDefaultCNI.yaml | 7 ++ packaging/flannel/microshift-flannel.service | 28 ++++++++ packaging/rpm/microshift.spec | 74 ++++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 packaging/flannel/00-disableDefaultCNI.yaml create mode 100644 packaging/flannel/microshift-flannel.service diff --git a/packaging/flannel/00-disableDefaultCNI.yaml b/packaging/flannel/00-disableDefaultCNI.yaml new file mode 100644 index 0000000000..e368f3a88d --- /dev/null +++ b/packaging/flannel/00-disableDefaultCNI.yaml @@ -0,0 +1,7 @@ +network: + # CNIPlugin is a user defined string value matching one of the above CNI values. MicroShift uses this + # value to decide whether to deploy the OVN-K as default CNI. An unset field defaults to "" during yaml parsing, and thus + # could mean that the cluster has been upgraded. In order to support the existing out-of-box behavior, MicroShift + # assumes an empty string to mean the OVN-K should be deployed. + # Allowed values are: unset or one of ["", "ovnk", "none"] + cniPlugin: "none" diff --git a/packaging/flannel/microshift-flannel.service b/packaging/flannel/microshift-flannel.service new file mode 100644 index 0000000000..a33b5a3ae8 --- /dev/null +++ b/packaging/flannel/microshift-flannel.service @@ -0,0 +1,28 @@ +[Unit] +Description=MicroShift +Wants=network-online.target crio.service +After=network-online.target crio.service + +# Control shutdown order by declaring this service to start Before the kubepods.slice +# transient systemd unit; this makes system shutdown delay MicroShift shutdown until +# all the pod containers are down. This is important because some services need to talk +# to the MicroShift API during shutdown (i.e. releasing leader election locks or cleaning +# up other resources) MicroShift restart or manual stop will not stop the kubepods. +Before=kubepods.slice + +[Service] +WorkingDirectory=/usr/bin/ +ExecStart=microshift run +Restart=always +User=root +Type=notify +Delegate=yes +CPUAccounting=yes +BlockIOAccounting=yes +MemoryAccounting=yes +LimitNOFILE=1048576 +TimeoutStartSec=4m + +[Install] +WantedBy=multi-user.target +Also=microshift-cleanup-kubelet.service diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 163c649e34..6346382281 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -165,6 +165,26 @@ The microshift-multus-release-info package provides release information files fo release. These files contain the list of container image references used by the Multus CNI for MicroShift and can be used to embed those images into osbuilder blueprints. +%package flannel +Summary: flannel CNI for MicroShift +ExclusiveArch: x86_64 aarch64 +Requires: microshift = %{version} + +%description flannel +The microshift-flannel package provides the required manifests for the flannel CNI and the dependent +kube-proxy to be installed on MicroShift. + +%package flannel-release-info +Summary: Release information for flannel CNI for MicroShift +BuildArch: noarch +Requires: microshift-release-info = %{version} + +%description flannel-release-info +The microshift-flannel-release-info package provides release information files for this +release. These files contain the list of container image references used by the flannel CNI +with the dependent kube-proxy for MicroShift and can be used to embed those images +into osbuilder blueprints. + %package low-latency Summary: Baseline configuration for running low latency workload on MicroShift BuildArch: noarch @@ -359,6 +379,45 @@ cat assets/optional/multus/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/ mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/multus/release-multus-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +# kube-proxy +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +# Copy all the manifests except the arch specific ones +install -p -m644 assets/optional/kube-proxy/0* %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +install -p -m644 assets/optional/kube-proxy/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy + +%ifarch %{arm} aarch64 +cat assets/optional/kube-proxy/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/kustomization.yaml +%endif + +%ifarch x86_64 +cat assets/optional/kube-proxy/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/kustomization.yaml +%endif + +# kube-proxy-release-info +mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release +install -p -m644 assets/optional/kube-proxy/release-kube-proxy-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + +# flannel +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel +install -d -m755 %{buildroot}%{_sysconfdir}/systemd/system +# Copy all the manifests except the arch specific ones +install -p -m644 assets/optional/flannel/0* %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel +install -p -m644 assets/optional/flannel/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel +install -p -m644 packaging/flannel/00-disableDefaultCNI.yaml %{buildroot}%{_sysconfdir}/microshift/config.d/00-disableDefaultCNI.yaml +install -p -m644 packaging/flannel/microshift-flannel.service %{buildroot}%{_sysconfdir}/systemd/system/microshift.service + +%ifarch %{arm} aarch64 +cat assets/optional/flannel/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel/kustomization.yaml +%endif + +%ifarch x86_64 +cat assets/optional/flannel/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel/kustomization.yaml +%endif + +# flannel-release-info +mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release +install -p -m644 assets/optional/flannel/release-flannel-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + # cleanup kubelet install -p -m644 packaging/tuned/microshift-cleanup-kubelet.service %{buildroot}%{_unitdir}/microshift-cleanup-kubelet.service @@ -510,6 +569,18 @@ fi %files multus-release-info %{_datadir}/microshift/release/release-multus-{x86_64,aarch64}.json +%files flannel +%dir %{_prefix}/lib/microshift/manifests.d/000-microshift-flannel +%dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy +%{_prefix}/lib/microshift/manifests.d/000-microshift-flannel/* +%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy/* +%config(noreplace) %{_sysconfdir}/microshift/config.d/00-disableDefaultCNI.yaml +%{_sysconfdir}/systemd/system/microshift.service + +%files flannel-release-info +%{_datadir}/microshift/release/release-flannel-{x86_64,aarch64}.json +%{_datadir}/microshift/release/release-kube-proxy-{x86_64,aarch64}.json + %files low-latency %{_prefix}/lib/tuned/microshift-baseline %config(noreplace) %{_sysconfdir}/tuned/microshift-baseline-variables.conf @@ -522,6 +593,9 @@ fi # Use Git command to generate the log and replace the VERSION string # LANG=C git log --date="format:%a %b %d %Y" --pretty="tformat:* %cd %an <%ae> VERSION%n- %s%n" packaging/rpm/microshift.spec %changelog +* Mon Sep 16 2024 Praveen Kumar 4.18.0 +- Add microshift-flannel subpackage + * Thu Sep 12 2024 Gregory Giguashvili 4.17.0 - Declare openvswitch3.3 package as obsolete to allow seemless upgrade to openvswitch3.4 From 53ed9fb0b56c968577ea6812671f9299ad848017 Mon Sep 17 00:00:00 2001 From: Praveen Kumar Date: Mon, 23 Sep 2024 16:07:29 +0530 Subject: [PATCH 5/5] Make flannel package optional to build `WITH_FLANNEL` env now we can build the flannel package with condition and default value is `0` which means not building it. --- Makefile | 3 +++ packaging/rpm/make-rpm.sh | 4 ++-- packaging/rpm/microshift.spec | 9 +++++++++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c942c9eea8..0d9e5d02c7 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,7 @@ PATCH := $(shell echo $(SOURCE_GIT_TAG) | awk -F'[._~-]' '{print $$3}') SRC_ROOT :=$(shell pwd) +WITH_FLANNEL ?= 0 OUTPUT_DIR :=_output RPM_BUILD_DIR :=$(OUTPUT_DIR)/rpmbuild ISO_DIR :=$(OUTPUT_DIR)/image-builder @@ -286,6 +287,7 @@ rpm: SOURCE_GIT_TAG=${SOURCE_GIT_TAG} \ SOURCE_GIT_COMMIT=${SOURCE_GIT_COMMIT} \ SOURCE_GIT_TREE_STATE=${SOURCE_GIT_TREE_STATE} \ + WITH_FLANNEL=${WITH_FLANNEL} \ ./packaging/rpm/make-rpm.sh rpm local .PHONY: rpm @@ -295,6 +297,7 @@ srpm: SOURCE_GIT_TAG=${SOURCE_GIT_TAG} \ SOURCE_GIT_COMMIT=${SOURCE_GIT_COMMIT} \ SOURCE_GIT_TREE_STATE=${SOURCE_GIT_TREE_STATE} \ + WITH_FLANNEL=${WITH_FLANNEL} \ ./packaging/rpm/make-rpm.sh srpm local .PHONY: srpm diff --git a/packaging/rpm/make-rpm.sh b/packaging/rpm/make-rpm.sh index f28b670ba1..65022ebd77 100755 --- a/packaging/rpm/make-rpm.sh +++ b/packaging/rpm/make-rpm.sh @@ -2,7 +2,7 @@ set -e -o pipefail # must be passed down to this script from Makefile -ENV_VARS="MICROSHIFT_VERSION RPM_RELEASE SOURCE_GIT_TAG SOURCE_GIT_TREE_STATE" +ENV_VARS="MICROSHIFT_VERSION RPM_RELEASE SOURCE_GIT_TAG SOURCE_GIT_TREE_STATE WITH_FLANNEL" for env in ${ENV_VARS} ; do if [[ -z "${!env}" ]] ; then echo "Error: Mandatory environment variable '${env}' is missing" @@ -68,7 +68,7 @@ EOF # algorithm. # shellcheck disable=SC2086 # We want word splitting to happen with RPMBUILD_OPT for flags to be interpreted correctly - rpmbuild --quiet ${RPMBUILD_OPT} --define "_topdir ${RPMBUILD_DIR}" --define "_binary_payload w19T8.zstdio" "${RPMBUILD_DIR}"SPECS/microshift.spec + rpmbuild --quiet ${RPMBUILD_OPT} --define "_topdir ${RPMBUILD_DIR}" --define "_binary_payload w19T8.zstdio" --define "with_flannel ${WITH_FLANNEL}" "${RPMBUILD_DIR}"SPECS/microshift.spec } print_info() { diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 6346382281..59cb5d0fb7 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -35,6 +35,9 @@ # Git related details %global shortcommit %(c=%{commit}; echo ${c:0:7}) +# Don't build flannel subpackage by default +%{!?with_flannel: %global with_flannel 0} + Name: microshift Version: %{version} Release: %{release}%{dist} @@ -165,6 +168,7 @@ The microshift-multus-release-info package provides release information files fo release. These files contain the list of container image references used by the Multus CNI for MicroShift and can be used to embed those images into osbuilder blueprints. +%if %{with_flannel} %package flannel Summary: flannel CNI for MicroShift ExclusiveArch: x86_64 aarch64 @@ -184,6 +188,7 @@ The microshift-flannel-release-info package provides release information files f release. These files contain the list of container image references used by the flannel CNI with the dependent kube-proxy for MicroShift and can be used to embed those images into osbuilder blueprints. +%endif %package low-latency Summary: Baseline configuration for running low latency workload on MicroShift @@ -379,6 +384,7 @@ cat assets/optional/multus/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/ mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/multus/release-multus-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +%if %{with_flannel} # kube-proxy install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy # Copy all the manifests except the arch specific ones @@ -417,6 +423,7 @@ cat assets/optional/flannel/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix} # flannel-release-info mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/flannel/release-flannel-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +%endif # cleanup kubelet install -p -m644 packaging/tuned/microshift-cleanup-kubelet.service %{buildroot}%{_unitdir}/microshift-cleanup-kubelet.service @@ -569,6 +576,7 @@ fi %files multus-release-info %{_datadir}/microshift/release/release-multus-{x86_64,aarch64}.json +%if %{with_flannel} %files flannel %dir %{_prefix}/lib/microshift/manifests.d/000-microshift-flannel %dir %{_prefix}/lib/microshift/manifests.d/000-microshift-kube-proxy @@ -580,6 +588,7 @@ fi %files flannel-release-info %{_datadir}/microshift/release/release-flannel-{x86_64,aarch64}.json %{_datadir}/microshift/release/release-kube-proxy-{x86_64,aarch64}.json +%endif %files low-latency %{_prefix}/lib/tuned/microshift-baseline