diff --git a/pkg/clusterstack/config.go b/pkg/clusterstack/config.go index e3c45d14..56cb078f 100644 --- a/pkg/clusterstack/config.go +++ b/pkg/clusterstack/config.go @@ -45,45 +45,45 @@ type CsctlConfig struct { } // GetCsctlConfig returns CsctlConfig. -func GetCsctlConfig(path string) (CsctlConfig, error) { +func GetCsctlConfig(path string) (*CsctlConfig, error) { configPath := filepath.Join(path, "csctl.yaml") configFileData, err := os.ReadFile(filepath.Clean(configPath)) if err != nil { - return CsctlConfig{}, fmt.Errorf("failed to read csctl config: %w", err) + return nil, fmt.Errorf("failed to read csctl config: %w", err) } - cs := CsctlConfig{} + cs := &CsctlConfig{} if err := yaml.Unmarshal(configFileData, &cs); err != nil { - return CsctlConfig{}, fmt.Errorf("failed to unmarshal csctl yaml: %w", err) + return nil, fmt.Errorf("failed to unmarshal csctl yaml: %w", err) } if cs.Config.Provider.Type == "" { - return CsctlConfig{}, fmt.Errorf("provider type must not be empty") + return nil, fmt.Errorf("provider type must not be empty") } if len(cs.Config.Provider.Type) > 253 { - return CsctlConfig{}, fmt.Errorf("provider name must not be greater than 253") + return nil, fmt.Errorf("provider name must not be greater than 253") } match, err := regexp.MatchString(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$`, cs.Config.Provider.Type) if err != nil { - return CsctlConfig{}, fmt.Errorf("failed to provider name match regex: %w", err) + return nil, fmt.Errorf("failed to provider name match regex: %w", err) } if !match { - return CsctlConfig{}, fmt.Errorf("invalid provider type: %q", cs.Config.Provider.Type) + return nil, fmt.Errorf("invalid provider type: %q", cs.Config.Provider.Type) } if cs.Config.ClusterStackName == "" { - return CsctlConfig{}, fmt.Errorf("cluster stack name must not be empty") + return nil, fmt.Errorf("cluster stack name must not be empty") } // Validate kubernetes version matched, err := regexp.MatchString(`^v\d+\.\d+\.\d+$`, cs.Config.KubernetesVersion) if err != nil { - return CsctlConfig{}, fmt.Errorf("failed to kubernetes match regex: %w", err) + return nil, fmt.Errorf("failed to kubernetes match regex: %w", err) } if !matched { - return CsctlConfig{}, fmt.Errorf("invalid kubernetes version: %q", cs.Config.KubernetesVersion) + return nil, fmt.Errorf("invalid kubernetes version: %q", cs.Config.KubernetesVersion) } return cs, nil diff --git a/pkg/clusterstack/metadata.go b/pkg/clusterstack/metadata.go index 8c02ab2a..0141f480 100644 --- a/pkg/clusterstack/metadata.go +++ b/pkg/clusterstack/metadata.go @@ -44,17 +44,17 @@ type MetaData struct { } // ParseMetaData parse the metadata file. -func ParseMetaData(path string) (MetaData, error) { +func ParseMetaData(path string) (*MetaData, error) { metadataPath := filepath.Join(path, "metadata.yaml") fileInfo, err := os.ReadFile(filepath.Clean(metadataPath)) if err != nil { - return MetaData{}, fmt.Errorf("failed to read metadata file: %w", err) + return nil, fmt.Errorf("failed to read metadata file: %w", err) } - metaData := MetaData{} + metaData := &MetaData{} if err := yaml.Unmarshal(fileInfo, &metaData); err != nil { - return MetaData{}, fmt.Errorf("failed to unmarshal metadata yaml: %w", err) + return nil, fmt.Errorf("failed to unmarshal metadata yaml: %w", err) } return metaData, nil diff --git a/pkg/clusterstack/mode.go b/pkg/clusterstack/mode.go index 84c27156..7e109cf7 100644 --- a/pkg/clusterstack/mode.go +++ b/pkg/clusterstack/mode.go @@ -24,22 +24,22 @@ import ( ) // HandleStableMode returns metadata for the stable mode. -func HandleStableMode(gitHubReleasePath string, currentReleaseHash, latestReleaseHash hash.ReleaseHash) (MetaData, error) { +func HandleStableMode(gitHubReleasePath string, currentReleaseHash, latestReleaseHash hash.ReleaseHash) (*MetaData, error) { metadata, err := ParseMetaData(gitHubReleasePath) if err != nil { - return MetaData{}, fmt.Errorf("failed to parse metadata: %w", err) + return nil, fmt.Errorf("failed to parse metadata: %w", err) } metadata.Versions.ClusterStack, err = BumpVersion(metadata.Versions.ClusterStack) if err != nil { - return MetaData{}, fmt.Errorf("failed to bump cluster stack: %w", err) + return nil, fmt.Errorf("failed to bump cluster stack: %w", err) } fmt.Printf("Bumped ClusterStack Version: %s\n", metadata.Versions.ClusterStack) if currentReleaseHash.ClusterAddonDir != latestReleaseHash.ClusterAddonDir || currentReleaseHash.ClusterAddonValues != latestReleaseHash.ClusterAddonValues { metadata.Versions.Components.ClusterAddon, err = BumpVersion(metadata.Versions.Components.ClusterAddon) if err != nil { - return MetaData{}, fmt.Errorf("failed to bump cluster addon: %w", err) + return nil, fmt.Errorf("failed to bump cluster addon: %w", err) } fmt.Printf("Bumped ClusterAddon Version: %s\n", metadata.Versions.Components.ClusterAddon) } else { @@ -49,7 +49,7 @@ func HandleStableMode(gitHubReleasePath string, currentReleaseHash, latestReleas if currentReleaseHash.NodeImageDir != latestReleaseHash.NodeImageDir { metadata.Versions.Components.NodeImage, err = BumpVersion(metadata.Versions.Components.NodeImage) if err != nil { - return MetaData{}, fmt.Errorf("failed to bump node image: %w", err) + return nil, fmt.Errorf("failed to bump node image: %w", err) } fmt.Printf("Bumped NodeImage Version: %s\n", metadata.Versions.Components.NodeImage) } else { @@ -64,15 +64,15 @@ func HandleStableMode(gitHubReleasePath string, currentReleaseHash, latestReleas } // HandleHashMode returns metadata of Hash mode. -func HandleHashMode(kubernetesVersion string) (MetaData, error) { +func HandleHashMode(kubernetesVersion string) (*MetaData, error) { commitHash, err := git.GetLatestGitCommit("./") if err != nil { - return MetaData{}, fmt.Errorf("failed to get latest commit hash: %w", err) + return nil, fmt.Errorf("failed to get latest commit hash: %w", err) } commitHash = fmt.Sprintf("v0-sha.%s", commitHash) - return MetaData{ + return &MetaData{ APIVersion: "metadata.clusterstack.x-k8s.io/v1alpha1", Versions: Versions{ Kubernetes: kubernetesVersion, diff --git a/pkg/cmd/create.go b/pkg/cmd/create.go index b0fa9649..aee5119f 100644 --- a/pkg/cmd/create.go +++ b/pkg/cmd/create.go @@ -58,13 +58,14 @@ var ( // CreateOptions contains config for creating a release. type CreateOptions struct { - ClusterStackPath string - ClusterStackReleaseDir string - Config clusterstack.CsctlConfig - Metadata clusterstack.MetaData - CurrentReleaseHash hash.ReleaseHash - LatestReleaseHash hash.ReleaseHash - NodeImageRegistry string + newClusterStackConvention bool + ClusterStackPath string + ClusterStackReleaseDir string + Config *clusterstack.CsctlConfig + Metadata *clusterstack.MetaData + CurrentReleaseHash hash.ReleaseHash + LatestReleaseHash hash.ReleaseHash + NodeImageRegistry string } // createCmd represents the create command. @@ -95,7 +96,17 @@ func GetCreateOptions(ctx context.Context, clusterStackPath string) (*CreateOpti createOption.ClusterStackPath = clusterStackPath createOption.Config = config - _, _, err = providerplugin.GetProviderExecutable(&config) + if _, err := os.Stat(filepath.Join(clusterStackPath, "clusteraddon.yaml")); err != nil { + // old if clusteraddon.yaml is not present. + if !os.IsNotExist(err) { + return nil, fmt.Errorf("failed to find clusteraddon.yaml: %w", err) + } + } else { + // new if clusteraddon.yaml is present. + createOption.newClusterStackConvention = true + } + + _, _, err = providerplugin.GetProviderExecutable(config) if err != nil { return createOption, fmt.Errorf("providerplugin.GetProviderExecutable(&config) failed: %w", err) } @@ -121,7 +132,7 @@ func GetCreateOptions(ctx context.Context, clusterStackPath string) (*CreateOpti // update the metadata kubernetes version with the csctl.yaml config createOption.Metadata.Versions.Kubernetes = config.Config.KubernetesVersion - latestRepoRelease, err := github.GetLatestReleaseFromRemoteRepository(ctx, mode, &config, gc) + latestRepoRelease, err := github.GetLatestReleaseFromRemoteRepository(ctx, mode, config, gc) if err != nil { return nil, fmt.Errorf("failed to get latest release form remote repository: %w", err) } @@ -144,7 +155,7 @@ func GetCreateOptions(ctx context.Context, clusterStackPath string) (*CreateOpti } } - releaseDirName, err := clusterstack.GetClusterStackReleaseDirectoryName(&createOption.Metadata, &createOption.Config) + releaseDirName, err := clusterstack.GetClusterStackReleaseDirectoryName(createOption.Metadata, createOption.Config) if err != nil { return nil, fmt.Errorf("failed to get cluster stack release directory name: %w", err) } @@ -222,7 +233,7 @@ func (c *CreateOptions) generateRelease() error { } // Build all the templated output and put it in a tmp directory - if err := template.GenerateOutputFromTemplate(c.ClusterStackPath, "./.tmp/", &c.Metadata); err != nil { + if err := template.GenerateOutputFromTemplate(c.ClusterStackPath, "./.tmp/", c.Metadata); err != nil { return fmt.Errorf("failed to generate tmp output: %w", err) } @@ -238,18 +249,30 @@ func (c *CreateOptions) generateRelease() error { } // Package Helm from the tmp directory to the release directory - if err := template.CreatePackage("./.tmp/", c.ClusterStackReleaseDir); err != nil { + if err := template.CreatePackage("./.tmp/", c.ClusterStackReleaseDir, c.newClusterStackConvention, c.Config, c.Metadata); err != nil { return fmt.Errorf("failed to create template package: %w", err) } - // Copy the cluster-addon-values.yaml config to release if old way - clusterAddonData, err := os.ReadFile(filepath.Join(c.ClusterStackPath, "cluster-addon-values.yaml")) - if err != nil { - return fmt.Errorf("failed to read cluster-addon-values.yaml: %w", err) - } + if c.newClusterStackConvention { + // Copy the clusteraddon.yaml config to release if new way + clusterAddonData, err := os.ReadFile(filepath.Join(c.ClusterStackPath, "clusteraddon.yaml")) + if err != nil { + return fmt.Errorf("failed to read clusteraddon.yaml: %w", err) + } + + if err := os.WriteFile(filepath.Join(c.ClusterStackReleaseDir, "clusteraddon.yaml"), clusterAddonData, os.FileMode(0o644)); err != nil { + return fmt.Errorf("failed to write clusteraddon.yaml: %w", err) + } + } else { + // Copy the cluster-addon-values.yaml config to release if old way + clusterAddonData, err := os.ReadFile(filepath.Join(c.ClusterStackPath, "cluster-addon-values.yaml")) + if err != nil { + return fmt.Errorf("failed to read cluster-addon-values.yaml: %w", err) + } - if err := os.WriteFile(filepath.Join(c.ClusterStackReleaseDir, "cluster-addon-values.yaml"), clusterAddonData, os.FileMode(0o644)); err != nil { - return fmt.Errorf("failed to write cluster-addon-values.yaml: %w", err) + if err := os.WriteFile(filepath.Join(c.ClusterStackReleaseDir, "cluster-addon-values.yaml"), clusterAddonData, os.FileMode(0o644)); err != nil { + return fmt.Errorf("failed to write cluster-addon-values.yaml: %w", err) + } } // Put the final metadata file into the output directory. @@ -268,7 +291,7 @@ func (c *CreateOptions) generateRelease() error { return fmt.Errorf("failed to write metadata: %w", err) } - err = providerplugin.CreateNodeImages(&c.Config, + err = providerplugin.CreateNodeImages(c.Config, c.ClusterStackPath, c.ClusterStackReleaseDir, c.NodeImageRegistry) diff --git a/pkg/template/package.go b/pkg/template/package.go index a0248cef..902a8209 100644 --- a/pkg/template/package.go +++ b/pkg/template/package.go @@ -17,20 +17,40 @@ limitations under the License. package template import ( + "archive/tar" + "compress/gzip" "fmt" + "io" + "io/fs" + "os" "path/filepath" + "github.com/SovereignCloudStack/csctl/pkg/clusterstack" "helm.sh/helm/v3/pkg/action" ) // CreatePackage creates the package for release. -func CreatePackage(src, dst string) error { +func CreatePackage(src, dst string, newType bool, config *clusterstack.CsctlConfig, metadata *clusterstack.MetaData) error { + fmt.Printf("path now: %q\n", filepath.Join(src, "cluster-class")) if err := createHelmPackage(filepath.Join(src, "cluster-class"), dst); err != nil { return fmt.Errorf("failed to create package for ClusterClass: %w", err) } - if err := createHelmPackage(filepath.Join(src, "cluster-addon"), dst); err != nil { - return fmt.Errorf("failed to create package for ClusterAddon: %w", err) + kubernetesVerion, err := config.ParseKubernetesVersion() + if err != nil { + return fmt.Errorf("failed to parse kubernetes version: %w", err) + } + + if newType { + clusterAddonDst := filepath.Join(dst, fmt.Sprintf("%s-%s-%s-cluster-addon-%s.tgz", config.Config.Provider.Type, config.Config.ClusterStackName, kubernetesVerion.String(), metadata.Versions.Components.ClusterAddon)) + if err := createTarPackage(filepath.Join(src, "cluster-addon"), clusterAddonDst); err != nil { + return fmt.Errorf("failed to create package for ClusterAddon: %w", err) + } + } else { + fmt.Printf("path now: %q\n", filepath.Join(src, "cluster-addon")) + if err := createHelmPackage(filepath.Join(src, "cluster-addon"), dst); err != nil { + return fmt.Errorf("failed to create helm package for ClusterAddon: %w", err) + } } return nil @@ -47,3 +67,64 @@ func createHelmPackage(src, dst string) error { return nil } + +func createTarPackage(src, dst string) error { + outFile, err := os.Create(filepath.Clean(dst)) + if err != nil { + return fmt.Errorf("failed to create tar output destination directory: %w", err) + } + defer outFile.Close() + + gw := gzip.NewWriter(outFile) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + if err := filepath.Walk(src, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore the root folder itself + if path == src { + return nil + } + + relPath, err := filepath.Rel(src, path) + if err != nil { + return fmt.Errorf("failed to calculate relative path of source: %q and destination: %q directory: %w", src, dst, err) + } + + // Use filepath.ToSlash to convert path separators to '/' + relPath = filepath.ToSlash(relPath) + + header, err := tar.FileInfoHeader(info, relPath) + if err != nil { + return fmt.Errorf("failed to get the tar info header: %w", err) + } + header.Name = relPath + + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("failed to set the write header: %w", err) + } + + if !info.IsDir() { + file, err := os.Open(filepath.Clean(path)) + if err != nil { + return fmt.Errorf("failed to open path: %w", err) + } + defer file.Close() + + if _, err := io.Copy(tw, file); err != nil { + return fmt.Errorf("failed to copy file: %w", err) + } + } + + return nil + }); err != nil { + return fmt.Errorf("failed to walk on the source directory: %w", err) + } + + return nil +} diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/cni/Chart.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/cni/Chart.yaml new file mode 100644 index 00000000..f04b67a8 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/cni/Chart.yaml @@ -0,0 +1,9 @@ +apiVersion: v2 +type: application +description: CNI chart for kind +name: CNI +version: '0' +maintainers: + - name: Syself + email: info@syself.com + url: https://github.com/syself diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/cni/templates/cni.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/cni/templates/cni.yaml new file mode 100644 index 00000000..fbe82f54 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/cni/templates/cni.yaml @@ -0,0 +1,120 @@ +# kindnetd networking manifest +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet + name: kindnet + name: kindnet +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - patch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kindnet +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kindnet +subjects: + - kind: ServiceAccount + name: kindnet + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kindnet + namespace: kube-system +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kindnet + namespace: kube-system + labels: + tier: node + app: kindnet + k8s-app: kindnet +spec: + selector: + matchLabels: + app: kindnet + template: + metadata: + labels: + tier: node + app: kindnet + k8s-app: kindnet + spec: + hostNetwork: true + tolerations: + - operator: Exists + effect: NoSchedule + serviceAccountName: kindnet + containers: + - name: kindnet-cni + image: kindest/kindnetd:v20230511-dc714da8 + env: + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + # We're using the dualstack CIDRs here. The order doesn't matter for kindnet as the loops are run concurrently. + # REF: https://github.com/kubernetes-sigs/kind/blob/3dbeb894e3092a336ab4278d3823e73a1d66aff7/images/kindnetd/cmd/kindnetd/main.go#L149-L175 + - name: POD_SUBNET + value: '${DOCKER_POD_CIDRS},${DOCKER_POD_IPV6_CIDRS}' + volumeMounts: + - name: cni-cfg + mountPath: /etc/cni/net.d + - name: xtables-lock + mountPath: /run/xtables.lock + readOnly: false + - name: lib-modules + mountPath: /lib/modules + readOnly: true + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: false + capabilities: + add: ["NET_RAW", "NET_ADMIN"] + volumes: + - name: cni-bin + hostPath: + path: /opt/cni/bin + type: DirectoryOrCreate + - name: cni-cfg + hostPath: + path: /etc/cni/net.d + type: DirectoryOrCreate + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: lib-modules + hostPath: + path: /lib/modules diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/cni/values.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/cni/values.yaml new file mode 100644 index 00000000..e69de29b diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/.helmignore b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.lock b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.lock new file mode 100644 index 00000000..dcf1105c --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.lock @@ -0,0 +1,6 @@ +dependencies: +- name: metrics-server + repository: https://kubernetes-sigs.github.io/metrics-server/ + version: 3.11.0 +digest: sha256:d92caa34d06b047b0390aa33a6f5d15d4fe0566143625204ebbbc69085133c4e +generated: "2024-01-08T18:57:56.42493498+05:30" diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.yaml new file mode 100644 index 00000000..1079e6f1 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/Chart.yaml @@ -0,0 +1,16 @@ +apiVersion: v2 +type: application +description: | + This chart installs and configures: + * Docker Ferrol Cluster Class +name: docker-ferrol-1-27-cluster-addon +version: << .ClusterAddonVersion >> +maintainers: + - name: Syself + email: info@syself.com + url: https://github.com/syself +dependencies: + - name: "metrics-server" + version: "3.11.0" + repository: "https://kubernetes-sigs.github.io/metrics-server/" + alias: "metrics-server" diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/charts/metrics-server-3.11.0.tgz b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/charts/metrics-server-3.11.0.tgz new file mode 100644 index 00000000..8860457e Binary files /dev/null and b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/charts/metrics-server-3.11.0.tgz differ diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/overwrite.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/overwrite.yaml new file mode 100644 index 00000000..e0033983 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/overwrite.yaml @@ -0,0 +1,5 @@ +values: | + metrics-server: + commonLabels: + domain: "{{ .Cluster.spec.controlPlaneEndpoint.host }}" + clusterAddonVersion: "v2" diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/templates/ngnix.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/templates/ngnix.yaml new file mode 100644 index 00000000..594cb5f6 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/templates/ngnix.yaml @@ -0,0 +1,101 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-ngnix-7777 + namespace: "kube-system" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: ngnix + app.kubernetes.io/version: 1.25.3 + helm.sh/chart: ngnix-15.10.1 +spec: + replicas: 1 + revisionHistoryLimit: 10 + strategy: + rollingUpdate: {} + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: ngnix + template: + metadata: + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: ngnix + app.kubernetes.io/version: 1.25.3 + helm.sh/chart: ngnix-15.10.1 + annotations: + spec: + + shareProcessNamespace: false + serviceAccountName: release-name-ngnix + automountServiceAccountToken: false + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: ngnix + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + hostNetwork: false + hostIPC: false + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + containers: + - name: nginx + image: docker.io/bitnami/nginx:1.25.3-debian-11-r6 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: false + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + env: + - name: BITNAMI_DEBUG + value: "false" + - name: NGINX_HTTP_PORT_NUMBER + value: "8080" + envFrom: + ports: + - name: http + containerPort: 8080 + livenessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + tcpSocket: + port: http + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 3 + tcpSocket: + port: http + resources: + limits: {} + requests: {} + volumeMounts: + volumes: diff --git a/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/values.yaml b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/values.yaml new file mode 100644 index 00000000..17847d61 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-addon/metrics-server/values.yaml @@ -0,0 +1,44 @@ +metrics-server: + fullnameOverride: metrics-server + replicas: 1 + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + + podDisruptionBudget: + apiVersion: policy/v1 + enabled: true + minAvailable: 1 + + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: CriticalAddonsOnly + operator: Exists + + service: + labels: + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "metrics-server" + + defaultArgs: + - --cert-dir=/tmp + - --kubelet-preferred-address-types=ExternalIP,Hostname + - --kubelet-use-node-status-port + - --metric-resolution=15s + + args: + - --kubelet-insecure-tls + + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app.kubernetes.io/name + operator: In + values: + - metrics-server + topologyKey: "kubernetes.io/hostname" diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/.helmignore b/tests/cluster-stacks/docker/valencia/cluster-class/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/Chart.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/Chart.yaml new file mode 100644 index 00000000..233c1344 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/Chart.yaml @@ -0,0 +1,11 @@ +apiVersion: v2 +description: | + This chart installs and configures: + * Docker Ferrol Cluster Class +maintainers: + - name: Syself + email: info@syself.com + url: https://github.com/syself +name: docker-ferrol-1-27-cluster-class +type: application +version: << .ClusterAddonVersion >> diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/_helpers.tpl b/tests/cluster-stacks/docker/valencia/cluster-class/templates/_helpers.tpl new file mode 100644 index 00000000..2339c125 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "cluster-class.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "cluster-class.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "cluster-class.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "cluster-class.labels" -}} +helm.sh/chart: {{ include "cluster-class.chart" . }} +{{ include "cluster-class.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "cluster-class.selectorLabels" -}} +app.kubernetes.io/name: {{ include "cluster-class.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "cluster-class.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "cluster-class.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/cluster-class.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/templates/cluster-class.yaml new file mode 100644 index 00000000..3f6cb505 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/cluster-class.yaml @@ -0,0 +1,101 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: ClusterClass +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }} + namespace: {{.Release.Namespace}} +spec: + controlPlane: + ref: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} + machineInfrastructure: + ref: + kind: DockerMachineTemplate + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerClusterTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} + workers: + machineDeployments: + - class: workeramd64 + template: + bootstrap: + ref: + apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 + kind: KubeadmConfigTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} + infrastructure: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} + variables: + - name: imageRepository + required: true + schema: + openAPIV3Schema: + type: string + default: "" + example: "registry.k8s.io" + description: "imageRepository sets the container registry to pull images from. If empty, nothing will be set and the from of kubeadm will be used." + patches: + - name: imageRepository + description: "Sets the imageRepository used for the KubeadmControlPlane." + enabledIf: '{{ ne .imageRepository "" }}' + definitions: + - selector: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlaneTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/kubeadmConfigSpec/clusterConfiguration/imageRepository" + valueFrom: + variable: imageRepository + - name: customImage + description: "Sets the container image that is used for running dockerMachines for the controlplane and worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - workeramd64 + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.controlPlane 0).name }} + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + controlPlane: true + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.controlPlane 0).name }} + - name: DockerMachineTemplateWorker + description: "Sets the container image that is used for running dockerMachines for the worker machineDeployments." + definitions: + - selector: + apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 + kind: DockerMachineTemplate + matchResources: + machineDeploymentClass: + names: + - workeramd64 + jsonPatches: + - op: add + path: "/spec/template/spec/customImage" + value: {{ (index .Values.images.worker 0).name }} diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-cluster-template.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-cluster-template.yaml new file mode 100644 index 00000000..0bdf7120 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-cluster-template.yaml @@ -0,0 +1,8 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerClusterTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-cluster + namespace: {{ .Release.Namespace }} +spec: + template: + spec: {} diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-machine-template.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-machine-template.yaml new file mode 100644 index 00000000..bc4c6cc6 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/docker-machine-template.yaml @@ -0,0 +1,11 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 +kind: DockerMachineTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-machinetemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + extraMounts: + - containerPath: "/var/run/docker.sock" + hostPath: "/var/run/docker.sock" diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-config-template-worker-docker.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-config-template-worker-docker.yaml new file mode 100644 index 00000000..861ccec4 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-config-template-worker-docker.yaml @@ -0,0 +1,15 @@ +apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 +kind: KubeadmConfigTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-worker-bootstraptemplate-docker + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + joinConfiguration: + nodeRegistration: + # We have to set the criSocket to containerd as kubeadm defaults to docker runtime if both containerd and docker sockets are found + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + fail-swap-on: "false" diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-control-plane-template.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-control-plane-template.yaml new file mode 100644 index 00000000..c76866e7 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/templates/kubeadm-control-plane-template.yaml @@ -0,0 +1,29 @@ +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlaneTemplate +metadata: + name: {{ .Release.Name }}-{{ .Chart.Version }}-control-plane + namespace: {{ .Release.Namespace }} +spec: + template: + spec: + kubeadmConfigSpec: + clusterConfiguration: + controllerManager: + extraArgs: { enable-hostpath-provisioner: 'true' } + apiServer: + # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied. + certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal] + initConfiguration: + nodeRegistration: + # We have to set the criSocket to containerd as kubeadm defaults to docker runtime if both containerd and docker sockets are found + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + fail-swap-on: "false" + joinConfiguration: + nodeRegistration: + # We have to set the criSocket to containerd as kubeadm defaults to docker runtime if both containerd and docker sockets are found + criSocket: unix:///var/run/containerd/containerd.sock + kubeletExtraArgs: + eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%' + fail-swap-on: "false" diff --git a/tests/cluster-stacks/docker/valencia/cluster-class/values.yaml b/tests/cluster-stacks/docker/valencia/cluster-class/values.yaml new file mode 100644 index 00000000..eb2bb411 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/cluster-class/values.yaml @@ -0,0 +1,5 @@ +images: + controlPlane: + - name: ghcr.io/sovereigncloudstack/docker-ferrol-1-27-controlplaneamd64:v1 + worker: + - name: ghcr.io/sovereigncloudstack/docker-ferrol-1-27-workeramd64:v1 diff --git a/tests/cluster-stacks/docker/valencia/clusteraddon.yaml b/tests/cluster-stacks/docker/valencia/clusteraddon.yaml new file mode 100644 index 00000000..73f52404 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/clusteraddon.yaml @@ -0,0 +1,24 @@ +apiVersion: clusteraddonconfig.x-k8s.io/v1alpha1 +clusterAddonVersion: clusteraddons.clusterstack.x-k8s.io/v1alpha1 +addonStages: + AfterControlPlaneInitialized: + - name: cni + action: apply + - name: metrics-server + action: apply + BeforeClusterUpgrade: + - name: ngnix + action: apply + waitForPostCondition: + objects: + - key: msdeployment + apiVersion: apps/v1 + kind: Deployment + name: metrics-server + namespace: kube-system + - key: ngnix + apiVersion: apps/v1 + kind: Deployment + name: cluster-addon-ngnix + namespace: kube-system + conditions: msdeployment.spec.replicas == ngnix.spec.replicas diff --git a/tests/cluster-stacks/docker/valencia/csctl.yaml b/tests/cluster-stacks/docker/valencia/csctl.yaml new file mode 100644 index 00000000..7558d624 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/csctl.yaml @@ -0,0 +1,7 @@ +apiVersion: csctl.clusterstack.x-k8s.io/v1alpha1 +config: + kubernetesVersion: v1.27.6 + clusterStackName: valencia + provider: + type: docker + apiVersion: docker.csctl.clusterstack.x-k8s.io/v1alpha1 diff --git a/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.controlplane b/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.controlplane new file mode 100644 index 00000000..58930156 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.controlplane @@ -0,0 +1,3 @@ +FROM kindest/node:v1.27.3 AS base +# This is an example. For a real infrastructure provider, you +# would have some scripts to create a custom machine image. \ No newline at end of file diff --git a/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.worker b/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.worker new file mode 100644 index 00000000..d2ce96d6 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/node-image/Dockerfile.worker @@ -0,0 +1,3 @@ +FROM kindest/node:v1.27.3 AS base +# This is an example. For a real infrastructure provider, you +# would have some scripts to create a custom machine image. diff --git a/tests/cluster-stacks/docker/valencia/node-image/scripts/base.sh b/tests/cluster-stacks/docker/valencia/node-image/scripts/base.sh new file mode 100755 index 00000000..f7493257 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/node-image/scripts/base.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +echo '--> Starting Base Installation.' +# Set locale +# localectl set-locale LANG=en_US.UTF-8 +# localectl set-locale LANGUAGE=en_US.UTF-8 + +# Suppress "WARNING: apt does not have a stable CLI interface. Use with caution in scripts." +echo 'Binary::apt::Apt::Cmd::Disable-Script-Warning "true";' >/etc/apt/apt.conf.d/disable-script-warning +export DEBIAN_FRONTEND=noninteractive + +# update all packages +apt update -y && apt -y upgrade && apt -y dist-upgrade + +# install basic tooling +apt -y install \ + at jq unzip wget socat mtr logrotate apt-transport-https network-manager + +# Install yq +YQ_VERSION=v4.30.5 #https://github.com/mikefarah/yq +YQ_BINARY=yq_linux_amd64 +wget -q https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${YQ_BINARY} -O /usr/bin/yq && + chmod +x /usr/bin/yq + +echo '--> Starting Base Configuration.' + +## disable swap +sed -i '/swap/d' /etc/fstab diff --git a/tests/cluster-stacks/docker/valencia/node-images.yaml b/tests/cluster-stacks/docker/valencia/node-images.yaml new file mode 100644 index 00000000..b19d1b71 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/node-images.yaml @@ -0,0 +1,4 @@ +apiVersion: docker.infrastructure.clusterstack.x-k8s.io/v1alpha1 +dockerImages: + - controlplaneamd64 + - workeramd64 diff --git a/tests/cluster-stacks/docker/valencia/topology-docker.yaml b/tests/cluster-stacks/docker/valencia/topology-docker.yaml new file mode 100644 index 00000000..14883841 --- /dev/null +++ b/tests/cluster-stacks/docker/valencia/topology-docker.yaml @@ -0,0 +1,26 @@ +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: "${CLUSTER_NAME}" + namespace: "${NAMESPACE}" +spec: + clusterNetwork: + services: + cidrBlocks: ["10.128.0.0/12"] + pods: + cidrBlocks: ["192.168.0.0/16"] + serviceDomain: "cluster.local" + topology: + class: docker-ferrol-1-27-v2 + controlPlane: + metadata: {} + replicas: 1 + variables: + - name: imageRepository + value: "" + version: v1.27.3 + workers: + machineDeployments: + - class: workeramd64 + name: md-0 + replicas: 1