From a55f4a3c44ba2cc3ebd5ff8e120d715c377a4242 Mon Sep 17 00:00:00 2001 From: Evgeny Slutsky Date: Thu, 29 Aug 2024 18:44:20 +0200 Subject: [PATCH 1/2] WIP: build runnable microshift bootc container Signed-off-by: Evgeny Slutsky --- microshift-okd/src/README.md | 27 ++++++++++++ microshift-okd/src/build.sh | 41 +++++++++++++++++++ microshift-okd/src/configure.sh | 16 ++++++++ microshift-okd/src/create_repos.sh | 37 +++++++++++++++++ .../src/microshift-okd-source.containerfile | 28 +++++++++++++ packaging/systemd/microshift.service | 4 +- 6 files changed, 151 insertions(+), 2 deletions(-) create mode 100644 microshift-okd/src/README.md create mode 100755 microshift-okd/src/build.sh create mode 100644 microshift-okd/src/configure.sh create mode 100644 microshift-okd/src/create_repos.sh create mode 100644 microshift-okd/src/microshift-okd-source.containerfile diff --git a/microshift-okd/src/README.md b/microshift-okd/src/README.md new file mode 100644 index 0000000000..79d1c32718 --- /dev/null +++ b/microshift-okd/src/README.md @@ -0,0 +1,27 @@ +## Build and Run Microshift upstream without subscription/pull-secret + +- building the container from src + > `cd microshift-okd/src && ./build.sh` + - this script will: + 1. replace microshift assets images to OKD upstream images + 1. will build microshift RPMs and repo based on current sources. + 1. will build micrsoshift_okd bootc container based on `centos-bootc:stream9` + 1. apply upstream customization (see below) + + +- running the container + > `sudo podman run --privileged --rm --name microshift-okd -d microshift-okd` + +- connect to the container + > `sudo podman exec -ti microshift-okd /bin/bash` + +## configuration customization +1. storage driver disabled (there is no lvms images upstream) +1. network CNI disabled (requires kernel modules) + - microshift service is not dependent on ovs + +## current state +- microshift service wont start because CNI is disabled. + - TODO: replace CNI with flannel .see this [PR](https://github.com/openshift/microshift/pull/3853) + - TODO: create rebase automation from OKD sources + diff --git a/microshift-okd/src/build.sh b/microshift-okd/src/build.sh new file mode 100755 index 0000000000..7dd45f40a8 --- /dev/null +++ b/microshift-okd/src/build.sh @@ -0,0 +1,41 @@ +#/bin/bash + + +# replace microshift assets to Upsteam from OKD +okd_url=quay.io/okd/scos-release +okd_releaseTag=4.17.0-0.okd-scos-2024-08-21-100712 + +oc adm release info ${okd_url}:${okd_releaseTag} >all_images + +for op in $(cat assets/release/release-x86_64.json | jq -e -r '.images | keys []') +do + image=$(oc adm release info --image-for=${op} ${okd_url}:${okd_releaseTag} || true) + if [ -n "${image}" ] ; then + echo "${op} ${image}" + jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' assets/release/release-x86_64.json >/tmp/release-x86_64.json.tmp + + + # delete openssl image from assets - just to verify if we still need it,since it doesnt referenced anywhere + jq '. | del (.images["openssl"])' assets/release/release-x86_64.json >/tmp/release-x86_64.json.tmp2 + + mv /tmp/release-x86_64.json.tmp2 assets/release/release-x86_64.json + fi +done + + +sudo podman stop microshift-okd +rm -rf microshift-okd/src/rpmbuild +rm -rf _output/rpmbuild +make build +make rpm +createrepo _output/rpmbuild/ +cp -rf _output/rpmbuild microshift-okd/src/ + +# build the container +sudo podman build -f microshift-okd/src/microshift-okd-source.containerfile -t microshift-okd + +# run container on the background +sudo podman run --privileged --rm --name microshift-okd -d microshift-okd + +# connect the container +sudo podman exec -ti microshift-okd /bin/bash \ No newline at end of file diff --git a/microshift-okd/src/configure.sh b/microshift-okd/src/configure.sh new file mode 100644 index 0000000000..1631520d63 --- /dev/null +++ b/microshift-okd/src/configure.sh @@ -0,0 +1,16 @@ +#!/bin/bash + + + cat > /etc/microshift/config.yaml < "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${CENTOS_NFV_SIG_REPO_FILE}" < Date: Mon, 9 Sep 2024 12:50:40 +0200 Subject: [PATCH 2/2] build rpms inside container with multistage build Signed-off-by: Evgeny Slutsky --- .dockerignore | 2 + microshift-okd/src/README.md | 27 ------- microshift-okd/src/build.sh | 41 ----------- microshift-okd/src/configure.sh | 16 ----- .../src/microshift-okd-source.containerfile | 28 -------- okd/src/README.md | 58 +++++++++++++++ okd/src/configure.sh | 8 +++ {microshift-okd => okd}/src/create_repos.sh | 14 +--- .../microshift-okd-multi-build.Containerfile | 55 ++++++++++++++ okd/src/use_okd_assets.sh | 72 +++++++++++++++++++ packaging/systemd/microshift.service | 4 +- scripts/devenv-builder/configure-vm.sh | 4 +- 12 files changed, 202 insertions(+), 127 deletions(-) create mode 100644 .dockerignore delete mode 100644 microshift-okd/src/README.md delete mode 100755 microshift-okd/src/build.sh delete mode 100644 microshift-okd/src/configure.sh delete mode 100644 microshift-okd/src/microshift-okd-source.containerfile create mode 100644 okd/src/README.md create mode 100644 okd/src/configure.sh rename {microshift-okd => okd}/src/create_repos.sh (54%) create mode 100644 okd/src/microshift-okd-multi-build.Containerfile create mode 100755 okd/src/use_okd_assets.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..5283323407 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +_output +.github \ No newline at end of file diff --git a/microshift-okd/src/README.md b/microshift-okd/src/README.md deleted file mode 100644 index 79d1c32718..0000000000 --- a/microshift-okd/src/README.md +++ /dev/null @@ -1,27 +0,0 @@ -## Build and Run Microshift upstream without subscription/pull-secret - -- building the container from src - > `cd microshift-okd/src && ./build.sh` - - this script will: - 1. replace microshift assets images to OKD upstream images - 1. will build microshift RPMs and repo based on current sources. - 1. will build micrsoshift_okd bootc container based on `centos-bootc:stream9` - 1. apply upstream customization (see below) - - -- running the container - > `sudo podman run --privileged --rm --name microshift-okd -d microshift-okd` - -- connect to the container - > `sudo podman exec -ti microshift-okd /bin/bash` - -## configuration customization -1. storage driver disabled (there is no lvms images upstream) -1. network CNI disabled (requires kernel modules) - - microshift service is not dependent on ovs - -## current state -- microshift service wont start because CNI is disabled. - - TODO: replace CNI with flannel .see this [PR](https://github.com/openshift/microshift/pull/3853) - - TODO: create rebase automation from OKD sources - diff --git a/microshift-okd/src/build.sh b/microshift-okd/src/build.sh deleted file mode 100755 index 7dd45f40a8..0000000000 --- a/microshift-okd/src/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#/bin/bash - - -# replace microshift assets to Upsteam from OKD -okd_url=quay.io/okd/scos-release -okd_releaseTag=4.17.0-0.okd-scos-2024-08-21-100712 - -oc adm release info ${okd_url}:${okd_releaseTag} >all_images - -for op in $(cat assets/release/release-x86_64.json | jq -e -r '.images | keys []') -do - image=$(oc adm release info --image-for=${op} ${okd_url}:${okd_releaseTag} || true) - if [ -n "${image}" ] ; then - echo "${op} ${image}" - jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' assets/release/release-x86_64.json >/tmp/release-x86_64.json.tmp - - - # delete openssl image from assets - just to verify if we still need it,since it doesnt referenced anywhere - jq '. | del (.images["openssl"])' assets/release/release-x86_64.json >/tmp/release-x86_64.json.tmp2 - - mv /tmp/release-x86_64.json.tmp2 assets/release/release-x86_64.json - fi -done - - -sudo podman stop microshift-okd -rm -rf microshift-okd/src/rpmbuild -rm -rf _output/rpmbuild -make build -make rpm -createrepo _output/rpmbuild/ -cp -rf _output/rpmbuild microshift-okd/src/ - -# build the container -sudo podman build -f microshift-okd/src/microshift-okd-source.containerfile -t microshift-okd - -# run container on the background -sudo podman run --privileged --rm --name microshift-okd -d microshift-okd - -# connect the container -sudo podman exec -ti microshift-okd /bin/bash \ No newline at end of file diff --git a/microshift-okd/src/configure.sh b/microshift-okd/src/configure.sh deleted file mode 100644 index 1631520d63..0000000000 --- a/microshift-okd/src/configure.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - - - cat > /etc/microshift/config.yaml < `sudo modprobe openvswitch` + + - run the container : + > `sudo podman run --privileged --rm --name microshift-okd -d microshift-okd` + +- connect to the container + > `sudo podman exec -ti microshift-okd /bin/bash` + +- verify everything is working: + ```bash + export KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig + > oc get nodes + NAME STATUS ROLES AGE VERSION + d2877aa41787 Ready control-plane,master,worker 7m39s v1.30.3 + + > oc get pods + NAMESPACE NAME READY STATUS RESTARTS AGE + kube-system csi-snapshot-controller-7d6c78bc58-5p7tb 1/1 Running 0 8m52s + kube-system csi-snapshot-webhook-5598db6db4-rmrpx 1/1 Running 0 8m54s + openshift-dns dns-default-2q89q 2/2 Running 0 7m34s + openshift-dns node-resolver-k2c5h 1/1 Running 0 8m54s + openshift-ingress router-default-db4b598b9-x8lvb 1/1 Running 0 8m52s + openshift-ovn-kubernetes ovnkube-master-c75c7 4/4 Running 1 (7m36s ago) 8m54s + openshift-ovn-kubernetes ovnkube-node-jfx86 1/1 Running 0 8m54s + openshift-service-ca service-ca-68d58669f8-rns2p 1/1 Running 0 8m51s + + + ``` + +## configuration customization +1. storage driver disabled (there is no lvms images upstream) - will be added in the stage of the project. + +## current state +- storage driver is disabled , will be added in the stage of the project. +- TODO: create rebase automation from OKD sources + +## known Issues +- when running `podman build` without sudo + ``` + make: *** [/src/vendor/github.com/openshift/build-machinery-go/make/targets/golang/build.mk:16: build] Error 1 + Error: building at STEP "RUN make build": while running runtime: exit status 2 + ``` + diff --git a/okd/src/configure.sh b/okd/src/configure.sh new file mode 100644 index 0000000000..248ebece24 --- /dev/null +++ b/okd/src/configure.sh @@ -0,0 +1,8 @@ +#!/bin/bash + + cat > /etc/microshift/config.yaml < "${USHIFT_LOCAL_REPO_FILE}" < "${OCP_MIRROR_REPO_FILE}" < "${CENTOS_NFV_SIG_REPO_FILE}" </etc/sudoers.d/microshift +COPY . /src +RUN chown -R microshift:microshift /microshift /src + +USER 1000:1000 +WORKDIR /src +# Preparing for the build +RUN echo '{"auths":{"fake":{"auth":"aWQ6cGFzcwo="}}}' > /tmp/.pull-secret && \ + /src/scripts/devenv-builder/configure-vm.sh --no-build --no-set-release-version --skip-dnf-update /tmp/.pull-secret && \ + /src/okd/src/use_okd_assets.sh --replace ${OKD_REPO} ${OKD_VERSION_TAG} + +# Building Microshift RPMs and local repo +RUN make build && \ + make rpm && \ + createrepo ${REPO_DIR} + +# Building microshift container from local rpms +FROM quay.io/centos-bootc/centos-bootc:stream9 +ARG REPO_CONFIG_SCRIPT=/tmp/create_repos.sh +ARG OKD_CONFIG_SCRIPT=/tmp/configure.sh +ARG USHIFT_RPM_REPO_NAME=microshift-local +ARG USHIFT_RPM_REPO_PATH=/tmp/rpm-repo + +ENV KUBECONFIG=/var/lib/microshift/resources/kubeadmin/kubeconfig +COPY --chmod=755 ./okd/src/create_repos.sh ${REPO_CONFIG_SCRIPT} +COPY --chmod=755 ./okd/src/configure.sh ${OKD_CONFIG_SCRIPT} +COPY --from=builder /src/_output/rpmbuild/RPMS ${USHIFT_RPM_REPO_PATH} + +# Installing MicroShift and cleanup +RUN ${REPO_CONFIG_SCRIPT} ${USHIFT_RPM_REPO_PATH} && \ + dnf install -y microshift && \ + ${REPO_CONFIG_SCRIPT} -delete && \ + rm -f ${REPO_CONFIG_SCRIPT} && \ + rm -rf $USHIFT_RPM_REPO_PATH && \ + dnf clean all + +RUN ${OKD_CONFIG_SCRIPT} && rm -rf ${OKD_CONFIG_SCRIPT} + +# Create a systemd unit to recursively make the root filesystem subtree +# shared as required by OVN images +COPY ./packaging/imagemode/systemd/microshift-make-rshared.service /etc/systemd/system/microshift-make-rshared.service +RUN systemctl enable microshift-make-rshared.service \ No newline at end of file diff --git a/okd/src/use_okd_assets.sh b/okd/src/use_okd_assets.sh new file mode 100755 index 0000000000..699f05f0cf --- /dev/null +++ b/okd/src/use_okd_assets.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -eo pipefail + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +MICROSHIFT_ROOT="${SCRIPTDIR}/../.." + +declare -A UNAME_TO_GOARCH_MAP=( ["x86_64"]="amd64" ["aarch64"]="arm64" ) + + +verify(){ + local -r okd_url=$1 + local -r okd_releaseTag=$2 + + #stdout=$(oc adm release info "${okd_url}:${okd_releaseTag}" 2>&1) + if ! stdout=$(oc adm release info "${okd_url}:${okd_releaseTag}" 2>&1) ; then + echo -e "error verifying okd release (URL: ${okd_url} , TAG: ${okd_releaseTag}) \nERROR: ${stdout}" + exit 1 + fi +} + +replace_assets(){ + local -r okd_url=$1 + local -r okd_releaseTag=$2 + local -r arch=$(uname -m) + local -r temp_release_json=$(mktemp "/tmp/release-${arch}.XXXXX.json") + + oc adm release info --image-for="${op}" "${okd_url}:${okd_releaseTag}" + + # replace Microshift images with upstream (from OKD release) + for op in $(jq -e -r '.images | keys []' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json") + do + local image + image=$(oc adm release info --image-for="${op}" "${okd_url}:${okd_releaseTag}" || true) + if [ -n "${image}" ] ; then + echo "${op} ${image}" + jq --arg a "${op}" --arg b "${image}" '.images[$a] = $b' "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" >"${temp_release_json}" + mv "${temp_release_json}" "${MICROSHIFT_ROOT}/assets/release/release-${arch}.json" + fi + done + + pod_image=$(oc adm release info --image-for=pod "${okd_url}:${okd_releaseTag}" || true) + # update the infra pods for crio + sed -i 's,pause_image .*,pause_image = '"\"${pod_image}\""',' "packaging/crio.conf.d/10-microshift_${UNAME_TO_GOARCH_MAP[${arch}]}.conf" + +} + +usage() { + echo "Usage:" + echo "$(basename "$0") --verify OKD_URL RELEASE_TAG verify upstream release" + echo "$(basename "$0") --replace OKD_URL RELEASE_TAG replace microshift assets with upstream images" + exit 1 +} + +if [ $# -eq 3 ] ; then + case "$1" in + --replace) + verify "$2" "$3" + replace_assets "$2" "$3" + ;; + --verify) + verify "$2" "$3" + ;; + *) + usage + ;; + esac +else + usage +fi + + diff --git a/packaging/systemd/microshift.service b/packaging/systemd/microshift.service index a33b5a3ae8..f97db77074 100644 --- a/packaging/systemd/microshift.service +++ b/packaging/systemd/microshift.service @@ -1,7 +1,7 @@ [Unit] Description=MicroShift -Wants=network-online.target crio.service -After=network-online.target crio.service +Wants=network-online.target crio.service openvswitch.service microshift-ovs-init.service +After=network-online.target crio.service openvswitch.service microshift-ovs-init.service # Control shutdown order by declaring this service to start Before the kubepods.slice # transient systemd unit; this makes system shutdown delay MicroShift shutdown until diff --git a/scripts/devenv-builder/configure-vm.sh b/scripts/devenv-builder/configure-vm.sh index 9d817458e7..bd1446078e 100755 --- a/scripts/devenv-builder/configure-vm.sh +++ b/scripts/devenv-builder/configure-vm.sh @@ -137,7 +137,9 @@ if ${INSTALL_BUILD_DEPS} || ${BUILD_AND_RUN}; then "${DNF_RETRY}" "update" fi "${DNF_RETRY}" "install" "gcc git golang cockpit make jq selinux-policy-devel rpm-build jq bash-completion avahi-tools createrepo" - sudo systemctl enable --now cockpit.socket + + # run only if booted with systemd + [[ -d /run/systemd/system ]] && sudo systemctl enable --now cockpit.socket fi GO_VER=1.21.3 # released 2023-10-10 (matches CI images)