Integrate openeuler images into CI/CD (opea-project#1842)

chensuyue · pre-commit-ci[bot] · jlwang1996 · commit 571c0319ca74 · 2025-12-18T11:15:16.000+08:00
* build image for openeuler Signed-off-by: chensuyue <suyue.chen@intel.com> * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Handle the image name check issue in image build yaml Signed-off-by: chensuyue <suyue.chen@intel.com> * Support for openEuler (opea-project#1813) * add openEuler support and update some tests Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * add openEuler support for NGINX Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * update copyright and add test for nginx Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> --------- Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * build images for openeuler Signed-off-by: chensuyue <suyue.chen@intel.com> * fix the dockerfile hadolint scan issue (opea-project#1850) Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * update dataprep Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> --------- Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * Fix some bugs in Embedding and Guardrails (opea-project#1858) * fix the dockerfile hadolint scan issue Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * update dataprep Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> * fix some bugs Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> --------- Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> * guardrails-openeuler image build Signed-off-by: chensuyue <suyue.chen@intel.com> * update conda env activate step Signed-off-by: chensuyue <suyue.chen@intel.com> * use local cache for BRIDGE_TOWER_MODEL Signed-off-by: chensuyue <suyue.chen@intel.com> --------- Signed-off-by: chensuyue <suyue.chen@intel.com> Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Signed-off-by: sunzhonghua2004 <137033036@qq.com>
diff --git a/.github/workflows/docker/compose/dataprep-compose.yaml b/.github/workflows/docker/compose/dataprep-compose.yaml
@@ -7,3 +7,7 @@ services:
     build:
       dockerfile: comps/dataprep/src/Dockerfile
     image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}
+  dataprep-openeuler:
+    build:
+      dockerfile: comps/dataprep/src/Dockerfile.openEuler
+    image: ${REGISTRY:-opea}/dataprep:${TAG:-latest}-openeuler
diff --git a/.github/workflows/docker/compose/embeddings-compose.yaml b/.github/workflows/docker/compose/embeddings-compose.yaml
@@ -7,3 +7,7 @@ services:
     build:
       dockerfile: comps/embeddings/src/Dockerfile
     image: ${REGISTRY:-opea}/embedding:${TAG:-latest}
+  embedding-openeuler:
+    build:
+      dockerfile: comps/embeddings/src/Dockerfile.openEuler
+    image: ${REGISTRY:-opea}/embedding:${TAG:-latest}-openeuler
diff --git a/.github/workflows/docker/compose/rerankings-compose.yaml b/.github/workflows/docker/compose/rerankings-compose.yaml
@@ -7,3 +7,7 @@ services:
     build:
       dockerfile: comps/rerankings/src/Dockerfile
     image: ${REGISTRY:-opea}/reranking:${TAG:-latest}
+  reranking-openeuler:
+    build:
+      dockerfile: comps/rerankings/src/Dockerfile.openEuler
+    image: ${REGISTRY:-opea}/reranking:${TAG:-latest}-openeuler
diff --git a/.github/workflows/docker/compose/retrievers-compose.yaml b/.github/workflows/docker/compose/retrievers-compose.yaml
@@ -7,3 +7,7 @@ services:
     build:
       dockerfile: comps/retrievers/src/Dockerfile
     image: ${REGISTRY:-opea}/retriever:${TAG:-latest}
+  retriever-openeuler:
+    build:
+      dockerfile: comps/retrievers/src/Dockerfile.openEuler
+    image: ${REGISTRY:-opea}/retriever:${TAG:-latest}-openeuler
diff --git a/.github/workflows/docker/compose/third_parties-compose.yaml b/.github/workflows/docker/compose/third_parties-compose.yaml
@@ -6,6 +6,10 @@ services:
     build:
       dockerfile: comps/third_parties/nginx/src/Dockerfile
     image: ${REGISTRY:-opea}/nginx:${TAG:-latest}
+  nginx-openeuler:
+    build:
+      dockerfile: comps/third_parties/nginx/src/Dockerfile.openEuler
+    image: ${REGISTRY:-opea}/nginx:${TAG:-latest}-openeuler
   embedding-multimodal-clip:
     build:
       dockerfile: comps/third_parties/clip/src/Dockerfile
diff --git a/.github/workflows/pr-dockerfile-path-scan.yaml b/.github/workflows/pr-dockerfile-path-scan.yaml
@@ -167,7 +167,7 @@ jobs:
           consistency="TRUE"
           yamls=$(find .github/workflows/docker/compose/ -name '*-compose.yaml')
           for build_yaml in $yamls; do
-            message=$(python3 .github/workflows/scripts/check-name-agreement.py "$build_yaml")
+            message=$(python3 .github/workflows/scripts/check_name_agreement.py "$build_yaml")
             if [[ "$message" != *"consistent"* ]]; then
               consistency="FALSE"
               echo "Inconsistent service name and image name found in file $build_yaml."
diff --git a/.github/workflows/scripts/check_name_agreement.py b/.github/workflows/scripts/check_name_agreement.py
@@ -19,8 +19,10 @@ def check_service_image_consistency(data):
         image_name = service_details.get("image", "")
         # Extract the image name part after the last '/'
         image_name_part = image_name.split("/")[-1].split(":")[0]
+        # Remove '-openeuler' suffix if it exists
+        fixed_service_name = service_name.rsplit("-openeuler", 1)[0]
         # Check if the service name is a substring of the image name part
-        if service_name not in image_name_part:
+        if fixed_service_name not in image_name_part:
             # Get the line number of the service name
             line_number = service_details.lc.line + 1
             inconsistencies.append((service_name, image_name, line_number))
diff --git a/.github/workflows/scripts/codeScan/.hadolint.yaml b/.github/workflows/scripts/codeScan/.hadolint.yaml
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 ignored:
+  - DL3033
   - DL3003
   - DL3006
   - DL3007
diff --git a/.github/workflows/scripts/get_test_matrix.sh b/.github/workflows/scripts/get_test_matrix.sh
@@ -54,10 +54,23 @@ function find_test_1() {
                     done
                 fi
                 if [ "$run_all_interation" = "true" ]; then
+                    set -x
                     find_test=$(find ./tests -type f -name "test_${service_name}*.sh") || true
                     if [ "$find_test" ]; then
-                        fill_in_matrix "$find_test"
+                        # limit test case. If the changed file only to be Dockerfile.openEuler, then only run the script integrate related test.
+                        check_files_except_openeuler_docker=$(printf '%s\n' "${changed_files[@]}"| grep ${service_path} | grep -vE 'Dockerfile.openEuler' | sort -u) || true
+                        if [ -z "$check_files_except_openeuler_docker" ]; then
+                            for test in ${find_test}; do
+                                find_dockerfile=$(grep -rl "openEuler" $test) || true
+                                if [ "$find_dockerfile" ]; then
+                                    fill_in_matrix "$test"
+                                fi
+                            done
+                        else
+                            fill_in_matrix "$find_test"
+                        fi
                     fi
+                    set +x
                 fi
             elif [[ $(echo ${service_path} | grep "third_parties") ]]; then
                 # new org with `src` and `third_parties` folder
diff --git a/Dockerfile.openEuler b/Dockerfile.openEuler
@@ -0,0 +1,38 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+# Base image for GenAIComps based OPEA Python applications
+# Build: docker build -t opea/comps-base:{version}-openeuler -f Dockerfile.openEuler .
+
+ARG IMAGE_NAME=openeuler/python
+ARG IMAGE_TAG=3.11.13-oe2403lts
+
+FROM ${IMAGE_NAME}:${IMAGE_TAG} AS base
+
+ENV HOME=/home/user
+
+RUN yum update -y && \
+    yum upgrade -y && \
+    yum install -y shadow && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p $HOME && \
+    chown -R user $HOME
+
+WORKDIR $HOME
+
+COPY *.toml *.py *.txt *.md LICENSE ./
+
+ARG uvpip='uv pip install --system --no-cache-dir'
+RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
+    $uvpip -r requirements.txt
+
+COPY comps/ comps/
+
+ENV PYTHONPATH=$PYTHONPATH:$HOME
+
+USER user
+
+ENTRYPOINT ["sh", "-c", "set && ls -la"]
diff --git a/comps/dataprep/src/Dockerfile.openEuler b/comps/dataprep/src/Dockerfile.openEuler
@@ -0,0 +1,66 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd. 
+# SPDX-License-Identifier: Apache-2.0
+
+FROM openeuler/python:3.11.13-oe2403lts
+
+ENV LANG=C.UTF-8
+
+ARG ARCH="cpu"
+
+RUN yum update -y && yum install -y \
+    gcc g++ make cmake \
+    java-1.8.0-openjdk \
+    cairo \
+    libxslt \
+    mesa-libGL \
+    jemalloc-devel \
+    mariadb-connector-c-devel-3.3.8 \
+    libpq-devel \
+    poppler-utils \
+    tesseract-tools \
+    ffmpeg \
+    wget curl \
+    shadow && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+ENV TESSDATA_PREFIX=/usr/share/tesseract/tessdata
+
+RUN LIBREOFFICE_URL=https://mirrors.tuna.tsinghua.edu.cn/libreoffice/libreoffice/stable/25.2.4/rpm/x86_64/LibreOffice_25.2.4_Linux_x86-64_rpm.tar.gz && \
+    wget $LIBREOFFICE_URL -O /tmp/libreOffice.tar.gz && \
+    tar --strip-components=1 -xvf /tmp/libreOffice.tar.gz -C /tmp && \
+    yum install -y /tmp/RPMS/*.rpm && \
+    yum clean all && \
+    rm -fr /tmp/libreOffice.tar.gz /tmp/RPMS && \
+    ln -sf /usr/bin/libreoffice25.2 /usr/bin/libreoffice
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p /home/user && \
+    chown -R user /home/user/
+
+COPY comps /home/user/comps
+
+ARG uvpip='uv pip install --system --no-cache-dir'
+RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
+    if [ ${ARCH} = "cpu" ]; then \
+      $uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
+      $uvpip -r /home/user/comps/dataprep/src/requirements-cpu.txt; \
+    else \
+      $uvpip -r /home/user/comps/dataprep/src/requirements-gpu.txt; \
+    fi
+
+ENV PYTHONPATH=$PYTHONPATH:/home/user
+
+RUN mkdir -p /home/user/comps/dataprep/src/uploaded_files && chown -R user /home/user/comps/dataprep/src/uploaded_files
+RUN mkdir -p /data && chown -R user /data
+
+USER user
+ENV NLTK_DATA=/home/user/nltk_data
+# air gapped support: predownload all needed nltk data
+RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab averaged_perceptron_tagger_eng stopwords
+# air gapped support: set model cache dir
+ENV HF_HUB_CACHE=/data
+
+WORKDIR /home/user/comps/dataprep/src
+
+ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_DATAPREP\" = \"true\" ] && echo 'opea_dataprep_multimodal_microservice.py' || echo 'opea_dataprep_microservice.py')"]
diff --git a/comps/embeddings/src/Dockerfile.openEuler b/comps/embeddings/src/Dockerfile.openEuler
@@ -0,0 +1,19 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd. 
+# SPDX-License-Identifier: Apache-2.0
+
+FROM openeuler/python:3.11.13-oe2403lts
+
+COPY comps /home/comps
+
+ARG uvpip='uv pip install --system --no-cache-dir'
+RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
+    $uvpip -r /home/comps/embeddings/src/requirements.txt
+
+ENV PYTHONPATH=$PYTHONPATH:/home
+
+WORKDIR /home/comps/embeddings/src/
+
+ENV MULTIMODAL_EMBEDDING="false"
+
+ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_EMBEDDING\" = \"true\" ] && echo 'opea_multimodal_embedding_microservice.py' || echo 'opea_embedding_microservice.py')"]
+
diff --git a/comps/rerankings/src/Dockerfile.openEuler b/comps/rerankings/src/Dockerfile.openEuler
@@ -0,0 +1,40 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd. 
+# SPDX-License-Identifier: Apache-2.0
+
+FROM openeuler/python:3.11.13-oe2403lts
+
+ENV LANG=C.UTF-8
+
+ARG ARCH="cpu"
+
+RUN yum update -y && yum install -y \
+    git \
+    mesa-libGL \
+    jemalloc-devel \
+    shadow && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p /home/user && \
+    chown -R user /home/user/
+
+COPY comps/rerankings/src/requirements* /home/user/comps/rerankings/src/
+
+ARG uvpip='uv pip install --system --no-cache-dir'
+RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
+    if [ ${ARCH} = "cpu" ]; then \
+      $uvpip torch --index-url https://download.pytorch.org/whl/cpu; \
+      $uvpip -r /home/user/comps/rerankings/src/requirements-cpu.txt; \
+    else \
+      $uvpip -r /home/user/comps/rerankings/src/requirements-gpu.txt; \
+    fi
+
+COPY comps /home/user/comps
+ENV PYTHONPATH=$PYTHONPATH:/home/user
+
+USER user
+
+WORKDIR /home/user/comps/rerankings/src
+
+ENTRYPOINT ["python", "opea_reranking_microservice.py"]
diff --git a/comps/retrievers/src/Dockerfile.openEuler b/comps/retrievers/src/Dockerfile.openEuler
@@ -0,0 +1,43 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM openeuler/python:3.11.13-oe2403lts
+
+ARG ARCH="cpu"
+
+RUN yum update -y && yum install -y \
+    cairo \
+    mesa-libGL \
+    glib2 \
+    jemalloc-devel \
+    mariadb-connector-c-devel-3.3.8 \
+    gcc g++ make cmake \
+    shadow && \
+    yum clean all && \
+    rm -rf /var/cache/yum
+
+RUN useradd -m -s /bin/bash user && \
+    mkdir -p /home/user && \
+    chown -R user /home/user/
+
+COPY comps /home/user/comps
+
+ARG uvpip='uv pip install --system --no-cache-dir'
+RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
+    if [ ${ARCH} = "cpu" ]; then \
+      $uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
+      $uvpip -r /home/user/comps/retrievers/src/requirements-cpu.txt; \
+    else \
+      $uvpip -r /home/user/comps/retrievers/src/requirements-gpu.txt; \
+    fi
+
+ENV PYTHONPATH=$PYTHONPATH:/home/user
+
+USER user
+ENV NLTK_DATA=/home/user/nltk_data
+# air gapped support: predownload all needed nltk data
+RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab stopwords
+
+WORKDIR /home/user/comps/retrievers/src
+
+ENTRYPOINT ["python", "opea_retrievers_microservice.py"]
diff --git a/comps/third_parties/nginx/src/Dockerfile.openEuler b/comps/third_parties/nginx/src/Dockerfile.openEuler
@@ -0,0 +1,25 @@
+# Copyright (C) 2025 Huawei Technologies Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+
+FROM openeuler/nginx:1.29.0-oe2403lts
+
+RUN yum update -y && \
+    yum install -y gettext && \
+    yum clean all && \
+    rm -fr /var/cache/yum
+
+COPY comps/third_parties/nginx/src/nginx.conf.template /etc/nginx/nginx.conf.template
+
+ENV FRONTEND_SERVICE_IP=localhost
+ENV FRONTEND_SERVICE_PORT=5173
+ENV BACKEND_SERVICE_NAME=chatqna
+ENV BACKEND_SERVICE_IP=localhost
+ENV BACKEND_SERVICE_PORT=8888
+ENV DATAPREP_SERVICE_IP=localhost
+ENV DATAPREP_SERVICE_PORT=6007
+
+COPY comps/third_parties/nginx/deployment/docker_compose/start-nginx.sh /usr/local/bin/start-nginx.sh
+RUN chmod +x /usr/local/bin/start-nginx.sh
+
+CMD ["/usr/local/bin/start-nginx.sh"]
diff --git a/tests/dataprep/test_dataprep_redis.sh b/tests/dataprep/test_dataprep_redis.sh
@@ -2,7 +2,7 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-set -x
+set -xe
 
 WORKPATH=$(dirname "$PWD")
 LOG_PATH="$WORKPATH/tests"
@@ -18,7 +18,8 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
 function build_docker_images() {
     cd $WORKPATH
     echo $(pwd)
-    docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
+    dockerfile_name="comps/dataprep/src/$1"
+    docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
     if [ $? -ne 0 ]; then
         echo "opea/dataprep built fail"
         exit 1
@@ -118,21 +119,14 @@ function validate_microservice() {
 
 }
 
-function stop_docker() {
-    cid=$(docker ps -aq --filter "name=dataprep-redis-server*" --filter "name=redis-vector-*" --filter "name=tei-embedding-*")
-    if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
-}
-
 function stop_service() {
     cd $WORKPATH/comps/dataprep/deployment/docker_compose/
     docker compose down || true
 }
 
 function main() {
 
-    stop_docker
-
-    build_docker_images
+    build_docker_images "Dockerfile"
     trap stop_service EXIT
 
     echo "Test normal env ..."
@@ -148,8 +142,13 @@ function main() {
         stop_service
     fi
 
-    stop_docker
-    echo y | docker system prune
+    echo "Test with openEuler OS ..."
+    build_docker_images "Dockerfile.openEuler"
+    start_service
+    validate_microservice
+    stop_service
+
+    docker system prune -f
 
 }
 
diff --git a/tests/embeddings/test_embeddings_tei.sh b/tests/embeddings/test_embeddings_tei.sh
diff --git a/tests/rerankings/test_rerankings_tei.sh b/tests/rerankings/test_rerankings_tei.sh
diff --git a/tests/retrievers/test_retrievers_redis.sh b/tests/retrievers/test_retrievers_redis.sh
diff --git a/tests/third_parties/test_third_parties_nginx.sh b/tests/third_parties/test_third_parties_nginx.sh
