Skip to content

Commit 1db0d6b

Browse files
zhihangdengzhihangdeng
committed
add support for openEuler and update tests
Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn>
1 parent 3a9cf90 commit 1db0d6b

File tree

11 files changed

+292
-45
lines changed

11 files changed

+292
-45
lines changed

‎.github/workflows/scripts/codeScan/.hadolint.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
# SPDX-License-Identifier: Apache-2.0
33

44
ignored:
5+
- DL3033
56
- DL3003
67
- DL3006
78
- DL3007

‎comps/dataprep/src/Dockerfile.openEuler‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
ENV LANG=C.UTF-8
7+
8+
ARG ARCH="cpu"
9+
10+
RUN yum update -y && yum install -y \
11+
gcc g++ make cmake \
12+
java-1.8.0-openjdk \
13+
cairo \
14+
mesa-libGL \
15+
jemalloc-devel \
16+
mariadb-connector-c-devel-3.3.8 \
17+
libpq-devel \
18+
poppler-utils \
19+
tesseract \
20+
ffmpeg \
21+
wget curl \
22+
shadow && \
23+
yum clean all && \
24+
rm -rf /var/cache/yum
25+
26+
RUN LIBREOFFICE_URL=https://mirrors.bfsu.edu.cn/libreoffice/libreoffice/stable/24.8.6/rpm/x86_64/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz && \
27+
wget $LIBREOFFICE_URL -O /tmp/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz && \
28+
tar -xvf /tmp/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz -C /tmp && \
29+
yum install -y /tmp/LibreOffice_24.8.6.2_Linux_x86-64_rpm/RPMS/*.rpm && \
30+
rm -fr /tmp/LibreOffice*
31+
32+
RUN useradd -m -s /bin/bash user && \
33+
mkdir -p /home/user && \
34+
chown -R user /home/user/
35+
36+
COPY comps /home/user/comps
37+
38+
ARG uvpip='uv pip install --system --no-cache-dir'
39+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
40+
if [ ${ARCH} = "cpu" ]; then \
41+
$uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
42+
$uvpip -r /home/user/comps/dataprep/src/requirements-cpu.txt; \
43+
else \
44+
$uvpip -r /home/user/comps/dataprep/src/requirements-gpu.txt; \
45+
fi
46+
47+
ENV PYTHONPATH=$PYTHONPATH:/home/user
48+
49+
RUN mkdir -p /home/user/comps/dataprep/src/uploaded_files && chown -R user /home/user/comps/dataprep/src/uploaded_files
50+
RUN mkdir -p /data && chown -R user /data
51+
52+
USER user
53+
ENV NLTK_DATA=/home/user/nltk_data
54+
# air gapped support: predownload all needed nltk data
55+
RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab averaged_perceptron_tagger_eng stopwords
56+
# air gapped support: set model cache dir
57+
ENV HF_HUB_CACHE=/data
58+
59+
WORKDIR /home/user/comps/dataprep/src
60+
61+
ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_DATAPREP\" = \"true\" ] && echo 'opea_dataprep_multimodal_microservice.py' || echo 'opea_dataprep_microservice.py')"]

‎comps/embeddings/src/Dockerfile.openEuler‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
COPY comps /home/comps
7+
8+
ARG uvpip='uv pip install --system --no-cache-dir'
9+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
10+
$uvpip -r /home/comps/embeddings/src/requirements.txt
11+
12+
ENV PYTHONPATH=$PYTHONPATH:/home
13+
14+
WORKDIR /home/comps/embeddings/src/
15+
16+
ENV MULTIMODAL_EMBEDDING="false"
17+
18+
ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_EMBEDDING\" = \"true\" ] && echo 'opea_multimodal_embedding_microservice.py' || echo 'opea_embedding_microservice.py')"]
19+

‎comps/guardrails/src/guardrails/Dockerfile.openEuler‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
ENV LANG=C.UTF-8
7+
8+
ARG ARCH="cpu"
9+
10+
RUN yum update -y && yum install -y \
11+
mesa-libGL \
12+
jemalloc-devel \
13+
shadow && \
14+
yum clean all && \
15+
rm -rf /var/cache/yum
16+
17+
RUN useradd -m -s /bin/bash user && \
18+
mkdir -p /home/user && \
19+
chown -R user /home/user/
20+
21+
COPY comps /home/user/comps
22+
23+
ARG uvpip='uv pip install --system --no-cache-dir'
24+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
25+
if [ ${ARCH} = "cpu" ]; then \
26+
$uvpip torch --index-url https://download.pytorch.org/whl/cpu; \
27+
$uvpip -r /home/user/comps/guardrails/src/guardrails/requirements-cpu.txt; \
28+
else \
29+
$uvpip -r /home/user/comps/guardrails/src/guardrails/requirements-gpu.txt; \
30+
fi
31+
32+
ENV PYTHONPATH=$PYTHONPATH:/home/user
33+
34+
USER user
35+
36+
WORKDIR /home/user/comps/guardrails/src/guardrails/
37+
38+
ENTRYPOINT ["python", "opea_guardrails_microservice.py"]

‎comps/rerankings/src/Dockerfile.openEuler‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
ENV LANG=C.UTF-8
7+
8+
ARG ARCH="cpu"
9+
10+
RUN yum update -y && yum install -y \
11+
git \
12+
mesa-libGL \
13+
jemalloc-devel \
14+
shadow && \
15+
yum clean all && \
16+
rm -rf /var/cache/yum
17+
18+
RUN useradd -m -s /bin/bash user && \
19+
mkdir -p /home/user && \
20+
chown -R user /home/user/
21+
22+
COPY comps/rerankings/src/requirements* /home/user/comps/rerankings/src/
23+
24+
ARG uvpip='uv pip install --system --no-cache-dir'
25+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
26+
if [ ${ARCH} = "cpu" ]; then \
27+
$uvpip torch --index-url https://download.pytorch.org/whl/cpu; \
28+
$uvpip -r /home/user/comps/rerankings/src/requirements-cpu.txt; \
29+
else \
30+
$uvpip -r /home/user/comps/rerankings/src/requirements-gpu.txt; \
31+
fi
32+
33+
COPY comps /home/user/comps
34+
ENV PYTHONPATH=$PYTHONPATH:/home/user
35+
36+
USER user
37+
38+
WORKDIR /home/user/comps/rerankings/src
39+
40+
ENTRYPOINT ["python", "opea_reranking_microservice.py"]

‎comps/retrievers/src/Dockerfile.openEuler‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
ARG ARCH="cpu"
7+
8+
RUN yum update -y && yum install -y \
9+
cairo \
10+
mesa-libGL \
11+
glib2 \
12+
jemalloc-devel \
13+
mariadb-connector-c-devel-3.3.8 \
14+
gcc g++ make cmake \
15+
shadow && \
16+
yum clean all && \
17+
rm -rf /var/cache/yum
18+
19+
RUN useradd -m -s /bin/bash user && \
20+
mkdir -p /home/user && \
21+
chown -R user /home/user/
22+
23+
COPY comps /home/user/comps
24+
25+
ARG uvpip='uv pip install --system --no-cache-dir'
26+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
27+
if [ ${ARCH} = "cpu" ]; then \
28+
$uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
29+
$uvpip -r /home/user/comps/retrievers/src/requirements-cpu.txt; \
30+
else \
31+
$uvpip -r /home/user/comps/retrievers/src/requirements-gpu.txt; \
32+
fi
33+
34+
ENV PYTHONPATH=$PYTHONPATH:/home/user
35+
36+
USER user
37+
ENV NLTK_DATA=/home/user/nltk_data
38+
# air gapped support: predownload all needed nltk data
39+
RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab stopwords
40+
41+
WORKDIR /home/user/comps/retrievers/src
42+
43+
ENTRYPOINT ["python", "opea_retrievers_microservice.py"]

‎tests/dataprep/test_dataprep_redis.sh‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,8 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
1818
function build_docker_images() {
1919
cd $WORKPATH
2020
echo $(pwd)
21-
docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
21+
dockerfile_name="comps/dataprep/src/$1"
22+
docker build -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
2223
if [ $? -ne 0 ]; then
2324
echo "opea/dataprep built fail"
2425
exit 1
@@ -118,21 +119,15 @@ function validate_microservice() {
118119

119120
}
120121

121-
function stop_docker() {
122-
cid=$(docker ps -aq --filter "name=dataprep-redis-server*" --filter "name=redis-vector-*" --filter "name=tei-embedding-*")
123-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
124-
}
125-
126122
function stop_service() {
127123
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
128124
docker compose down || true
129125
}
130126

131127
function main() {
132128

133-
stop_docker
134129

135-
build_docker_images
130+
build_docker_images "Dockerfile"
136131
trap stop_service EXIT
137132

138133
echo "Test normal env ..."
@@ -148,8 +143,13 @@ function main() {
148143
stop_service
149144
fi
150145

151-
stop_docker
152-
echo y | docker system prune
146+
echo "Test with openEuler OS ..."
147+
build_docker_images "Dockerfile.openEuler"
148+
start_service
149+
validate_microservice
150+
stop_service
151+
152+
docker system prune -f
153153

154154
}
155155

‎tests/embeddings/test_embeddings_tei.sh‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,8 @@ export DATA_PATH=${model_cache}
1111
function build_docker_images() {
1212
cd $WORKPATH
1313
echo $(pwd)
14-
docker build --no-cache -t opea/embedding:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/embeddings/src/Dockerfile .
14+
dockerfile_name="comps/embeddings/src/$1"
15+
docker build --no-cache -t opea/embedding:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
1516
if [ $? -ne 0 ]; then
1617
echo "opea/embedding built fail"
1718
exit 1
@@ -73,22 +74,31 @@ function validate_microservice_with_openai() {
7374
}
7475

7576
function stop_docker() {
76-
cid=$(docker ps -aq --filter "name=tei-embedding-*")
77-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
77+
function stop_service() {
78+
cd $WORKPATH/comps/embeddings/deployment/docker_compose/
79+
docker compose down || true
80+
}
7881
}
7982

8083
function main() {
8184

82-
stop_docker
85+
build_docker_images "Dockerfile"
86+
trap stop_service EXIT
8387

84-
build_docker_images
88+
echo "Test normal env ..."
8589
start_service
90+
validate_microservice
91+
validate_microservice_with_openai
92+
stop_service
8693

94+
echo "Test with openEuler OS ..."
95+
build_docker_images "Dockerfile.openEuler"
96+
start_service
8797
validate_microservice
8898
validate_microservice_with_openai
99+
stop_service
89100

90-
stop_docker
91-
echo y | docker system prune
101+
docker system prune -f
92102

93103
}
94104

‎tests/guardrails/test_guardrails_bias_detection_distilbias.sh‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ ip_address=$(hostname -I | awk '{print $1}')
1010
function build_docker_images() {
1111
echo "Start building docker images for microservice"
1212
cd $WORKPATH
13-
docker build --no-cache -t opea/guardrails-bias-detection:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/guardrails/src/bias_detection/Dockerfile .
13+
dockerfile_name="comps/guardrails/src/bias_detection/$1"
14+
docker build --no-cache -t opea/guardrails-bias-detection:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
1415
if [ $? -ne 0 ]; then
1516
echo "opea/guardrails-bias-detection built fail"
1617
exit 1
@@ -53,24 +54,28 @@ function validate_microservice() {
5354
echo "Validate microservice completed"
5455
}
5556

56-
function stop_docker() {
57-
cid=$(docker ps -aq --filter "name=guardrails-bias-detection-server")
58-
echo "Shutdown legacy containers "$cid
59-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
57+
function stop_service() {
58+
cd $WORKPATH/comps/guardrails/deployment/docker_compose/
59+
docker compose down || true
6060
}
6161

6262
function main() {
6363

64-
stop_docker
64+
build_docker_images "Dockerfile"
65+
trap stop_service EXIT
6566

66-
build_docker_images
67+
echo "Test normal env ..."
6768
start_service
69+
validate_microservice
70+
stop_service
6871

72+
echo "Test with openEuler OS ..."
73+
build_docker_images "Dockerfile.openEuler"
74+
start_service
6975
validate_microservice
76+
stop_service
7077

71-
stop_docker
72-
echo "cleanup container images and volumes"
73-
echo y | docker system prune > /dev/null 2>&1
78+
docker system prune -f
7479

7580
}
7681

0 commit comments

Comments
 (0)