Skip to content

Commit 2c01b8b

Browse files
zhihangdengzhihangdeng
committed
Add support for openEuler and update dataprep tests
Signed-off-by: zhihang <zhihangdeng@link.cuhk.edu.cn>
1 parent 3a9cf90 commit 2c01b8b

15 files changed

+293
-161
lines changed

‎comps/dataprep/src/Dockerfile.openEuler‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
# Copyright (C) 2024 Intel Corporation
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
FROM openeuler/python:3.11.13-oe2403lts
5+
6+
ENV LANG=C.UTF-8
7+
8+
ARG ARCH="cpu"
9+
10+
RUN yum update -y && yum install -y \
11+
gcc g++ make cmake \
12+
java-1.8.0-openjdk \
13+
cairo \
14+
mesa-libGL \
15+
jemalloc-devel \
16+
mariadb-connector-c-devel-3.3.8 \
17+
libpq-devel \
18+
poppler-utils \
19+
tesseract \
20+
ffmpeg \
21+
wget curl \
22+
shadow && \
23+
yum clean all && \
24+
rm -rf /var/cache/yum
25+
26+
RUN LIBREOFFICE_URL=https://mirrors.bfsu.edu.cn/libreoffice/libreoffice/stable/24.8.6/rpm/x86_64/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz && \
27+
wget $LIBREOFFICE_URL -O /tmp/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz && \
28+
tar -xvf /tmp/LibreOffice_24.8.6_Linux_x86-64_rpm.tar.gz -C /tmp && \
29+
yum install -y /tmp/LibreOffice_24.8.6.2_Linux_x86-64_rpm/RPMS/*.rpm && \
30+
rm -fr /tmp/LibreOffice*
31+
32+
RUN useradd -m -s /bin/bash user && \
33+
mkdir -p /home/user && \
34+
chown -R user /home/user/
35+
36+
COPY comps /home/user/comps
37+
38+
ARG uvpip='uv pip install --system --no-cache-dir'
39+
RUN pip install --no-cache-dir --upgrade pip setuptools uv && \
40+
if [ ${ARCH} = "cpu" ]; then \
41+
$uvpip torch torchvision --index-url https://download.pytorch.org/whl/cpu; \
42+
$uvpip -r /home/user/comps/dataprep/src/requirements-cpu.txt; \
43+
else \
44+
$uvpip -r /home/user/comps/dataprep/src/requirements-gpu.txt; \
45+
fi
46+
47+
ENV PYTHONPATH=$PYTHONPATH:/home/user
48+
49+
RUN mkdir -p /home/user/comps/dataprep/src/uploaded_files && chown -R user /home/user/comps/dataprep/src/uploaded_files
50+
RUN mkdir -p /data && chown -R user /data
51+
52+
USER user
53+
ENV NLTK_DATA=/home/user/nltk_data
54+
# air gapped support: predownload all needed nltk data
55+
RUN mkdir -p /home/user/nltk_data && python -m nltk.downloader -d /home/user/nltk_data punkt_tab averaged_perceptron_tagger_eng stopwords
56+
# air gapped support: set model cache dir
57+
ENV HF_HUB_CACHE=/data
58+
59+
WORKDIR /home/user/comps/dataprep/src
60+
61+
ENTRYPOINT ["sh", "-c", "python $( [ \"$MULTIMODAL_DATAPREP\" = \"true\" ] && echo 'opea_dataprep_multimodal_microservice.py' || echo 'opea_dataprep_microservice.py')"]

‎tests/dataprep/test_dataprep_arango.sh‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright (C) 2024 Intel Corporation
33
# SPDX-License-Identifier: Apache-2.0
44

5-
set -x
5+
set -xe
66

77
export WORKPATH=$(dirname "$PWD")
88
export LOG_PATH="$WORKPATH/tests"
@@ -17,7 +17,8 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
1717
function build_docker_images() {
1818
cd $WORKPATH
1919
echo $(pwd)
20-
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
20+
dockerfile_name="comps/dataprep/src/$1"
21+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
2122
if [ $? -ne 0 ]; then
2223
echo "opea/dataprep built fail"
2324
exit 1
@@ -113,25 +114,29 @@ function validate_microservice() {
113114
check_result "dataprep - del" '{"status":true}' dataprep-arangodb ${LOG_PATH}/dataprep_arangodb.log
114115
}
115116

116-
function stop_docker() {
117-
cd $WORKPATH/comps/third_parties/arangodb/deployment/docker_compose/
118-
docker compose -f compose.yaml down --remove-orphans
119-
120-
cd $WORKPATH/comps/dataprep/deployment/docker_compose
121-
docker compose -f compose.yaml down --remove-orphans
122-
117+
function stop_service() {
118+
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
119+
docker compose down || true
123120
}
124121

125122
function main() {
126-
stop_docker
127123

128-
build_docker_images
124+
build_docker_images "Dockerfile"
125+
trap stop_service EXIT
126+
127+
echo "Test normal env ..."
129128
start_service
129+
validate_microservice
130+
stop_service
130131

132+
echo "Test with openEuler OS ..."
133+
build_docker_images "Dockerfile.openEuler"
134+
start_service
131135
validate_microservice
136+
stop_service
137+
138+
docker system prune -f
132139

133-
stop_docker
134-
echo y | docker system prune
135140

136141
}
137142

‎tests/dataprep/test_dataprep_elasticsearch.sh‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright (C) 2024 Intel Corporation
33
# SPDX-License-Identifier: Apache-2.0
44

5-
set -x
5+
set -xe
66

77
WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
@@ -18,9 +18,9 @@ function build_docker_images() {
1818
echo $WORKPATH
1919
# piull elasticsearch image
2020
docker pull docker.elastic.co/elasticsearch/elasticsearch:8.16.0
21-
21+
dockerfile_name="comps/dataprep/src/$1"
2222
# build dataprep image for elasticsearch
23-
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
23+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
2424
if [ $? -ne 0 ]; then
2525
echo "opea/dataprep built fail"
2626
exit 1
@@ -102,22 +102,29 @@ function validate_microservice() {
102102
fi
103103
}
104104

105-
function stop_docker() {
106-
cid=$(docker ps -aq --filter "name=elasticsearch-vector-db" --filter "name=dataprep-elasticsearch")
107-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
105+
function stop_service() {
106+
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
107+
docker compose down || true
108108
}
109109

110110
function main() {
111111

112-
stop_docker
112+
build_docker_images "Dockerfile"
113+
trap stop_service EXIT
114+
start_service
113115

114-
build_docker_images
116+
echo "Test normal env ..."
115117
start_service
118+
validate_microservice
119+
stop_service
116120

121+
echo "Test with openEuler OS ..."
122+
build_docker_images "Dockerfile.openEuler"
123+
start_service
117124
validate_microservice
125+
stop_service
118126

119-
stop_docker
120-
echo y | docker system prune
127+
docker system prune -f
121128

122129
}
123130

‎tests/dataprep/test_dataprep_mariadb.sh‎

Lines changed: 17 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright (C) 2024 Intel Corporation
33
# SPDX-License-Identifier: Apache-2.0
44

5-
set -x
5+
set -xe
66

77
WORKPATH=$(dirname "$PWD")
88
LOG_PATH="$WORKPATH/tests"
@@ -16,8 +16,9 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
1616
function build_docker_images() {
1717
cd $WORKPATH
1818

19+
dockerfile_name="comps/dataprep/src/$1"
1920
# build dataprep image for mariadb
20-
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f $WORKPATH/comps/dataprep/src/Dockerfile .
21+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
2122
if [ $? -ne 0 ]; then
2223
echo "opea/dataprep built fail"
2324
exit 1
@@ -81,25 +82,29 @@ function validate_microservice() {
8182
check_result "dataprep - del" '{"status":true}' dataprep-mariadb-vector ${LOG_PATH}/dataprep_mariadb.log
8283
}
8384

84-
function stop_docker() {
85-
cid=$(docker ps -aq --filter "name=dataprep-mariadb-vector")
86-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
87-
88-
cid=$(docker ps -aq --filter "name=mariadb-server")
89-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
85+
function stop_service() {
86+
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
87+
docker compose down || true
9088
}
9189

9290
function main() {
9391

94-
stop_docker
92+
build_docker_images "Dockerfile"
93+
trap stop_service EXIT
94+
start_service
9595

96-
build_docker_images
96+
echo "Test normal env ..."
9797
start_service
98+
validate_microservice
99+
stop_service
98100

101+
echo "Test with openEuler OS ..."
102+
build_docker_images "Dockerfile.openEuler"
103+
start_service
99104
validate_microservice
105+
stop_service
100106

101-
stop_docker
102-
echo y | docker system prune
107+
docker system prune -f
103108

104109
}
105110

‎tests/dataprep/test_dataprep_milvus.sh‎

Lines changed: 14 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
1717
function build_docker_images() {
1818
cd $WORKPATH
1919
echo $(pwd)
20+
dockerfile_name="comps/dataprep/src/$1"
2021
# dataprep milvus image
21-
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
22+
docker build --no-cache -t opea/dataprep:${TAG} --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
2223
if [ $? -ne 0 ]; then
2324
echo "opea/dataprep built fail"
2425
exit 1
@@ -91,37 +92,32 @@ function validate_microservice() {
9192
check_result "dataprep - del" '{"status":true}' dataprep-milvus-server ${LOG_PATH}/dataprep_milvus.log
9293
}
9394

94-
function stop_docker() {
95+
function stop_service() {
9596
cd $WORKPATH/comps/third_parties/milvus/deployment/docker_compose/
96-
docker compose -f compose.yaml down --remove-orphans
97+
docker compose down || true
9798

9899
cd $WORKPATH/comps/dataprep/deployment/docker_compose
99-
docker compose -f compose.yaml down --remove-orphans
100+
docker compose down || true
100101

101102
}
102103

103104
function main() {
104105

105-
stop_docker
106-
107-
build_docker_images
108-
trap stop_docker EXIT
109106

107+
build_docker_images "Dockerfile"
108+
trap stop_service EXIT
110109
echo "Test normal env ..."
111110
start_service
112111
validate_microservice
113-
stop_docker
114-
115-
if [[ -n "${DATA_PATH}" ]]; then
116-
echo "Test air gapped env ..."
117-
prepare_dataprep_models ${DATA_PATH}
118-
start_service true
119-
validate_microservice true
120-
stop_docker
121-
fi
112+
stop_service
122113

123-
echo y | docker system prune
114+
echo "Test with openEuler OS ..."
115+
build_docker_images "Dockerfile.openEuler"
116+
start_service
117+
validate_microservice
118+
stop_service
124119

120+
docker system prune -f
125121
}
126122

127123
main

‎tests/dataprep/test_dataprep_milvus_multimodal.sh‎

Lines changed: 25 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,8 @@ source ${SCRIPT_DIR}/dataprep_utils.sh
3030
function build_docker_images() {
3131
cd $WORKPATH
3232
echo $(pwd)
33-
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/dataprep/src/Dockerfile .
33+
dockerfile_name="comps/dataprep/src/$1"
34+
docker build --no-cache -t opea/dataprep:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
3435

3536
if [ $? -ne 0 ]; then
3637
echo "opea/dataprep built fail"
@@ -43,14 +44,16 @@ function build_docker_images() {
4344
function build_lvm_docker_images() {
4445
cd $WORKPATH
4546
echo $(pwd)
46-
docker build --no-cache -t opea/lvm-llava:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/third_parties/llava/src/Dockerfile .
47+
dockerfile_name="comps/third_parties/llava/src/$1"
48+
docker build --no-cache -t opea/lvm-llava:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
4749
if [ $? -ne 0 ]; then
4850
echo "opea/lvm-llava built fail"
4951
exit 1
5052
else
5153
echo "opea/lvm-llava built successful"
5254
fi
53-
docker build --no-cache -t opea/lvm:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f comps/lvms/src/Dockerfile .
55+
dockerfile_name="comps/lvms/src/$1"
56+
docker build --no-cache -t opea/lvm:comps --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy -f "${dockerfile_name}" .
5457
if [ $? -ne 0 ]; then
5558
echo "opea/lvm built fail"
5659
exit 1
@@ -71,7 +74,7 @@ function start_lvm() {
7174
cd $WORKPATH
7275
echo $(pwd)
7376
echo "Building LVM Docker Images"
74-
build_lvm_docker_images
77+
build_lvm_docker_images $1
7578
echo "Starting LVM Services"
7679
start_lvm_service
7780

@@ -387,13 +390,16 @@ function validate_microservice() {
387390
}
388391

389392
function stop_docker() {
390-
cid=$(docker ps -aq --filter "name=dataprep-multimodal-milvus-server*" --filter "name=milvus-*")
391-
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
393+
cd $WORKPATH/comps/dataprep/deployment/docker_compose/
394+
docker compose down || true
395+
392396
cid=$(docker ps -aq --filter "name=test-comps-lvm*")
393397
if [[ ! -z "$cid" ]]; then docker stop $cid && docker rm $cid && sleep 1s; fi
394398

395399
}
396400

401+
402+
397403
function delete_data() {
398404
cd ${LOG_PATH}
399405
rm -rf ${tmp_dir}
@@ -402,17 +408,24 @@ function delete_data() {
402408

403409
function main() {
404410

405-
stop_docker
406-
build_docker_images
407411
prepare_data
412+
build_docker_images "Dockerfile"
413+
trap stop_service EXIT
408414

409-
start_lvm
415+
echo "Test normal env ..."
416+
start_lvm "Dockerfile"
417+
start_service
418+
validate_microservice
419+
stop_service
420+
421+
echo "Test with openEuler OS ..."
422+
build_docker_images "Dockerfile.openEuler"
423+
start_lvm "Dockerfile.openEuler"
410424
start_service
411-
412425
validate_microservice
426+
stop_service
413427
delete_data
414-
stop_docker
415-
echo y | docker system prune
428+
docker system prune -f
416429

417430
}
418431

0 commit comments

Comments
 (0)