Skip to content

Commit f627bcb

Browse files
qili93qili93
authored
update npu dockerfile and codestyle, test=develop (PaddlePaddle#241)
* update npu dockerfile and codestyle, test=develop * update readme
1 parent b28f432 commit f627bcb

14 files changed

Lines changed: 222 additions & 95 deletions

‎.flake8‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[flake8]
2+
select = C,E,F,W
3+
exclude =
4+
./build,
5+
./backends/npu/build,
6+
./backends/mlu/build,
7+
./backends/custom_cpu/build
8+
ignore =
9+
# E, see https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
10+
E203,
11+
E401,E402,
12+
E501,
13+
E721,E722,E731,E741,
14+
15+
# F, see https://flake8.pycqa.org/en/latest/user/error-codes.html
16+
F405,
17+
F811,F841,
18+
19+
# W, see https://pycodestyle.pycqa.org/en/latest/intro.html#error-codes
20+
W503
21+
per-file-ignores =
22+
# Ignore unused imports in __init__.py
23+
__init__.py: F401
24+
# Ignore undefined variables in CMake config and some dygraph_to_static tests
25+
.cmake-format.py: F821

‎.pre-commit-config.yaml‎

Lines changed: 75 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,31 +1,75 @@
11
repos:
2-
- repo: https://github.com/Lucas-C/pre-commit-hooks.git
3-
sha: v1.0.1
4-
hooks:
5-
- id: remove-crlf
6-
files: (?!.*third_party)^.*$ | (?!.*book)^.*$
7-
- repo: https://github.com/PaddlePaddle/mirrors-yapf.git
8-
sha: 0d79c0c469bab64f7229c9aca2b1186ef47f0e37
9-
hooks:
10-
- id: yapf
11-
files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
2+
# Common hooks
123
- repo: https://github.com/pre-commit/pre-commit-hooks
13-
sha: 5bf6c09bfa1297d3692cadd621ef95f1284e33c0
4+
rev: v4.1.0
145
hooks:
156
- id: check-added-large-files
167
- id: check-merge-conflict
178
- id: check-symlinks
189
- id: detect-private-key
19-
files: (?!.*third_party)^.*$ | (?!.*book)^.*$
2010
- id: end-of-file-fixer
11+
- id: trailing-whitespace
12+
files: (.*\.(py|bzl|md|rst|c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps|cmake|yaml|yml|hook)|BUILD|.*\.BUILD|WORKSPACE|CMakeLists\.txt)$
13+
- repo: https://github.com/Lucas-C/pre-commit-hooks.git
14+
rev: v1.1.14
15+
hooks:
16+
- id: remove-crlf
17+
- id: remove-tabs
18+
name: Tabs remover (C++)
19+
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$
20+
args: [--whitespaces-count, '2']
21+
- id: remove-tabs
22+
name: Tabs remover (Python)
23+
files: (.*\.(py|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
24+
args: [--whitespaces-count, '4']
25+
- repo: local
26+
hooks:
27+
- id: copyright_checker
28+
name: copyright_checker
29+
entry: python ./tools/codestyle/copyright.hook
30+
language: system
31+
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$
32+
# For Python files
33+
- repo: https://github.com/psf/black.git
34+
rev: 22.8.0
35+
hooks:
36+
- id: black
37+
files: (.*\.(py|pyi|bzl)|BUILD|.*\.BUILD|WORKSPACE)$
38+
- repo: https://github.com/pycqa/isort
39+
rev: 5.10.1
40+
hooks:
41+
- id: isort
42+
- repo: https://github.com/PyCQA/flake8
43+
rev: 4.0.1
44+
hooks:
45+
- id: flake8
46+
- repo: https://github.com/PyCQA/autoflake
47+
rev: v1.7.7
48+
hooks:
49+
- id: autoflake
50+
args:
51+
- --in-place
52+
- --remove-all-unused-imports
53+
- --ignore-pass-after-docstring
54+
- --ignore-init-module-imports
55+
- --exclude=python/paddle/fluid/[!t]**,python/paddle/fluid/tra**
2156
- repo: local
2257
hooks:
23-
- id: clang-format-with-version-check
58+
- id: pylint-doc-string
59+
name: pylint
60+
description: Check python docstring style using docstring_checker.
61+
entry: bash ./tools/codestyle/pylint_pre_commit.hook
62+
language: system
63+
files: \.(py)$
64+
# For C++ files
65+
- repo: local
66+
hooks:
67+
- id: clang-format
2468
name: clang-format
2569
description: Format files with ClangFormat.
2670
entry: bash ./tools/codestyle/clang_format.hook -i
2771
language: system
28-
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps)$
72+
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|xpu|kps)$
2973
- repo: local
3074
hooks:
3175
- id: cpplint-cpp-source
@@ -34,18 +78,24 @@ repos:
3478
entry: bash ./tools/codestyle/cpplint_pre_commit.hook
3579
language: system
3680
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx)$
81+
args:
82+
- --extensions=c,cc,cxx,cpp,cu,cuh,h,hpp,hxx,kps
83+
- --filter=-readability/fn_size,-build/include_what_you_use,-build/c++11,-whitespace/parens
84+
- --quiet
85+
# For CMake files
3786
- repo: local
3887
hooks:
39-
- id: pylint-doc-string
40-
name: pylint
41-
description: Check python docstring style using docstring_checker.
42-
entry: bash ./tools/codestyle/pylint_pre_commit.hook
88+
- id: auto-generate-cmakelists
89+
name: auto-generate-cmakelists
90+
entry: bash ./tools/gen_ut_cmakelists.hook
4391
language: system
44-
files: \.(py)$
45-
- repo: local
92+
files: testslist.csv$
93+
- repo: https://github.com/cheshirekow/cmake-format-precommit
94+
rev: v0.6.13
4695
hooks:
47-
- id: copyright_checker
48-
name: copyright_checker
49-
entry: python ./tools/codestyle/copyright.hook
50-
language: system
51-
files: \.(c|cc|cxx|cpp|cu|h|hpp|hxx|proto|xpu|kps|py|sh)$
96+
- id: cmake-format
97+
- repo: https://github.com/cmake-lint/cmake-lint
98+
rev: 1.4.2
99+
hooks:
100+
- id: cmakelint
101+
args: [--config=./tools/codestyle/.cmakelintrc]

‎Paddle‎

Submodule Paddle updated 2408 files

‎backends/npu/README_cn.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,9 @@ git submodule update --remote --init --recursive
3838
# 1) 进入硬件后端(昇腾NPU)目录
3939
cd backends/npu
4040

41-
# 2) 编译之前需要先保证环境下装有飞桨安装包，可以直接安装 CPU 版本
42-
# 注意：如果是 aarch64 环境，需要源码编译得到 Paddle CPU 的 WHL 安装包
41+
# 2) 编译之前需要先保证环境下装有飞桨安装包，直接安装飞桨 CPU 版本即可
42+
# 默认 NPU 开发镜像中已经装有飞桨 CPU 安装包 (飞桨 develop 分支的 nightly build 版本)
43+
# 也可以通过如下命令安装，如果是 aarch64 环境，则需要源码编译得到 Paddle CPU 的 WHL 安装包
4344
pip install paddlepaddle==0.0.0 -f https://www.paddlepaddle.org.cn/whl/linux/cpu-mkl/develop.html
4445

4546
# 3) 编译选项，是否打开单元测试编译，默认值为 ON

‎backends/npu/tools/dockerfile/Dockerfile.npu.ubuntu18-aarch64-gcc82‎

Lines changed: 12 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,13 +18,14 @@ COPY version.info /usr/local/Ascend/driver/version.info
1818
WORKDIR /usr/local/Ascend
1919
ARG CANN_VERSION
2020

21-
# install CANN requirment
22-
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/60RC1alpha003/softwareinstall/instg/atlasdeploy_03_0022.html
21+
# install CANN requirement
22+
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/600alpha002/softwareinstall/instg/atlasdeploy_03_0022.html
2323
RUN apt-get install -y zlib1g zlib1g-dev libsqlite3-dev openssl libssl-dev libffi-dev unzip pciutils \
2424
net-tools libblas-dev gfortran libblas3 liblapack-dev liblapack3 libopenblas-dev
2525

2626
# install CANN requirement
27-
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/60RC1alpha003/softwareinstall/instg/atlasdeploy_03_0022.html
27+
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/600alpha002/softwareinstall/instg/atlasdeploy_03_0022.html
28+
RUN pip install --upgrade pip setuptools wheel
2829
RUN pip install 'numpy>=1.14.3' 'decorator>=4.4.0' 'sympy>=1.4' 'cffi>=1.12.3' 'protobuf>=3.11.3'
2930
RUN pip install attrs pyyaml pathlib2 scipy requests psutil absl-py
3031

@@ -63,12 +64,16 @@ ENV HCCL_SECURITY_MODE=1
6364
# map this foler in docker run
6465
RUN rm -rf /usr/local/Ascend/driver
6566

67+
# install paddlepaddle-cpu whl package
68+
RUN wget -q https://paddle-device.bj.bcebos.com/develop/cpu/paddlepaddle-0.0.0-cp37-cp37m-linux_aarch64.whl && \
69+
pip install paddlepaddle-0.0.0-cp37-cp37m-linux_aarch64.whl && rm -rf paddlepaddle-0.0.0-cp37-cp37m-linux_aarch64.whl
70+
6671
# Clean
6772
RUN apt-get clean -y
73+
RUN pip cache purge
6874

69-
# install gcc-10
70-
RUN wget -O gcc-10-with-glibc233.tar.gz https://bcloud-buildkit.bj.bcebos.com/build_env/gcc/ampere/gcc-10-with-glibc2.33.tar.gz
71-
RUN tar xf gcc-10-with-glibc233.tar.gz -C /opt/compiler/
72-
RUN rm -rf gcc-10-with-glibc233.tar.gz
75+
# install gcc-10 for baidu icoding on aarch64
76+
RUN wget -O gcc-10-with-glibc233.tar.gz https://bcloud-buildkit.bj.bcebos.com/build_env/gcc/ampere/gcc-10-with-glibc2.33.tar.gz && \
77+
tar xf gcc-10-with-glibc233.tar.gz -C /opt/compiler/ && rm -rf gcc-10-with-glibc233.tar.gz
7378

7479
EXPOSE 22

‎backends/npu/tools/dockerfile/Dockerfile.npu.ubuntu18-x86_64-gcc82‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,12 +19,13 @@ WORKDIR /usr/local/Ascend
1919
ARG CANN_VERSION
2020

2121
# install CANN requirment
22-
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/60RC1alpha003/softwareinstall/instg/atlasdeploy_03_0022.html
22+
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/600alpha002/softwareinstall/instg/atlasdeploy_03_0022.html
2323
RUN apt-get install -y zlib1g zlib1g-dev libsqlite3-dev openssl libssl-dev libffi-dev unzip pciutils \
2424
net-tools libblas-dev gfortran libblas3 liblapack-dev liblapack3 libopenblas-dev
2525

2626
# install CANN requirement
27-
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/60RC1alpha003/softwareinstall/instg/atlasdeploy_03_0022.html
27+
# https://www.hiascend.com/document/detail/zh/CANNCommunityEdition/600alpha002/softwareinstall/instg/atlasdeploy_03_0022.html
28+
RUN pip install --upgrade pip setuptools wheel
2829
RUN pip install 'numpy>=1.14.3' 'decorator>=4.4.0' 'sympy>=1.4' 'cffi>=1.12.3' 'protobuf>=3.11.3'
2930
RUN pip install attrs pyyaml pathlib2 scipy requests psutil absl-py
3031

@@ -63,7 +64,12 @@ ENV HCCL_SECURITY_MODE=1
6364
# map this foler in docker run
6465
RUN rm -rf /usr/local/Ascend/driver
6566

67+
# install paddlepaddle-cpu whl package
68+
RUN wget -q https://paddle-wheel.bj.bcebos.com/develop/linux/linux-cpu-mkl-avx/paddlepaddle-0.0.0-cp37-cp37m-linux_x86_64.whl && \
69+
pip install paddlepaddle-0.0.0-cp37-cp37m-linux_x86_64.whl && rm -rf paddlepaddle-0.0.0-cp37-cp37m-linux_x86_64.whl
70+
6671
# Clean
6772
RUN apt-get clean -y
73+
RUN pip cache purge
6874

6975
EXPOSE 22

‎backends/npu/tools/dockerfile/build-aarch64.sh‎

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,18 @@
1717
set -ex
1818

1919
# Usage:
20-
# export CANN_VERSION=5.1.RC2
21-
# export CANN_VERSION=6.0.0.alpha001
20+
# export CANN_VERSION=6.0.0.alpha002
2221
# bash build-aarch64.sh ${CANN_VERSION}
2322

24-
CANN_VERSION=${1:-6.0.0.alpha001} # default 6.0.0.alpha001
23+
CANN_VERSION=${1:-6.0.0.alpha002} # default 6.0.0.alpha002
2524
CANN_TOOLKIT=Ascend-cann-toolkit_${CANN_VERSION}_linux-aarch64.run
2625

2726
DOCKER_VERSION=${CANN_VERSION//[^0-9]/}
2827
DOCKER_VERSION=${DOCKER_VERSION:0:3}
2928

3029
# download aarch64 pkgs
31-
if [ ${DOCKER_VERSION} -eq 512 ]; then
32-
CANN_URL=https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/CANN%20${CANN_VERSION}/${CANN_TOOLKIT}
33-
else
34-
CANN_URL=https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/${CANN_VERSION}/${CANN_TOOLKIT}
35-
fi
36-
3730
if [ ! -f ${CANN_TOOLKIT} ]; then
38-
wget ${CANN_URL}
31+
wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/${CANN_VERSION}/${CANN_TOOLKIT}
3932
fi
4033

4134
# copy file to current directory

‎backends/npu/tools/dockerfile/build-x86_64.sh‎

Lines changed: 4 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,18 @@
1717
set -x
1818

1919
# Usage:
20-
# export CANN_VERSION=5.1.RC2
21-
# export CANN_VERSION=6.0.0.alpha001
20+
# export CANN_VERSION=6.0.0.alpha002
2221
# bash build-x86_64.sh ${CANN_VERSION}
2322

24-
CANN_VERSION=${1:-6.0.0.alpha001} # default 6.0.0.alpha001
23+
CANN_VERSION=${1:-6.0.0.alpha002} # default 6.0.0.alpha002
2524
CANN_TOOLKIT=Ascend-cann-toolkit_${CANN_VERSION}_linux-x86_64.run
2625

2726
DOCKER_VERSION=${CANN_VERSION//[^0-9]/}
2827
DOCKER_VERSION=${DOCKER_VERSION:0:3}
2928

30-
# download aarch64 pkgs
31-
if [ ${DOCKER_VERSION} -eq 512 ]; then
32-
CANN_URL=https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/CANN%20${CANN_VERSION}/${CANN_TOOLKIT}
33-
else
34-
CANN_URL=https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/${CANN_VERSION}/${CANN_TOOLKIT}
35-
fi
36-
29+
# download x86_64 pkgs
3730
if [ ! -f ${CANN_TOOLKIT} ]; then
38-
wget ${CANN_URL}
31+
wget https://ascend-repo.obs.cn-east-2.myhuaweicloud.com/CANN/${CANN_VERSION}/${CANN_TOOLKIT}
3932
fi
4033

4134
# copy file to current directory

‎tools/codestyle/.cmakelintrc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
filter=-readability/wonkycase,-syntax,-convention/filename,-package/stdargs,-whitespace/indent,-whitespace/extra,-linelength,-readability/mixedcase

‎tools/codestyle/clang_format.hook‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,14 @@ readonly VERSION="13.0.0"
66
version=$(clang-format -version)
77

88
if ! [[ $(python -V 2>&1 | awk '{print $2}' | awk -F '.' '{print $1$2}') -ge 36 ]]; then
9-
echo "clang-format installation by pip need python version great equal 3.6,
9+
echo "clang-format installation by pip need python version great equal 3.6,
1010
please change the default python to higher version."
1111
exit 1
1212
fi
1313

1414
if ! [[ $version == *"$VERSION"* ]]; then
1515
# low version of pip may not have the source of clang-format whl
16-
pip install --upgrade pip
16+
pip install --upgrade pip
1717
pip install clang-format==13.0.0
1818
fi
1919

0 commit comments

Comments
 (0)