diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..18019a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM ubuntu:18.04 +LABEL maintainer=megengine@megvii.com + +RUN apt update \ + && apt install -y curl \ + && apt install -y ffmpeg \ + && apt install -y yasm \ + && apt install -y clang \ + && apt install -y redis-server \ + && apt install -y python3 \ + && apt install -y python3-pip \ + && apt install -y git \ + && apt install -y vim \ + && apt install -y build-essential + +RUN curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf -o run.sh \ + && chmod a+x run.sh \ + && ./run.sh -y \ + && export PATH=$HOME/.cargo/bin:${PATH} \ + && cargo --version + +RUN mkdir -p $HOME/megflow-runspace +WORKDIR $HOME/megflow-runspace +COPY . $HOME/megflow-runspace/ + +RUN PATH=$HOME/.cargo/bin:${PATH} \ + && cargo build \ + && cd flow-python \ + && python3 setup.py install --user \ + && cd examples \ + && cargo run --example run_with_plugins -- -p logical_test diff --git a/README.md b/README.md index bfbc52f..3c02848 100644 --- a/README.md +++ b/README.md @@ -11,20 +11,38 @@ MegFlow 是一个面向视觉应用的流式计算框架, 目标是简单、高 - 支持 demux/reorder/transform 等通用函数式组件 - Python 插件内置有栈协程，不依赖 asyncio - 基础测试工具，支持插件沙盒，用于单测插件 - + ## HowTo -* [how to build and run in 15 minutes](docs/how-to-build.zh.md) +* [how to run in 15 minutes](docs/how-to-run-in-15-minutes.zh.md) +* [how to build with docker](docs/how-to-build-with-docker.zh.md) +* [how to build from source](docs/how-to-build-from-source.zh.md) * [how to add my service](docs/how-to-add-graph.zh.md) * [how to add plugins](docs/how-to-add-plugins.zh.md) * [how to optimize and debug](docs/how-to-debug.zh.md) * [how to contribute](docs/how-to-contribute.zh.md) * [FAQ](docs/FAQ.zh.md) +## Current Support List + +| 系统环境 | 支持情况 | +| - | - | +| Windows 10 WSL ubuntu18.04 | ✔️ | +| x86 Ubuntu16.04 有 GPU | ✔️ | +| x86 Ubuntu18.04 无 GPU | ✔️ | +| ARM | / | + +| Python 版本 | 支持情况 | +| - | - | +| 3.6 | ✔️ | +| 3.7 | ✔️ | +| 3.8 | ✔️ | +| 3.9 | / | + ## Built-in Applications * 猫猫围栏 * 电梯电动车报警 -## Coming soon +## Coming Soon - 进程级别的节点、子图支持 - 插件自动化测试部署 - 调试工具，建图实时预览 profile 工具 diff --git a/docs/how-to-build.zh.md b/docs/how-to-build-from-source.zh.md similarity index 87% rename from docs/how-to-build.zh.md rename to docs/how-to-build-from-source.zh.md index 3837ded..d3b19dc 100644 --- a/docs/how-to-build.zh.md +++ b/docs/how-to-build-from-source.zh.md @@ -1,19 +1,12 @@ # Building from Source -## Prerequisites - -### 软硬件环境 - -| 测试通过的环境 | 备注 | -| - | - | -| win10 WSL ubuntu18.04 | - | -| x86 Ubuntu16.04 服务器有 GPU | - | -| x86 Ubuntu18.04 无 GPU | 运行时选 CPU config | +如果仅仅是想运行应用，推荐使用 [docker](how-to-build-with-docker.zh.md) 或 [prebuilt .whl 包](how-to-run-in-15-minutes.zh.md) ，源码安装需要 Rust/ffmpeg 环境。 -支持主流 x86 Linux 版本，ARM 还在开发。 +## Prerequisites ### 安装 Rust ```bash +$ sudo apt install yasm git build-essential ffmpeg curl $ curl --proto '=https' --tlsv1.2 https://sh.rustup.rs -sSf | sh ``` @@ -41,18 +34,18 @@ $ conda --version conda 4.10.3 ``` -创建一个 Python3.8（已测试 3.6.13/3.7.11/3.8.11 可用。**3.9 暂不可用**，这里以 3.8 为例）的环境，激活。 +创建一个 Python3.x（这里以 3.8 为例） 的环境，激活。 ```bash $ conda create --name py38 python=3.8 $ conda activate py38 ``` + ## Build MegFlow 需要编译 ffmpeg。考虑到 ffmpeg 依赖较多、本身又是常用工具，最简单的办法就是直接装 ffmpeg 把编译依赖装上 ```bash -$ sudo apt install yasm # ffmpeg 编译依赖 $ sudo apt install ffmpeg $ ffmpeg ffmpeg version 3.4.8... @@ -74,7 +67,7 @@ $ python3 setup.py install --user P.S. 默认 ffmpeg 依赖自动从 github 上拉取源码构建，这会使得首次构建的时间较长。若希望缩短首次构建时间，或者希望依赖一个指定版本的 ffmpeg，可以启用环境变量`CARGO_FEATURE_PREBUILD`并参考[rust-ffmpeg](https://github.com/zmwangx/rust-ffmpeg/wiki/Notes-on-building)自行构建 -## Python “开机自检”用例 +## Python“开机自检” ```bash $ cd examples $ cargo run --example run_with_plugins -- -p logical_test diff --git a/docs/how-to-build-with-docker.zh.md b/docs/how-to-build-with-docker.zh.md new file mode 100644 index 0000000..f1b6049 --- /dev/null +++ b/docs/how-to-build-with-docker.zh.md @@ -0,0 +1,33 @@ +# Building with docker + +## Build Docker Image + +MegFlow 提供了 [Dockerfile](../Dockerfile)，能够“可复现地”生成运行环境、减少依赖缺失的痛苦 + +```bash +$ cd MegFlow +$ docker build -t megflow . +``` +稍等一段时间（取决于网络和 CPU）镜像构建完成并做了基础自测 +```bash +$ docker images +REPOSITORY TAG IMAGE ID CREATED SIZE +megflow latest c65e37e1df6c 18 hours ago 5.05GB +``` +直接用 ${IMAGE ID} 进入开始跑应用 +```bash +$ docker run -p 18081:8081 -p 18082:8082 -i -t c65e37e1df6c /bin/bash +``` + +## Python Built-in Applications + +接下来开始运行好玩的 Python 应用 + +* [猫猫围栏运行手册](../flow-python/examples/cat_finder/README.md) + * 图片注册猫猫 + * 部署视频围栏，注册的猫离开围栏时会发通知 + * 未注册的不会提示 +* [电梯电瓶车告警](../flow-python/examples/electric_bicycle/README.md) + * 电梯里看到电瓶车立即报警 +* Comming Soon + * OCR： 通用字符识别 diff --git a/docs/how-to-run-in-15-minutes.zh.md b/docs/how-to-run-in-15-minutes.zh.md new file mode 100644 index 0000000..8e5ec21 --- /dev/null +++ b/docs/how-to-run-in-15-minutes.zh.md @@ -0,0 +1,79 @@ +# Run in 15 minutes + + + +## 安装 python3.x （推荐 conda） + +打开 [miniconda 官网](https://docs.conda.io/en/latest/miniconda.html) 下载 miniconda 安装包，修改权限并安装。 + +```bash +$ wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +$ chmod a+x Miniconda3-latest-Linux-x86_64.sh +$ ./Miniconda3-latest-Linux-x86_64.sh +``` + +安装时接受 conda 修改默认 .bashrc 环境变量（zsh 用户还需自行修改 .zshrc 中的 conda initialize 配置）。成功后 `conda` 可正常运行 +```bash +$ conda --version +conda 4.10.3 +``` + +创建一个 Python3.x（这里以 3.8 为例） 的环境，激活。 +```bash +$ conda create --name py38 python=3.8 +$ conda activate py38 +``` + +## 安装 Prebuilt 包 + +从 [MegFlow release](https://github.com/MegEngine/MegFlow/releases) 下载对应 python 版本的 .whl 包，安装 +```bash +$ python3 -m pip install pyflow-0.1.0-py38-none-linux_x86_64.whl --force-reinstall +``` +完成后应该可以 `import pyflow` +```bash +$ python3 +Python 3.8.3 (default, May 19 2020, 18:47:26) +[GCC 7.3.0] :: Anaconda, Inc. on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> import pyflow +``` +.whl 打包了可执行文件 `run_with_plugins`，如果使用 conda 位置应该在 +```bash +$ cd ${HOME}/miniconda3/envs/py38/lib/python3.8/site-packages/pyflow/ +$ sudo apt install build-essential -y +$ ldd run_with_plugins # 可看到仅依赖常见库 +$ ./run_with_plugins --help +run_with_plugins 1.0 +megvii +... +``` + +## Python“开机自检” + +```bash +$ cd ${MegFlow_PATH}/flow-python/examples # 这行必须 +$ run_with_plugins -p logical_test +``` + +`logical_test` 是 examples 下最基础的计算图测试用例，运行能正常结束表示 MegFlow 编译成功、基本语义无问题。 + +此处常见问题：`error while loading shared libraries: libpython3.8.xxx`。如果使用 conda 只需要 +```bash +$ export LD_LIBRARY_PATH=/home/`whoami`/miniconda3/pkgs/python-3.8.11-h12debd9_0_cpython/lib:${LD_LIBRARY_PATH} +``` + +> 工作原理：[pyflow](../flow-python/pyflow/__init__.py) 仅是一层接口，由 run_with_plugins “注入”建图/调度/优化等实现。 + +## Python Built-in Applications + +接下来开始运行好玩的 Python 应用 + +* [猫猫围栏运行手册](../flow-python/examples/cat_finder/README.md) + * 图片注册猫猫 + * 部署视频围栏，注册的猫离开围栏时会发通知 + * 未注册的不会提示 +* [电梯电瓶车告警](../flow-python/examples/electric_bicycle/README.md) + * 电梯里看到电瓶车立即报警 +* Comming Soon + * OCR： 通用字符识别 diff --git a/flow-python/gen_whl.sh b/flow-python/gen_whl.sh new file mode 100755 index 0000000..9f55e8d --- /dev/null +++ b/flow-python/gen_whl.sh @@ -0,0 +1,28 @@ +# build py36~39 version +conda activate py36 +cargo build --example run_with_plugins --release +cp ../target/release/examples/run_with_plugins ./pyflow/ +ldd pyflow/run_with_plugins +rm -rf ./build +python3 whl-py36-setup.py bdist_wheel -p linux-x86_64 -d py36_dist --python-tag py36 + +conda activate py37 +cargo build --example run_with_plugins --release +ldd pyflow/run_with_plugins +cp ../target/release/examples/run_with_plugins ./pyflow/ +rm -rf ./build +python3 whl-py37-setup.py bdist_wheel -p linux-x86_64 -d py37_dist --python-tag py37 + +conda activate py38 +cargo build --example run_with_plugins --release +ldd pyflow/run_with_plugins +cp ../target/release/examples/run_with_plugins ./pyflow/ +rm -rf ./build +python3 whl-py38-setup.py bdist_wheel -p linux-x86_64 -d py38_dist --python-tag py38 + + +rm -rf dist +mkdir dist +cp py36_dist/pyflow-0.1.0-py36-none-linux_x86_64.whl dist/ +cp py37_dist/pyflow-0.1.0-py37-none-linux_x86_64.whl dist/ +cp py38_dist/pyflow-0.1.0-py38-none-linux_x86_64.whl dist/ diff --git a/flow-python/setup.py b/flow-python/setup.py index 52a9043..04bcf0c 100644 --- a/flow-python/setup.py +++ b/flow-python/setup.py @@ -9,11 +9,33 @@ #!/usr/bin/env python # coding=utf-8 -from setuptools import setup +from setuptools import setup, find_packages setup( name="pyflow", version="0.1.0", packages=["pyflow"], + author="Megvii IPU-SDK Team", + author_email="megengine@megvii.com", + url="https://github.com/MegEngine/MegFlow", + include_package_data=True, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: C++', + 'Programming Language :: Rust', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + python_requires='>=3.7,<3.8', ) diff --git a/flow-python/whl-py36-setup.py b/flow-python/whl-py36-setup.py new file mode 100644 index 0000000..10536c7 --- /dev/null +++ b/flow-python/whl-py36-setup.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# MegFlow is Licensed under the Apache License, Version 2.0 (the "License") +# +# Copyright (c) 2019-2021 Megvii Inc. All rights reserved. +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +#!/usr/bin/env python +# coding=utf-8 +from setuptools import setup, find_packages + +setup( + name="pyflow", + version="0.1.0", + packages=["pyflow"], + author="Megvii IPU-SDK Team", + author_email="megengine@megvii.com", + url="https://github.com/MegEngine/MegFlow", + include_package_data=True, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: C++', + 'Programming Language :: Rust', + 'Programming Language :: Python :: 3.6', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + python_requires='>=3.6,<3.7', + package_data={ + "":['run_with_plugins'] + } +) + diff --git a/flow-python/whl-py37-setup.py b/flow-python/whl-py37-setup.py new file mode 100644 index 0000000..cb8eb63 --- /dev/null +++ b/flow-python/whl-py37-setup.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# MegFlow is Licensed under the Apache License, Version 2.0 (the "License") +# +# Copyright (c) 2019-2021 Megvii Inc. All rights reserved. +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +#!/usr/bin/env python +# coding=utf-8 +from setuptools import setup, find_packages + +setup( + name="pyflow", + version="0.1.0", + packages=["pyflow"], + author="Megvii IPU-SDK Team", + author_email="megengine@megvii.com", + url="https://github.com/MegEngine/MegFlow", + include_package_data=True, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: C++', + 'Programming Language :: Rust', + 'Programming Language :: Python :: 3.7', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + python_requires='>=3.7,<3.8', + package_data={ + "":['run_with_plugins'] + } +) + diff --git a/flow-python/whl-py38-setup.py b/flow-python/whl-py38-setup.py new file mode 100644 index 0000000..8f90cb0 --- /dev/null +++ b/flow-python/whl-py38-setup.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# MegFlow is Licensed under the Apache License, Version 2.0 (the "License") +# +# Copyright (c) 2019-2021 Megvii Inc. All rights reserved. +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT ARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +#!/usr/bin/env python +# coding=utf-8 +from setuptools import setup, find_packages + +setup( + name="pyflow", + version="0.1.0", + packages=["pyflow"], + author="Megvii IPU-SDK Team", + author_email="megengine@megvii.com", + url="https://github.com/MegEngine/MegFlow", + include_package_data=True, + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Natural Language :: English', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: C++', + 'Programming Language :: Rust', + 'Programming Language :: Python :: 3.8', + 'Topic :: Software Development :: Libraries :: Application Frameworks', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + python_requires='>=3.8,<3.9', + package_data={ + "":['run_with_plugins'] + } +) +