diff --git a/.github/workflows/macos-x86-cpu.yml b/.github/workflows/macos-x86-cpu.yml new file mode 100644 index 0000000..fd12362 --- /dev/null +++ b/.github/workflows/macos-x86-cpu.yml @@ -0,0 +1,28 @@ +name: macos-x86-cpu +on: + push: + branches: [master] + pull_request: + branches: [master] +jobs: + build: + name: macos-x86-cpu + runs-on: macos-latest + steps: + - name: cancel-previous-runs + uses: styfle/cancel-workflow-action@0.9.1 + with: + access_token: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + toolchain: stable + - run: brew install yasm + - run: cargo build + - run: cargo test + - uses: actions/setup-python@v2 + with: + python-version: '3.8' + architecture: 'x64' + - run: cd flow-python && python3.8 setup.py install --user + - run: cd flow-python/examples && cargo run --example run_with_plugins -- -p logical_test \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 85851fc..7787025 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1975,8 +1975,7 @@ dependencies = [ [[package]] name = "stackful" version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e518515b243405015c2431f737169af4071526330ac199320da7799c51a236e9" +source = "git+https://github.com/nbdd0121/stackful.git?rev=4789e26#4789e2622ccf6efcdb7693f358609b5efd81987c" dependencies = [ "cc", "futures-executor", diff --git a/README.md b/README.md index 1d5f7bc..92b6bff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,9 @@ # MegFlow + +[![GitHub license](https://img.shields.io/badge/license-apache--2--Clause-brightgreen.svg)](./LICENSE) +[![ubuntu](https://img.shields.io/github/workflow/status/MegEngine/MegFlow/ubuntu-x86-cpu?label=ubuntu)](https://github.com/MegEngine/MegFlow/actions/workflows/ubuntu-x86-cpu.yml?query=workflow%3A) +[![macos](https://img.shields.io/github/workflow/status/MegEngine/MegFlow/ubuntu-x86-cpu?label=macos)](https://github.com/MegEngine/MegFlow/actions/workflows/macos-x86-cpu.yml?query=workflow%3A) + MegFlow 是一个面向视觉应用的流式计算框架, 目标是简单、高性能、帮助机器学习应用快速落地。 ## Features diff --git a/docs/FAQ.zh.md b/docs/FAQ.zh.md index 5d7f6d1..189eaf6 100644 --- a/docs/FAQ.zh.md +++ b/docs/FAQ.zh.md @@ -3,9 +3,8 @@ Q：`error while loading shared libraries: libpython3.8.xxx`。 A：如果使用 conda 只需要 - ```bash -$ export LD_LIBRARY_PATH=/home/`whoami`/miniconda3/pkgs/python-3.8.11-h12debd9_0_cpython/lib:${LD_LIBRARY_PATH}` +$ export LD_LIBRARY_PATH=`conda info --base`/pkgs/python-3.8.11-xxx/lib:${LD_LIBRARY_PATH} ``` ___ Q：`cargo run --example run_with_plugins -- -p logical_test ` 无法运行怎么办？ diff --git a/docs/how-to-add-plugins.zh.md b/docs/how-to-add-plugins.zh.md index f39fea0..36f63ac 100644 --- a/docs/how-to-add-plugins.zh.md +++ b/docs/how-to-add-plugins.zh.md @@ -2,8 +2,8 @@ 从一个最简单的例子开始 ``` -import pyflow -@pyflow.register(name="alias", inputs=["inp"], outputs=["out"]) +import megflow +@megflow.register(name="alias", inputs=["inp"], outputs=["out"]) class Node: def __init__(self, name, args): pass diff --git a/docs/how-to-build-from-source.zh.md b/docs/how-to-build-from-source.zh.md index e8f13f2..46a4299 100644 --- a/docs/how-to-build-from-source.zh.md +++ b/docs/how-to-build-from-source.zh.md @@ -66,7 +66,7 @@ $ python3 setup.py install --user **FAQ**：`error while loading shared libraries: libpython3.8.xxx`。如果使用 conda 只需要 ```bash -$ export LD_LIBRARY_PATH=/home/`whoami`/miniconda3/pkgs/python-3.8.11-h12debd9_0_cpython/lib:${LD_LIBRARY_PATH} +$ export LD_LIBRARY_PATH=`conda info --base`/pkgs/python-3.8.11-xxx/lib:${LD_LIBRARY_PATH} ``` P.S. 默认 ffmpeg 依赖自动从 github 上拉取源码构建，这会使得首次构建的时间较长。若希望缩短首次构建时间，或者希望依赖一个指定版本的 ffmpeg，可以启用环境变量`CARGO_FEATURE_PREBUILD`并参考[rust-ffmpeg](https://github.com/zmwangx/rust-ffmpeg/wiki/Notes-on-building)自行构建 diff --git a/docs/how-to-run-in-15-minutes.zh.md b/docs/how-to-run-in-15-minutes.zh.md index 8048e49..36a1273 100644 --- a/docs/how-to-run-in-15-minutes.zh.md +++ b/docs/how-to-run-in-15-minutes.zh.md @@ -33,10 +33,10 @@ $ conda activate py38 从 [MegFlow release](https://github.com/MegEngine/MegFlow/releases) 下载对应 python 版本的 .whl 包，安装 ```bash -$ python3 -m pip install pyflow-0.1.0-py38-none-linux_x86_64.whl --force-reinstall +$ python3 -m pip install megflow-0.1.0-py38-none-linux_x86_64.whl --force-reinstall ``` -.whl 封装了 `pyflow`和 Rust 可执行文件 `run_with_plugins_python_wrap`。完成后应该可以运行 +.whl 封装了 `megflow`和 Rust 可执行文件 `run_with_plugins_python_wrap`。完成后应该可以运行 ```bash $ run_with_plugins_python_wrap --help run_with_plugins 1.0 @@ -46,7 +46,7 @@ $ python3 Python 3.8.3 (default, May 19 2020, 18:47:26) [GCC 7.3.0] :: Anaconda, Inc. on linux Type "help", "copyright", "credits" or "license" for more information. ->>> import pyflow +>>> import megflow ``` **FAQ**：`error while loading shared libraries: libpython3.8.xxx`。如果使用 conda 只需要 @@ -56,7 +56,7 @@ $ export LD_LIBRARY_PATH=/home/`whoami`/miniconda3/pkgs/python-3.8.11-h12debd9_0 被封装的内容安装在 miniconda3 路径下，有兴趣不妨确认一下 ```bash -$ cd ${HOME}/miniconda3/envs/py38/lib/python3.8/site-packages/pyflow/ +$ cd ${HOME}/miniconda3/envs/py38/lib/python3.8/site-packages/megflow/ $ sudo apt install build-essential -y $ ldd run_with_plugins # 可看到仅依赖常见库 $ ./run_with_plugins --help @@ -73,7 +73,8 @@ $ cd ${MegFlow_PATH}/flow-python/examples $ run_with_plugins_python_wrap -p logical_test ... ``` -> 工作原理：[pyflow](../flow-python/pyflow/__init__.py) 仅是一层接口，由 run_with_plugins “注入”建图/调度/优化等实现。 + +> 工作原理：[megflow](../flow-python/megflow/__init__.py) 仅是一层接口，由 run_with_plugins “注入”建图/调度/优化等实现。 ## Python Built-in Applications diff --git a/flow-python/examples/cat_finder/det.py b/flow-python/examples/cat_finder/det.py index 1122c39..95fe1cd 100644 --- a/flow-python/examples/cat_finder/det.py +++ b/flow-python/examples/cat_finder/det.py @@ -9,7 +9,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register +from megflow import register from loguru import logger from warehouse.detection_yolox import * import numpy as np diff --git a/flow-python/examples/cat_finder/redis_proxy.py b/flow-python/examples/cat_finder/redis_proxy.py index 6f0aae7..3afe524 100644 --- a/flow-python/examples/cat_finder/redis_proxy.py +++ b/flow-python/examples/cat_finder/redis_proxy.py @@ -14,7 +14,7 @@ import base64 import numpy as np from loguru import logger -from pyflow import register +from megflow import register @register(inputs=['inp'], outputs=['out']) class RedisProxy: diff --git a/flow-python/examples/cat_finder/reid_image.py b/flow-python/examples/cat_finder/reid_image.py index 01a3ed3..5ac24ae 100644 --- a/flow-python/examples/cat_finder/reid_image.py +++ b/flow-python/examples/cat_finder/reid_image.py @@ -12,7 +12,7 @@ import megengine as mge import numpy as np from loguru import logger -from pyflow import register +from megflow import register from warehouse.reid_alignedreid import * from warehouse.quality_naive import * diff --git a/flow-python/examples/cat_finder/reid_video.py b/flow-python/examples/cat_finder/reid_video.py index 985bd08..57da218 100644 --- a/flow-python/examples/cat_finder/reid_video.py +++ b/flow-python/examples/cat_finder/reid_video.py @@ -12,7 +12,7 @@ import numpy as np import megengine as mge from loguru import logger -from pyflow import register +from megflow import register from warehouse.reid_alignedreid import * from warehouse.quality_naive import * diff --git a/flow-python/examples/cat_finder/shaper.py b/flow-python/examples/cat_finder/shaper.py index 09d2974..d093553 100644 --- a/flow-python/examples/cat_finder/shaper.py +++ b/flow-python/examples/cat_finder/shaper.py @@ -9,7 +9,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register +from megflow import register from loguru import logger from warehouse.quality_naive import Quality import cv2 diff --git a/flow-python/examples/cat_finder/track.py b/flow-python/examples/cat_finder/track.py index bac2989..006e948 100644 --- a/flow-python/examples/cat_finder/track.py +++ b/flow-python/examples/cat_finder/track.py @@ -10,7 +10,7 @@ # coding=utf-8 from loguru import logger -from pyflow import register, Envelope +from megflow import register, Envelope from warehouse.track_iou import Tracker import numpy as np diff --git a/flow-python/examples/electric_bicycle/det.py b/flow-python/examples/electric_bicycle/det.py index 5617b86..246d266 100644 --- a/flow-python/examples/electric_bicycle/det.py +++ b/flow-python/examples/electric_bicycle/det.py @@ -11,7 +11,7 @@ import cv2 import numpy as np -from pyflow import register +from megflow import register from loguru import logger from warehouse.detection_memd import * diff --git a/flow-python/examples/electric_bicycle/redis_proxy.py b/flow-python/examples/electric_bicycle/redis_proxy.py index bbbd1b2..0f3e8fd 100644 --- a/flow-python/examples/electric_bicycle/redis_proxy.py +++ b/flow-python/examples/electric_bicycle/redis_proxy.py @@ -14,7 +14,7 @@ import base64 import numpy as np from loguru import logger -from pyflow import register +from megflow import register @register(inputs=['inp'], outputs=['out']) class RedisProxy: diff --git a/flow-python/examples/electric_bicycle/shaper.py b/flow-python/examples/electric_bicycle/shaper.py index 80edd3d..dd93a65 100644 --- a/flow-python/examples/electric_bicycle/shaper.py +++ b/flow-python/examples/electric_bicycle/shaper.py @@ -9,7 +9,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register +from megflow import register from loguru import logger from warehouse.quality_naive import Quality import cv2 diff --git a/flow-python/examples/electric_bicycle/track.py b/flow-python/examples/electric_bicycle/track.py index b1ad651..38811fa 100644 --- a/flow-python/examples/electric_bicycle/track.py +++ b/flow-python/examples/electric_bicycle/track.py @@ -10,7 +10,7 @@ # coding=utf-8 from loguru import logger -from pyflow import register, Envelope +from megflow import register, Envelope from warehouse.track_iou import Tracker import numpy as np diff --git a/flow-python/examples/logical_test/buffer.py b/flow-python/examples/logical_test/buffer.py index 3f267d7..e772deb 100644 --- a/flow-python/examples/logical_test/buffer.py +++ b/flow-python/examples/logical_test/buffer.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import res_register +from megflow import res_register @res_register() class Buffer: diff --git a/flow-python/examples/logical_test/printer.py b/flow-python/examples/logical_test/printer.py index b08f41e..27858f4 100644 --- a/flow-python/examples/logical_test/printer.py +++ b/flow-python/examples/logical_test/printer.py @@ -8,7 +8,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register +from megflow import register instance_id = 0 diff --git a/flow-python/examples/logical_test/process.py b/flow-python/examples/logical_test/process.py index 973db8d..2088282 100644 --- a/flow-python/examples/logical_test/process.py +++ b/flow-python/examples/logical_test/process.py @@ -8,7 +8,8 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register +from megflow import register +import multiprocessing as mp from multiprocessing import Process, Pipe def repeat(n, s, r): @@ -31,6 +32,7 @@ def __init__(self, name, args): self.send = s1 self.recv = r2 + mp.set_start_method('fork') self.p = Process(target=repeat, args=(10, s2, r1)) self.p.start() diff --git a/flow-python/examples/logical_test/source.py b/flow-python/examples/logical_test/source.py index 3c4b78d..eb91962 100644 --- a/flow-python/examples/logical_test/source.py +++ b/flow-python/examples/logical_test/source.py @@ -8,7 +8,7 @@ #!/usr/bin/env python # coding=utf-8 -from pyflow import register, Envelope, sleep +from megflow import register, Envelope, sleep @register(outputs=['out']) class Source: diff --git a/flow-python/gen_whl.sh b/flow-python/gen_whl.sh index 3289f82..fb109ee 100755 --- a/flow-python/gen_whl.sh +++ b/flow-python/gen_whl.sh @@ -3,26 +3,26 @@ CONDA_BASE=$(conda info --base) source $CONDA_BASE/etc/profile.d/conda.sh conda activate py36 cargo build --example run_with_plugins --release -cp ../target/release/examples/run_with_plugins ./pyflow/run_with_plugins_inner +cp ../target/release/examples/run_with_plugins ./megflow/run_with_plugins_inner rm -rf ./build py='py36' python3 setup.py bdist_wheel -p linux-x86_64 -d py36_dist conda activate py37 cargo build --example run_with_plugins --release -cp ../target/release/examples/run_with_plugins ./pyflow/run_with_plugins_inner +cp ../target/release/examples/run_with_plugins ./megflow/run_with_plugins_inner rm -rf ./build py='py37' python3 setup.py bdist_wheel -p linux-x86_64 -d py37_dist conda activate py38 cargo build --example run_with_plugins --release -cp ../target/release/examples/run_with_plugins ./pyflow/run_with_plugins_inner +cp ../target/release/examples/run_with_plugins ./megflow/run_with_plugins_inner rm -rf ./build py='py38' python3 setup.py bdist_wheel -p linux-x86_64 -d py38_dist rm -rf dist mkdir dist -cp py36_dist/pyflow-0.1.0-py36-none-linux_x86_64.whl dist/ -cp py37_dist/pyflow-0.1.0-py37-none-linux_x86_64.whl dist/ -cp py38_dist/pyflow-0.1.0-py38-none-linux_x86_64.whl dist/ +cp py36_dist/megflow-0.1.0-py36-none-linux_x86_64.whl dist/ +cp py37_dist/megflow-0.1.0-py37-none-linux_x86_64.whl dist/ +cp py38_dist/megflow-0.1.0-py38-none-linux_x86_64.whl dist/ rm -rf py36_dist py37_dist py38_dist diff --git a/flow-python/pyflow/__init__.py b/flow-python/megflow/__init__.py similarity index 100% rename from flow-python/pyflow/__init__.py rename to flow-python/megflow/__init__.py diff --git a/flow-python/pyflow/command_line.py b/flow-python/megflow/command_line.py similarity index 58% rename from flow-python/pyflow/command_line.py rename to flow-python/megflow/command_line.py index 469397d..81d27b4 100644 --- a/flow-python/pyflow/command_line.py +++ b/flow-python/megflow/command_line.py @@ -1,19 +1,22 @@ +#!/usr/bin/env python +# coding=utf-8 import os import sys import subprocess import pkg_resources + + def main(): - bin_exist = pkg_resources.resource_exists('pyflow', 'run_with_plugins') + bin_exist = pkg_resources.resource_exists('megflow', 'run_with_plugins_inner') if not bin_exist: print('cannot find run_with_plugins, exit!') sys.exit(-1) - bin_path = pkg_resources.resource_filename('pyflow', 'run_with_plugins') + bin_path = pkg_resources.resource_filename('megflow', 'run_with_plugins_inner') sys.argv[0] = bin_path ret = subprocess.Popen(sys.argv) ret.wait() + if __name__ == '__main__': main() - - diff --git a/flow-python/pyflow/registry.py b/flow-python/megflow/registry.py similarity index 100% rename from flow-python/pyflow/registry.py rename to flow-python/megflow/registry.py diff --git a/flow-python/setup.py b/flow-python/setup.py index cab2062..5c73ec4 100644 --- a/flow-python/setup.py +++ b/flow-python/setup.py @@ -9,21 +9,27 @@ #!/usr/bin/env python # coding=utf-8 -from setuptools import setup, find_packages +from setuptools import setup import sys import os +devel_version = os.environ.get("DEVEL_VERSION") +if not devel_version: + devel_version = "0.1.0" # fall back + +py_version = sys.version_info + if __name__ == '__main__': - py = os.getenv('py') - if py is None: - py = 'py{}{}'.format(sys.version_info[0], sys.version_info[1]) - assert py.startswith('py3') - minor = int(py[-1]) setup( - options={'bdist_wheel':{'python_tag': py}}, - name="pyflow", - version="0.1.0", - packages=["pyflow"], + options={ + 'bdist_wheel': { + 'python_tag': "py{}{}".format(py_version.major, + py_version.minor), + } + }, + name="megflow", + version=devel_version, + packages=["megflow"], author="Megvii IPU-SDK Team", author_email="megengine@megvii.com", url="https://github.com/MegEngine/MegFlow", @@ -35,7 +41,8 @@ 'Natural Language :: English', 'Operating System :: POSIX :: Linux', 'Programming Language :: Rust', - 'Programming Language :: Python :: 3.{}'.format(minor), + 'Programming Language :: Python :: {}.{}'.format( + py_version.major, py_version.minor), 'Topic :: Software Development :: Libraries :: Application Frameworks', 'Topic :: Scientific/Engineering', 'Topic :: Scientific/Engineering :: Mathematics', @@ -44,12 +51,12 @@ 'Topic :: Software Development :: Libraries', 'Topic :: Software Development :: Libraries :: Python Modules', ], - python_requires='>=3.{},<3.{}'.format(minor, minor+1), - package_data={ - "":['run_with_plugins_inner'] - }, + python_requires='>={}.{},<{}.{}'.format(py_version.major, + py_version.minor, + py_version.major, + py_version.minor + 1), + package_data={"": ['run_with_plugins_inner']}, entry_points={ - 'console_scripts':['run_with_plugins=pyflow.command_line:main'], + 'console_scripts':['run_with_plugins=megflow.command_line:main'], }, ) - diff --git a/flow-rs/src/loader/python/envelope.rs b/flow-rs/src/loader/python/envelope.rs index 4fd9cb3..e2f9e10 100644 --- a/flow-rs/src/loader/python/envelope.rs +++ b/flow-rs/src/loader/python/envelope.rs @@ -19,7 +19,7 @@ use pyo3::{ static ERR_MSG: &str = "use after move"; -#[pyclass(name = "Envelope", module = "pyflow")] +#[pyclass(name = "Envelope", module = "megflow")] pub(crate) struct PyEnvelope { pub imp: Option>, } diff --git a/flow-rs/src/loader/python/mod.rs b/flow-rs/src/loader/python/mod.rs index da62c6e..2a7a42d 100644 --- a/flow-rs/src/loader/python/mod.rs +++ b/flow-rs/src/loader/python/mod.rs @@ -136,7 +136,7 @@ impl Loader for PythonLoader { } ONCE_REGISTER.call_once(|| { - let module = py.import("pyflow").expect("module pyflow not found"); + let module = py.import("megflow").expect("module megflow not found"); utils::utils_register(module).expect("python utility functions register fault"); envelope::envelope_register(module).expect("python envelope register fault"); }); @@ -144,7 +144,7 @@ impl Loader for PythonLoader { py.import(module_name.as_str())?; let plugins_param: HashMap> = py - .import("pyflow")? + .import("megflow")? .getattr("collect")? .call0()? .extract()?;