diff --git a/.github/workflows/ci-lint-cvat-recording-oracle.yaml b/.github/workflows/ci-lint-cvat-recording-oracle.yaml index 046a857d0b..baa61a0dae 100644 --- a/.github/workflows/ci-lint-cvat-recording-oracle.yaml +++ b/.github/workflows/ci-lint-cvat-recording-oracle.yaml @@ -29,5 +29,5 @@ jobs: python-version: '3.10' cache: 'poetry' cache-dependency-path: ${{ env.WORKING_DIR }}/poetry.lock - - run: poetry install --no-root --only lint + - run: poetry install --no-root --only dev - run: poetry run pre-commit run --all-files diff --git a/.github/workflows/ci-test-cvat-exchange-oracle.yaml b/.github/workflows/ci-test-cvat-exchange-oracle.yaml index 8ec4a147ba..8f49f2b7f4 100644 --- a/.github/workflows/ci-test-cvat-exchange-oracle.yaml +++ b/.github/workflows/ci-test-cvat-exchange-oracle.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v7 - name: CVAT Exchange Oracle tests working-directory: ./packages/examples/cvat/exchange-oracle - run: docker compose -f docker-compose.test.yml up --attach test --exit-code-from test + run: docker compose -f docker-compose.test.yml -f docker-compose.test.head.yml up --attach test --exit-code-from test diff --git a/.github/workflows/ci-test-cvat-recording-oracle.yaml b/.github/workflows/ci-test-cvat-recording-oracle.yaml index bf975f3ce1..944e079de2 100644 --- a/.github/workflows/ci-test-cvat-recording-oracle.yaml +++ b/.github/workflows/ci-test-cvat-recording-oracle.yaml @@ -15,4 +15,4 @@ jobs: - uses: actions/checkout@v7 - name: CVAT Recording Oracle tests working-directory: ./packages/examples/cvat/recording-oracle - run: docker compose -f docker-compose.test.yml up --attach test --exit-code-from test + run: docker compose -f docker-compose.test.yml -f docker-compose.test.head.yml up --attach test --exit-code-from test diff --git a/packages/examples/cvat/exchange-oracle/Dockerfile b/packages/examples/cvat/exchange-oracle/Dockerfile index 69e547480d..48a5b31fc8 100644 --- a/packages/examples/cvat/exchange-oracle/Dockerfile +++ b/packages/examples/cvat/exchange-oracle/Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update -y && \ apt-get install -y jq ffmpeg libsm6 libxext6 && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache poetry +RUN pip install --no-cache 'poetry==1.8.5' COPY pyproject.toml poetry.lock ./ diff --git a/packages/examples/cvat/exchange-oracle/README.md b/packages/examples/cvat/exchange-oracle/README.md index d6d91ce62b..b1bcb63f6e 100644 --- a/packages/examples/cvat/exchange-oracle/README.md +++ b/packages/examples/cvat/exchange-oracle/README.md @@ -71,8 +71,47 @@ Available at `/docs` route ### Tests -To run tests +#### "oneshot" run + +A single command to build, run, and tear down the test suite: + +```sh +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + up --build test --attach test --exit-code-from test; \ + docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + down +``` + +Use this option for CI and for clean single time test runs. + +#### Running separate elements + +Dev builds require faster iteration and some components may require more control. The following +commands allow running just the services, build, tear down, and run the test suite: + ```sh -docker compose -p "test" -f docker-compose.test.yml up --build test --attach test --exit-code-from test; \ - docker compose -p "test" -f docker-compose.test.yml down -``` \ No newline at end of file +# run services +docker compose -p "test" \ + -f docker-compose.test.yml \ + up -d --build + +# run the tests +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + -f docker-compose.test.head.dev.yml \ + up --build test --attach test --exit-code-from test + +# tear down +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + -f docker-compose.test.head.dev.yml \ + down +``` + +The dev setup mounts the local directory to speed the things up. diff --git a/packages/examples/cvat/exchange-oracle/alembic/env.py b/packages/examples/cvat/exchange-oracle/alembic/env.py index ac430d2eda..b642b5942c 100644 --- a/packages/examples/cvat/exchange-oracle/alembic/env.py +++ b/packages/examples/cvat/exchange-oracle/alembic/env.py @@ -12,17 +12,19 @@ config.set_main_option("sqlalchemy.url", Config.postgres_config.connection_url()) # Interpret the config file for Python logging. -# This line sets up loggers basically. if config.config_file_name is not None: - fileConfig(config.config_file_name) + fileConfig( + config.config_file_name, + disable_existing_loggers=config.attributes.get("disable_existing_loggers", True), + ) + +from src.db import Base # noqa: E402 # add your model's MetaData object here # for 'autogenerate' support # from myapp import mymodel # target_metadata = mymodel.Base.metadata -from src.db import Base # noqa: E402 - target_metadata = Base.metadata # other values from the config, defined by the needs of env.py, diff --git a/packages/examples/cvat/exchange-oracle/debug.py b/packages/examples/cvat/exchange-oracle/debug.py index f11128dd65..4b89982d89 100644 --- a/packages/examples/cvat/exchange-oracle/debug.py +++ b/packages/examples/cvat/exchange-oracle/debug.py @@ -94,10 +94,11 @@ def patched_get_escrow(chain_id: int, escrow_address: str) -> EscrowData: count=1, factory_address="", launcher="", + job_requester_id="", status="Pending", token="HMT", # noqa: S106 total_funded_amount=10, - created_at=datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc), + created_at=int(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timestamp()), manifest=(f"http://{Config.storage_config.endpoint_url}/manifests/{manifest_file}"), ) diff --git a/packages/examples/cvat/exchange-oracle/docker-compose.test.head.dev.yml b/packages/examples/cvat/exchange-oracle/docker-compose.test.head.dev.yml new file mode 100644 index 0000000000..e50efbbe73 --- /dev/null +++ b/packages/examples/cvat/exchange-oracle/docker-compose.test.head.dev.yml @@ -0,0 +1,7 @@ +services: + test: + build: + context: ./ + dockerfile: dockerfiles/test.dev.Dockerfile + volumes: + - "./:/app:ro" diff --git a/packages/examples/cvat/exchange-oracle/docker-compose.test.head.yml b/packages/examples/cvat/exchange-oracle/docker-compose.test.head.yml new file mode 100644 index 0000000000..b885070446 --- /dev/null +++ b/packages/examples/cvat/exchange-oracle/docker-compose.test.head.yml @@ -0,0 +1,35 @@ +services: + test: + build: + context: ./ + dockerfile: dockerfiles/test.ci.Dockerfile + environment: + SQLALCHEMY_SILENCE_UBER_WARNING: 1 + ENVIRONMENT: 'test' + PG_PORT: 5432 + PG_HOST: 'postgres' + PG_USER: 'test' + PG_PASSWORD: 'test' + PG_DB: 'exchange_oracle_test' + WEB3_HTTP_PROVIDER_URI: 'http://blockchain-node:8545' + STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' + STORAGE_ACCESS_KEY: 'dev' + STORAGE_SECRET_KEY: 'devdevdev' + STORAGE_RESULTS_BUCKET_NAME: 'results' + STORAGE_USE_SSL: 'False' + STORAGE_PROVIDER: 'aws' + ENABLE_CUSTOM_CLOUD_HOST: 'Yes' + REDIS_HOST: 'redis' + depends_on: + postgres: + condition: service_started + redis: + condition: service_started + minio-mc: + condition: service_completed_successfully + # Used to reference localhost since minio:9000 is rejected by the SDK URL validator + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - test-network + tty: true diff --git a/packages/examples/cvat/exchange-oracle/docker-compose.test.yml b/packages/examples/cvat/exchange-oracle/docker-compose.test.yml index f6a1ad6e3d..216ac36132 100644 --- a/packages/examples/cvat/exchange-oracle/docker-compose.test.yml +++ b/packages/examples/cvat/exchange-oracle/docker-compose.test.yml @@ -64,41 +64,6 @@ services: networks: - test-network - test: - build: - context: ./ - dockerfile: dockerfiles/test.Dockerfile - environment: - SQLALCHEMY_SILENCE_UBER_WARNING: 1 - ENVIRONMENT: 'test' - PG_PORT: 5432 - PG_HOST: 'postgres' - PG_USER: 'test' - PG_PASSWORD: 'test' - PG_DB: 'exchange_oracle_test' - WEB3_HTTP_PROVIDER_URI: 'http://blockchain-node:8545' - STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' - STORAGE_ACCESS_KEY: 'dev' - STORAGE_SECRET_KEY: 'devdevdev' - STORAGE_RESULTS_BUCKET_NAME: 'results' - STORAGE_USE_SSL: 'False' - STORAGE_PROVIDER: 'aws' - ENABLE_CUSTOM_CLOUD_HOST: 'Yes' - REDIS_HOST: 'redis' - depends_on: - postgres: - condition: service_started - redis: - condition: service_started - minio-mc: - condition: service_completed_successfully - # Used to reference localhost since minio:9000 is rejected by the SDK URL validator - extra_hosts: - - "host.docker.internal:host-gateway" - networks: - - test-network - tty: true - networks: test-network: driver: bridge \ No newline at end of file diff --git a/packages/examples/cvat/exchange-oracle/dockerfiles/test.Dockerfile b/packages/examples/cvat/exchange-oracle/dockerfiles/test.ci.Dockerfile similarity index 90% rename from packages/examples/cvat/exchange-oracle/dockerfiles/test.Dockerfile rename to packages/examples/cvat/exchange-oracle/dockerfiles/test.ci.Dockerfile index eaa436f5d0..bbce75ef27 100644 --- a/packages/examples/cvat/exchange-oracle/dockerfiles/test.Dockerfile +++ b/packages/examples/cvat/exchange-oracle/dockerfiles/test.ci.Dockerfile @@ -6,7 +6,7 @@ RUN apt-get update -y && \ apt-get install -y jq ffmpeg libsm6 libxext6 && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache poetry +RUN pip install --no-cache 'poetry==1.8.5' COPY pyproject.toml poetry.lock ./ diff --git a/packages/examples/cvat/exchange-oracle/dockerfiles/test.dev.Dockerfile b/packages/examples/cvat/exchange-oracle/dockerfiles/test.dev.Dockerfile new file mode 100644 index 0000000000..94cdf9cf62 --- /dev/null +++ b/packages/examples/cvat/exchange-oracle/dockerfiles/test.dev.Dockerfile @@ -0,0 +1,22 @@ +# Local dev test image: only dependencies are baked in; the source tree is bind-mounted at +# runtime by docker-compose.test.head.dev.yml. Skipping the source COPY keeps rebuilds fast when +# iterating locally. For CI / clean self-contained runs, use test.ci.Dockerfile instead. +FROM python:3.10 + +WORKDIR /app + +RUN apt-get update -y && \ + apt-get install -y jq ffmpeg libsm6 libxext6 && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache 'poetry==1.8.5' + +COPY pyproject.toml poetry.lock ./ + +RUN --mount=type=cache,target=/root/.cache \ + poetry config virtualenvs.create false && \ + poetry install --no-interaction --no-ansi --no-root + +RUN python -m pip uninstall -y poetry pip + +CMD ["pytest"] diff --git a/packages/examples/cvat/exchange-oracle/poetry.lock b/packages/examples/cvat/exchange-oracle/poetry.lock index 5f65733022..720a2f24dd 100644 --- a/packages/examples/cvat/exchange-oracle/poetry.lock +++ b/packages/examples/cvat/exchange-oracle/poetry.lock @@ -228,63 +228,6 @@ tornado = ["tornado (>=4.3)"] twisted = ["twisted"] zookeeper = ["kazoo"] -[[package]] -name = "argon2-cffi" -version = "23.1.0" -description = "Argon2 for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, - {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, -] - -[package.dependencies] -argon2-cffi-bindings = "*" - -[package.extras] -dev = ["argon2-cffi[tests,typing]", "tox (>4)"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] -tests = ["hypothesis", "pytest"] -typing = ["mypy"] - -[[package]] -name = "argon2-cffi-bindings" -version = "21.2.0" -description = "Low-level CFFI bindings for Argon2" -optional = false -python-versions = ">=3.6" -files = [ - {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, - {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, -] - -[package.dependencies] -cffi = ">=1.0.1" - -[package.extras] -dev = ["cogapp", "pre-commit", "pytest", "wheel"] -tests = ["pytest"] - [[package]] name = "async-timeout" version = "4.0.3" @@ -2128,20 +2071,19 @@ socks = ["socksio (==1.*)"] [[package]] name = "human-protocol-sdk" -version = "4.3.0" +version = "7.3.1" description = "A python library to launch escrow contracts to the HUMAN network." optional = false python-versions = "*" files = [ - {file = "human_protocol_sdk-4.3.0-py3-none-any.whl", hash = "sha256:498276ba47157615df7e914374fcb51f788e1892e2c169432deb2055108da525"}, - {file = "human_protocol_sdk-4.3.0.tar.gz", hash = "sha256:a1d172899c79c67d9b4266854252e02ae17f157bf1744710843e376c98c95738"}, + {file = "human_protocol_sdk-7.3.1-py3-none-any.whl", hash = "sha256:b3a410b6526f2742dad76cf866a2f6b35406631395df83e913cffa58bcc6a4db"}, + {file = "human_protocol_sdk-7.3.1.tar.gz", hash = "sha256:2df38610d2d4714373d7d0f7e30f7ca218b19746d849b2aa3a3bd0b3b2dfbdd4"}, ] [package.dependencies] aiohttp = "<4.0.0" cryptography = "*" eth-typing = "*" -minio = "*" pgpy = "*" validators = "*" web3 = "*" @@ -2590,24 +2532,6 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] -[[package]] -name = "minio" -version = "7.2.15" -description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" -optional = false -python-versions = ">=3.9" -files = [ - {file = "minio-7.2.15-py3-none-any.whl", hash = "sha256:c06ef7a43e5d67107067f77b6c07ebdd68733e5aa7eed03076472410ca19d876"}, - {file = "minio-7.2.15.tar.gz", hash = "sha256:5247df5d4dca7bfa4c9b20093acd5ad43e82d8710ceb059d79c6eea970f49f79"}, -] - -[package.dependencies] -argon2-cffi = "*" -certifi = "*" -pycryptodome = "*" -typing-extensions = "*" -urllib3 = "*" - [[package]] name = "msgpack" version = "1.1.0" @@ -5070,4 +4994,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10,<3.13" -content-hash = "d4b0b2cd3671d6e118b5fe545fe3ec2e69e9345bf321443ebf7f34368c085ab4" +content-hash = "84f9ce0ad56ffab8a5101eba4d9102f9df7d27d3663fdc08aaeafa49f7f467ab" diff --git a/packages/examples/cvat/exchange-oracle/pyproject.toml b/packages/examples/cvat/exchange-oracle/pyproject.toml index 7ca5e23a21..428e3a87f4 100644 --- a/packages/examples/cvat/exchange-oracle/pyproject.toml +++ b/packages/examples/cvat/exchange-oracle/pyproject.toml @@ -15,7 +15,6 @@ psycopg2 = "^2.9.6" sqlalchemy-utils = "^0.41.1" alembic = "^1.11.1" httpx = "^0.24.1" -pytest = "^7.2.2" cvat-sdk = "2.37.0" sqlalchemy = "^2.0.16" apscheduler = "^3.10.1" @@ -35,12 +34,13 @@ starlette = ">=0.40.0" # avoid the vulnerability with multipart/form-data cryptography = "<44.0.0" # human-protocol-sdk -> pgpy dep requires cryptography < 45 aiocache = {extras = ["msgpack", "redis"], version = "^0.12.3"} # convenient api for redis (async) cachelib = "^0.13.0" # convenient api for redis (sync) -human-protocol-sdk = "^4.3.0" +human-protocol-sdk = "^7.3.1" [tool.poetry.group.dev.dependencies] pre-commit = "^3.0.4" ruff = "^0.6.0" +pytest = "^7.2.2" pytest-mock = "^3.14.0" [tool.ruff] diff --git a/packages/examples/cvat/exchange-oracle/src/chain/escrow.py b/packages/examples/cvat/exchange-oracle/src/chain/escrow.py index cae6250c8c..df4deed4ad 100644 --- a/packages/examples/cvat/exchange-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/exchange-oracle/src/chain/escrow.py @@ -1,10 +1,11 @@ import json from functools import partial +import httpx from human_protocol_sdk.constants import ChainId, Status from human_protocol_sdk.encryption import Encryption, EncryptionUtils from human_protocol_sdk.escrow import EscrowData, EscrowUtils -from human_protocol_sdk.storage import StorageUtils +from human_protocol_sdk.utils import validate_url from src.chain.web3 import get_token_symbol from src.core.config import Config @@ -12,6 +13,24 @@ from src.services.cache import Cache +class ManifestNotAvailableError(Exception): + """Raised when the escrow manifest cannot be retrieved.""" + + +def _get_manifest_content(manifest: str) -> str: + if validate_url(manifest): + try: + response = httpx.get(manifest, follow_redirects=True) + response.raise_for_status() + except Exception as e: + raise ManifestNotAvailableError( + f"failed to download manifest from {manifest}: {e}" + ) from e + return response.text + + return manifest + + def get_escrow(chain_id: int, escrow_address: str) -> EscrowData: escrow = EscrowUtils.get_escrow(ChainId(chain_id), escrow_address) if not escrow: @@ -48,7 +67,7 @@ def validate_escrow( def download_manifest(chain_id: int, escrow_address: str) -> dict: escrow = get_escrow(chain_id, escrow_address) - manifest_content = StorageUtils.download_file_from_url(escrow.manifest).decode("utf-8") + manifest_content = _get_manifest_content(escrow.manifest) if EncryptionUtils.is_encrypted(manifest_content): encryption = Encryption( diff --git a/packages/examples/cvat/exchange-oracle/src/core/tasks/skeletons_from_boxes.py b/packages/examples/cvat/exchange-oracle/src/core/tasks/skeletons_from_boxes.py index 7459b904cb..0809d65316 100644 --- a/packages/examples/cvat/exchange-oracle/src/core/tasks/skeletons_from_boxes.py +++ b/packages/examples/cvat/exchange-oracle/src/core/tasks/skeletons_from_boxes.py @@ -29,7 +29,7 @@ class RoiInfo: # RoI is centered on the bbox center # Coordinates can be out of image boundaries. - # In this case RoI includes extra margins to be centered on bbox center + # In this case RoI includes extra margins to be centered on bbox center. roi_x: int roi_y: int roi_w: int diff --git a/packages/examples/cvat/exchange-oracle/src/endpoints/serializers.py b/packages/examples/cvat/exchange-oracle/src/endpoints/serializers.py index 219d40af40..05b01ffb80 100644 --- a/packages/examples/cvat/exchange-oracle/src/endpoints/serializers.py +++ b/packages/examples/cvat/exchange-oracle/src/endpoints/serializers.py @@ -1,16 +1,18 @@ from contextlib import ExitStack, suppress -from typing import Literal -from human_protocol_sdk.storage import StorageFileNotFoundError from sqlalchemy.orm import Session import src.services.cvat as cvat_service -from src.chain.escrow import get_escrow_fund_token_symbol, get_escrow_manifest -from src.core.manifest import TaskManifest +from src.chain.escrow import ( + ManifestNotAvailableError, + get_escrow_fund_token_symbol, + get_escrow_manifest, +) +from src.core.manifest import parse_manifest from src.core.types import AssignmentStatuses, ProjectStatuses from src.db import SessionLocal from src.schemas import exchange as service_api -from src.utils.assignments import compose_assignment_url, parse_manifest +from src.utils.assignments import compose_assignment_url PROJECT_COMPLETED_STATUSES = { ProjectStatuses.recorded, @@ -21,7 +23,6 @@ def serialize_job( project: str | cvat_service.Project, *, - manifest: None | TaskManifest | Literal[False] = None, session: Session | None = None, ) -> service_api.JobResponse: with ExitStack() as es: @@ -37,11 +38,8 @@ def serialize_job( f"or a cvat_service.Project instance, not {project!r}" ) - if manifest is None: - with suppress(StorageFileNotFoundError): - manifest = parse_manifest( - get_escrow_manifest(project.chain_id, project.escrow_address) - ) + with suppress(ManifestNotAvailableError): + manifest = parse_manifest(get_escrow_manifest(project.chain_id, project.escrow_address)) if project.status == ProjectStatuses.canceled: api_status = service_api.JobStatuses.canceled @@ -63,7 +61,7 @@ def serialize_job( reward_token=reward_token, created_at=project.created_at, updated_at=project.updated_at, - qualifications=manifest.annotation.qualifications, + qualifications=manifest.annotation.qualifications if manifest else [], ) @@ -78,7 +76,6 @@ def serialize_assignment( assignment: str | cvat_service.Assignment, *, project: None | str | cvat_service.Project = None, - manifest: None | TaskManifest | Literal[False] = None, session: Session | None = None, ) -> service_api.AssignmentResponse: with ExitStack() as es: @@ -104,11 +101,8 @@ def serialize_assignment( f"or a cvat_service.Project instance, not {project!r}" ) - if manifest is None: - with suppress(StorageFileNotFoundError): - manifest = parse_manifest( - get_escrow_manifest(project.chain_id, project.escrow_address) - ) + with suppress(ManifestNotAvailableError): + manifest = parse_manifest(get_escrow_manifest(project.chain_id, project.escrow_address)) assignment_status_mapping = { AssignmentStatuses.created: service_api.AssignmentStatuses.active, diff --git a/packages/examples/cvat/exchange-oracle/tests/conftest.py b/packages/examples/cvat/exchange-oracle/tests/conftest.py index 058049d12d..c2b92ac0ed 100644 --- a/packages/examples/cvat/exchange-oracle/tests/conftest.py +++ b/packages/examples/cvat/exchange-oracle/tests/conftest.py @@ -8,16 +8,19 @@ import pytest from fastapi.testclient import TestClient -from sqlalchemy import TextClause, text +from sqlalchemy import TextClause, inspect, text from sqlalchemy.orm import Session from sqlalchemy_utils import create_database, database_exists, drop_database from alembic import command as alembic_command from alembic.config import Config from src import app -from src.db import SessionLocal, engine +from src.db import Base, SessionLocal, engine alembic_config = Config(Path(__file__).parent.parent / "alembic.ini") +# Don't let Alembic's fileConfig() disable the app's loggers when migrations run in-process, +# otherwise swallowed cron/webhook handler errors are hidden from the pytest report. +alembic_config.attributes["disable_existing_loggers"] = False @dataclass @@ -49,39 +52,49 @@ def setup_db(alembic) -> None: if database_exists(engine.url): drop_database(engine.url) create_database(engine.url) - yield # Run the test cases - # Upgrade to the latest version after all tests are done, - # this helps with inspection of the latest schema. - with engine.connect() as connection: - connection.execute(alembic.upgrade) - - -@pytest.fixture(autouse=True) -def init_db(alembic) -> None: - """ - Runs the recorded Alembic upgrade and downgrade SQL for each test. - This ensures correctness of alembic migrations. - """ + # Apply the schema once for the whole session. Per-test resets only truncate the DB. try: with engine.connect() as connection: + # Validate the migration round-trip once + connection.execute(alembic.upgrade) + connection.execute(alembic.downgrade) + connection.execute(alembic.upgrade) except Exception as e: raise RuntimeError( - "Failed to upgrade migrations, `alembic upgrade head` would fail." - " inspect the cause error and change migrations accordingly." + "Alembic migrations must upgrade and downgrade cleanly " + "(`alembic upgrade head` / `alembic downgrade base`). Fix the migrations." ) from e - yield # Run the test case + # The per-test cleanup only DELETEs rows; it does not reset sequences. Guard that assumption so + # that a future migration adding a serial/identity column fails loudly here rather than silently + # leaking sequence state (e.g. auto-increment ids) across tests. + sequences = inspect(engine).get_sequence_names() + assert not sequences, ( + f"Found DB sequences {sequences}: the per-test cleanup DELETEs rows without resetting " + "sequences, so ids would leak between tests. Reset them per test (e.g. TRUNCATE ... " + "RESTART IDENTITY, or ALTER SEQUENCE) and update the cleanup." + ) + + +# Clear every table between tests. DELETE in reverse dependency order is dramatically faster than +# TRUNCATE on this schema: TRUNCATE does per-table file truncation + fsync (~270ms for all tables), +# while DELETE on already-empty tables is ~0. No table uses an autoincrement primary key, so there +# are no sequences to reset (guarded in setup_db). Each DELETE is a single SQL expression. +_CLEANUP_SQL = text( + ";\n".join( + str(table.delete().compile(dialect=engine.dialect)) + for table in reversed(Base.metadata.sorted_tables) + ) +) - try: - with engine.connect() as connection: - connection.execute(alembic.downgrade) - except Exception as e: - raise RuntimeError( - "Failed to downgrade migrations, `alembic downgrade head:base` would fail." - " inspect the cause error and change migrations accordingly." - ) from e + +@pytest.fixture(autouse=True) +def init_db() -> None: + """Reset the database to an empty, migrated schema before each test.""" + with engine.begin() as connection: + connection.execute(_CLEANUP_SQL) @pytest.fixture(scope="module") diff --git a/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_escrow.py index c2569a6861..d1b68d29ed 100644 --- a/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_escrow.py @@ -43,10 +43,11 @@ def setUp(self): count=0, factory_address=FACTORY_ADDRESS, launcher=JOB_LAUNCHER_ADDRESS, + job_requester_id=JOB_LAUNCHER_ADDRESS, status=Status.Pending.name, token=TOKEN_ADDRESS, total_funded_amount=1000, - created_at="", + created_at="0", recording_oracle=RECORDING_ORACLE_ADDRESS, exchange_oracle=EXCHANGE_ORACLE_ADDRESS, reputation_oracle=REPUTATION_ORACLE_ADDRESS, @@ -83,9 +84,9 @@ def test_validate_escrow_without_funds(self): def test_get_escrow_manifest(self): with ( patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function, - patch("src.chain.escrow.StorageUtils.download_file_from_url") as mock_download, + patch("src.chain.escrow._get_manifest_content") as mock_download, ): - mock_download.return_value = json.dumps({"title": "test"}).encode() + mock_download.return_value = json.dumps({"title": "test"}) mock_function.return_value = self.escrow_data manifest = get_escrow_manifest(chain_id, escrow_address) assert isinstance(manifest, dict) @@ -94,7 +95,7 @@ def test_get_escrow_manifest(self): def test_get_encrypted_escrow_manifest(self): with ( patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function, - patch("src.chain.escrow.StorageUtils.download_file_from_url") as mock_download, + patch("src.chain.escrow._get_manifest_content") as mock_download, patch("src.core.config.Config.encryption_config.pgp_private_key", PGP_PRIVATE_KEY1), patch("src.core.config.Config.encryption_config.pgp_passphrase", PGP_PASSPHRASE), patch( @@ -112,7 +113,7 @@ def test_get_encrypted_escrow_manifest(self): ) assert encrypted_manifest != original_manifest - mock_download.return_value = encrypted_manifest.encode() + mock_download.return_value = encrypted_manifest downloaded_manifest_content = get_escrow_manifest(chain_id, escrow_address) assert downloaded_manifest_content == original_manifest_content diff --git a/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_kvstore.py index 86438fbedb..8c65f49917 100644 --- a/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/exchange-oracle/tests/integration/chain/test_kvstore.py @@ -40,10 +40,11 @@ def setup(self): count=0, factory_address=FACTORY_ADDRESS, launcher=JOB_LAUNCHER_ADDRESS, + job_requester_id=JOB_LAUNCHER_ADDRESS, status=Status.Pending.name, token=TOKEN_ADDRESS, total_funded_amount=1000, - created_at="", + created_at="0", recording_oracle=RECORDING_ORACLE_ADDRESS, reputation_oracle=REPUTATION_ORACLE_ADDRESS, ) diff --git a/packages/examples/cvat/exchange-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py b/packages/examples/cvat/exchange-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py index 1638fbcaba..380ed680b6 100644 --- a/packages/examples/cvat/exchange-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py +++ b/packages/examples/cvat/exchange-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py @@ -144,10 +144,10 @@ def test_process_incoming_reputation_oracle_webhook_escrow_completed( mocker.call(prefix=compose_data_bucket_prefix(escrow_address, chain_id)), mocker.call(prefix=compose_results_bucket_prefix(escrow_address, chain_id)), ] - assert delete_project_mock.mock_calls == [ - mocker.call(project1.cvat_id), - mocker.call(project2.cvat_id), - ] + assert {c.args[0] for c in delete_project_mock.mock_calls} == { + project1.cvat_id, + project2.cvat_id, + } assert delete_cloudstorage_mock.mock_calls == [mocker.call(1)] outgoing_webhooks = list( @@ -197,10 +197,10 @@ def test_process_incoming_reputation_oracle_webhooks_escrow_completed_exceptions mocker.call(prefix=compose_data_bucket_prefix(escrow_address, chain_id)), mocker.call(prefix=compose_results_bucket_prefix(escrow_address, chain_id)), ] - assert delete_project_mock.mock_calls == [ - mocker.call(project1.cvat_id), - mocker.call(project2.cvat_id), - ] + assert {c.args[0] for c in delete_project_mock.mock_calls} == { + project1.cvat_id, + project2.cvat_id, + } assert delete_cloudstorage_mock.mock_calls == [mocker.call(1)] outgoing_webhooks = list( @@ -229,10 +229,10 @@ def test_process_incoming_reputation_oracle_webhooks_escrow_completed_exceptions mocker.call(prefix=compose_data_bucket_prefix(escrow_address, chain_id)), mocker.call(prefix=compose_results_bucket_prefix(escrow_address, chain_id)), ] - assert delete_project_mock.mock_calls == [ - mocker.call(project1.cvat_id), - mocker.call(project2.cvat_id), - ] + assert {c.args[0] for c in delete_project_mock.mock_calls} == { + project1.cvat_id, + project2.cvat_id, + } outgoing_webhooks = list( session.scalars( diff --git a/packages/examples/cvat/recording-oracle/Dockerfile b/packages/examples/cvat/recording-oracle/Dockerfile index 69e547480d..bdc0dd597c 100644 --- a/packages/examples/cvat/recording-oracle/Dockerfile +++ b/packages/examples/cvat/recording-oracle/Dockerfile @@ -3,10 +3,10 @@ FROM python:3.10 WORKDIR /app RUN apt-get update -y && \ - apt-get install -y jq ffmpeg libsm6 libxext6 && \ + apt-get install -y jq libgl1 libsm6 libxext6 && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache poetry +RUN pip install --no-cache 'poetry==1.8.5' COPY pyproject.toml poetry.lock ./ diff --git a/packages/examples/cvat/recording-oracle/README.md b/packages/examples/cvat/recording-oracle/README.md index 6b412f5081..77bee0b36d 100644 --- a/packages/examples/cvat/recording-oracle/README.md +++ b/packages/examples/cvat/recording-oracle/README.md @@ -68,8 +68,46 @@ Available at `/docs` route ### Tests -To run tests +#### "oneshot" run + +A single command to build, run, and tear down the test suite: + ```sh -docker compose -p "test" -f docker-compose.test.yml up --build test --attach test --exit-code-from test; \ - docker compose -p "test" -f docker-compose.test.yml down +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + up --build test --attach test --exit-code-from test; \ + docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml down ``` + +Use this option for CI and for clean single time test runs. + +#### Running separate elements + +Dev builds require faster iteration and some components may require more control. The following +commands allow running just the services, build, tear down, and run the test suite: + +```sh +# run services +docker compose -p "test" \ + -f docker-compose.test.yml \ + up -d --build + +# run the tests +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + -f docker-compose.test.head.dev.yml \ + up --build test --attach test --exit-code-from test + +# tear down +docker compose -p "test" \ + -f docker-compose.test.yml \ + -f docker-compose.test.head.yml \ + -f docker-compose.test.head.dev.yml \ + down +``` + +The dev setup mounts the local directory to speed the things up. diff --git a/packages/examples/cvat/recording-oracle/alembic/env.py b/packages/examples/cvat/recording-oracle/alembic/env.py index 1e85b134f0..fde70d21e7 100644 --- a/packages/examples/cvat/recording-oracle/alembic/env.py +++ b/packages/examples/cvat/recording-oracle/alembic/env.py @@ -12,9 +12,11 @@ config.set_main_option("sqlalchemy.url", Config.postgres_config.connection_url()) # Interpret the config file for Python logging. -# This line sets up loggers basically. if config.config_file_name is not None: - fileConfig(config.config_file_name) + fileConfig( + config.config_file_name, + disable_existing_loggers=config.attributes.get("disable_existing_loggers", True), + ) from src.db import Base # noqa: E402 diff --git a/packages/examples/cvat/recording-oracle/debug.py b/packages/examples/cvat/recording-oracle/debug.py index 4a8d40a42c..37c00dbd9a 100644 --- a/packages/examples/cvat/recording-oracle/debug.py +++ b/packages/examples/cvat/recording-oracle/debug.py @@ -58,10 +58,11 @@ def patched_get_escrow(chain_id: int, escrow_address: str) -> EscrowData: count=1, factory_address="", launcher="", + job_requester_id="", status="Pending", token="HMT", # noqa: S106 total_funded_amount=10, - created_at=datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc), + created_at=int(datetime.datetime(2023, 1, 1, tzinfo=datetime.timezone.utc).timestamp()), manifest=(f"http://{Config.storage_config.endpoint_url}/manifests/{manifest_file}"), ) diff --git a/packages/examples/cvat/recording-oracle/docker-compose.test.head.dev.yml b/packages/examples/cvat/recording-oracle/docker-compose.test.head.dev.yml new file mode 100644 index 0000000000..e50efbbe73 --- /dev/null +++ b/packages/examples/cvat/recording-oracle/docker-compose.test.head.dev.yml @@ -0,0 +1,7 @@ +services: + test: + build: + context: ./ + dockerfile: dockerfiles/test.dev.Dockerfile + volumes: + - "./:/app:ro" diff --git a/packages/examples/cvat/recording-oracle/docker-compose.test.head.yml b/packages/examples/cvat/recording-oracle/docker-compose.test.head.yml new file mode 100644 index 0000000000..044c537d3b --- /dev/null +++ b/packages/examples/cvat/recording-oracle/docker-compose.test.head.yml @@ -0,0 +1,37 @@ +services: + test: + build: + context: ./ + dockerfile: dockerfiles/test.ci.Dockerfile + environment: + SQLALCHEMY_SILENCE_UBER_WARNING: 1 + ENVIRONMENT: 'test' + PG_PORT: 5432 + PG_HOST: 'postgres' + PG_USER: 'test' + PG_PASSWORD: 'test' + PG_DB: 'recording_oracle_test' + WEB3_HTTP_PROVIDER_URI: 'http://localhost:8545' + STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' + STORAGE_ACCESS_KEY: 'dev' + STORAGE_SECRET_KEY: 'devdevdev' + STORAGE_RESULTS_BUCKET_NAME: 'results' + STORAGE_PROVIDER: 'aws' + STORAGE_USE_SSL: False + EXCHANGE_ORACLE_STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' + EXCHANGE_ORACLE_STORAGE_ACCESS_KEY: 'dev' + EXCHANGE_ORACLE_STORAGE_SECRET_KEY: 'devdevdev' + EXCHANGE_ORACLE_STORAGE_RESULTS_BUCKET_NAME: 'results' + EXCHANGE_ORACLE_STORAGE_USE_SSL: False + EXCHANGE_ORACLE_STORAGE_PROVIDER: 'aws' + depends_on: + postgres: + condition: service_started + minio-mc: + condition: service_completed_successfully + # Used to reference localhost since minio:9000 is rejected by the SDK URL validator. + extra_hosts: + - "host.docker.internal:host-gateway" + networks: + - test-network + tty: true diff --git a/packages/examples/cvat/recording-oracle/docker-compose.test.yml b/packages/examples/cvat/recording-oracle/docker-compose.test.yml index 2d7ca3d412..82c78ff286 100644 --- a/packages/examples/cvat/recording-oracle/docker-compose.test.yml +++ b/packages/examples/cvat/recording-oracle/docker-compose.test.yml @@ -11,18 +11,6 @@ services: networks: - test-network - blockchain-node: - build: - context: ../../../../ - dockerfile: docker-setup/Dockerfile.core - healthcheck: - test: yarn local:readiness - interval: 15s - timeout: 30s - retries: 0 - networks: - - test-network - minio: container_name: minio image: minio/minio:RELEASE.2022-05-26T05-48-41Z @@ -58,45 +46,6 @@ services: networks: - test-network - test: - build: - context: ./ - dockerfile: dockerfiles/test.Dockerfile - environment: - SQLALCHEMY_SILENCE_UBER_WARNING: 1 - ENVIRONMENT: 'test' - PG_PORT: 5432 - PG_HOST: 'postgres' - PG_USER: 'test' - PG_PASSWORD: 'test' - PG_DB: 'recording_oracle_test' - WEB3_HTTP_PROVIDER_URI: 'http://blockchain-node:8545' - STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' - STORAGE_ACCESS_KEY: 'dev' - STORAGE_SECRET_KEY: 'devdevdev' - STORAGE_RESULTS_BUCKET_NAME: 'results' - STORAGE_PROVIDER: 'aws' - STORAGE_USE_SSL: False - EXCHANGE_ORACLE_STORAGE_ENDPOINT_URL: 'host.docker.internal:9000' - EXCHANGE_ORACLE_STORAGE_ACCESS_KEY: 'dev' - EXCHANGE_ORACLE_STORAGE_SECRET_KEY: 'devdevdev' - EXCHANGE_ORACLE_STORAGE_RESULTS_BUCKET_NAME: 'results' - EXCHANGE_ORACLE_STORAGE_USE_SSL: False - EXCHANGE_ORACLE_STORAGE_PROVIDER: 'aws' - depends_on: - postgres: - condition: service_started - blockchain-node: - condition: service_healthy - minio-mc: - condition: service_completed_successfully - # Used to reference localhost since minio:9000 is rejected by the SDK URL validator. - extra_hosts: - - "host.docker.internal:host-gateway" - networks: - - test-network - tty: true - networks: test-network: - driver: bridge \ No newline at end of file + driver: bridge diff --git a/packages/examples/cvat/recording-oracle/dockerfiles/blockchain-node.Dockerfile b/packages/examples/cvat/recording-oracle/dockerfiles/blockchain-node.Dockerfile deleted file mode 100644 index a4f747cc28..0000000000 --- a/packages/examples/cvat/recording-oracle/dockerfiles/blockchain-node.Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -# TODO: make this shared and part of local setup -FROM node:24.13-slim - -# curl is needed for healthcheck -RUN apt-get update && apt-get install -y curl - -WORKDIR /usr/src/app - -# Copy expected yarn dist -COPY .yarn ./.yarn -COPY .yarnrc.yml ./ -# Copy files for deps installation -COPY package.json yarn.lock ./ - -COPY tsconfig.base.json ./ -COPY packages/core ./packages/core -RUN yarn workspace @human-protocol/core install -RUN yarn workspace @human-protocol/core build - -EXPOSE 8545 -CMD yarn workspace @human-protocol/core local diff --git a/packages/examples/cvat/recording-oracle/dockerfiles/test.Dockerfile b/packages/examples/cvat/recording-oracle/dockerfiles/test.ci.Dockerfile similarity index 78% rename from packages/examples/cvat/recording-oracle/dockerfiles/test.Dockerfile rename to packages/examples/cvat/recording-oracle/dockerfiles/test.ci.Dockerfile index eaa436f5d0..1c39e52f2a 100644 --- a/packages/examples/cvat/recording-oracle/dockerfiles/test.Dockerfile +++ b/packages/examples/cvat/recording-oracle/dockerfiles/test.ci.Dockerfile @@ -3,10 +3,10 @@ FROM python:3.10 WORKDIR /app RUN apt-get update -y && \ - apt-get install -y jq ffmpeg libsm6 libxext6 && \ + apt-get install -y jq libgl1 libsm6 libxext6 && \ rm -rf /var/lib/apt/lists/* -RUN pip install --no-cache poetry +RUN pip install --no-cache 'poetry==1.8.5' COPY pyproject.toml poetry.lock ./ diff --git a/packages/examples/cvat/recording-oracle/dockerfiles/test.dev.Dockerfile b/packages/examples/cvat/recording-oracle/dockerfiles/test.dev.Dockerfile new file mode 100644 index 0000000000..fe1143c473 --- /dev/null +++ b/packages/examples/cvat/recording-oracle/dockerfiles/test.dev.Dockerfile @@ -0,0 +1,22 @@ +# Local dev test image: only dependencies are baked in; the source tree is bind-mounted at +# runtime by docker-compose.test.head.dev.yml. Skipping the source COPY keeps rebuilds fast when +# iterating locally. For CI / clean self-contained runs, use test.ci.Dockerfile instead. +FROM python:3.10 + +WORKDIR /app + +RUN apt-get update -y && \ + apt-get install -y jq libgl1 libsm6 libxext6 && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --no-cache 'poetry==1.8.5' + +COPY pyproject.toml poetry.lock ./ + +RUN --mount=type=cache,target=/root/.cache \ + poetry config virtualenvs.create false && \ + poetry install --no-interaction --no-ansi --no-root + +RUN python -m pip uninstall -y poetry pip + +CMD ["pytest"] diff --git a/packages/examples/cvat/recording-oracle/poetry.lock b/packages/examples/cvat/recording-oracle/poetry.lock index a6883c0a02..d467345a9f 100644 --- a/packages/examples/cvat/recording-oracle/poetry.lock +++ b/packages/examples/cvat/recording-oracle/poetry.lock @@ -208,63 +208,6 @@ tornado = ["tornado (>=4.3)"] twisted = ["twisted"] zookeeper = ["kazoo"] -[[package]] -name = "argon2-cffi" -version = "23.1.0" -description = "Argon2 for Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "argon2_cffi-23.1.0-py3-none-any.whl", hash = "sha256:c670642b78ba29641818ab2e68bd4e6a78ba53b7eff7b4c3815ae16abf91c7ea"}, - {file = "argon2_cffi-23.1.0.tar.gz", hash = "sha256:879c3e79a2729ce768ebb7d36d4609e3a78a4ca2ec3a9f12286ca057e3d0db08"}, -] - -[package.dependencies] -argon2-cffi-bindings = "*" - -[package.extras] -dev = ["argon2-cffi[tests,typing]", "tox (>4)"] -docs = ["furo", "myst-parser", "sphinx", "sphinx-copybutton", "sphinx-notfound-page"] -tests = ["hypothesis", "pytest"] -typing = ["mypy"] - -[[package]] -name = "argon2-cffi-bindings" -version = "21.2.0" -description = "Low-level CFFI bindings for Argon2" -optional = false -python-versions = ">=3.6" -files = [ - {file = "argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082"}, - {file = "argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f"}, - {file = "argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3e385d1c39c520c08b53d63300c3ecc28622f076f4c2b0e6d7e796e9f6502194"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2c3e3cc67fdb7d82c4718f19b4e7a87123caf8a93fde7e23cf66ac0337d3cb3f"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a22ad9800121b71099d0fb0a65323810a15f2e292f2ba450810a7316e128ee5"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f9f8b450ed0547e3d473fdc8612083fd08dd2120d6ac8f73828df9b7d45bb351"}, - {file = "argon2_cffi_bindings-21.2.0-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:93f9bf70084f97245ba10ee36575f0c3f1e7d7724d67d8e5b08e61787c320ed7"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:3b9ef65804859d335dc6b31582cad2c5166f0c3e7975f324d9ffaa34ee7e6583"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4966ef5848d820776f5f562a7d45fdd70c2f330c961d0d745b784034bd9f48d"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20ef543a89dee4db46a1a6e206cd015360e5a75822f76df533845c3cbaf72670"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ed2937d286e2ad0cc79a7087d3c272832865f779430e0cc2b4f3718d3159b0cb"}, - {file = "argon2_cffi_bindings-21.2.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:5e00316dabdaea0b2dd82d141cc66889ced0cdcbfa599e8b471cf22c620c329a"}, -] - -[package.dependencies] -cffi = ">=1.0.1" - -[package.extras] -dev = ["cogapp", "pre-commit", "pytest", "wheel"] -tests = ["pytest"] - [[package]] name = "async-timeout" version = "5.0.1" @@ -2031,20 +1974,19 @@ socks = ["socksio (==1.*)"] [[package]] name = "human-protocol-sdk" -version = "4.3.0" +version = "7.3.1" description = "A python library to launch escrow contracts to the HUMAN network." optional = false python-versions = "*" files = [ - {file = "human_protocol_sdk-4.3.0-py3-none-any.whl", hash = "sha256:498276ba47157615df7e914374fcb51f788e1892e2c169432deb2055108da525"}, - {file = "human_protocol_sdk-4.3.0.tar.gz", hash = "sha256:a1d172899c79c67d9b4266854252e02ae17f157bf1744710843e376c98c95738"}, + {file = "human_protocol_sdk-7.3.1-py3-none-any.whl", hash = "sha256:b3a410b6526f2742dad76cf866a2f6b35406631395df83e913cffa58bcc6a4db"}, + {file = "human_protocol_sdk-7.3.1.tar.gz", hash = "sha256:2df38610d2d4714373d7d0f7e30f7ca218b19746d849b2aa3a3bd0b3b2dfbdd4"}, ] [package.dependencies] aiohttp = "<4.0.0" cryptography = "*" eth-typing = "*" -minio = "*" pgpy = "*" validators = "*" web3 = "*" @@ -2052,38 +1994,6 @@ web3 = "*" [package.extras] agreement = ["numpy", "pyerf"] -[[package]] -name = "hypothesis" -version = "6.97.3" -description = "A library for property-based testing" -optional = false -python-versions = ">=3.8" -files = [ - {file = "hypothesis-6.97.3-py3-none-any.whl", hash = "sha256:6256d768ec866426bfce6ed78418c6e3e43119a0dbece2e0229a1ae5929ae53d"}, - {file = "hypothesis-6.97.3.tar.gz", hash = "sha256:00216ddadaee17ba73451e262f973970a97d34fd75ec34ef57510147264c34d1"}, -] - -[package.dependencies] -attrs = ">=22.2.0" -exceptiongroup = {version = ">=1.0.0", markers = "python_version < \"3.11\""} -sortedcontainers = ">=2.1.0,<3.0.0" - -[package.extras] -all = ["backports.zoneinfo (>=0.2.1)", "black (>=19.10b0)", "click (>=7.0)", "django (>=3.2)", "dpcontracts (>=0.4)", "lark (>=0.10.1)", "libcst (>=0.3.16)", "numpy (>=1.17.3)", "pandas (>=1.1)", "pytest (>=4.6)", "python-dateutil (>=1.4)", "pytz (>=2014.1)", "redis (>=3.0.0)", "rich (>=9.0.0)", "tzdata (>=2023.4)"] -cli = ["black (>=19.10b0)", "click (>=7.0)", "rich (>=9.0.0)"] -codemods = ["libcst (>=0.3.16)"] -dateutil = ["python-dateutil (>=1.4)"] -django = ["django (>=3.2)"] -dpcontracts = ["dpcontracts (>=0.4)"] -ghostwriter = ["black (>=19.10b0)"] -lark = ["lark (>=0.10.1)"] -numpy = ["numpy (>=1.17.3)"] -pandas = ["pandas (>=1.1)"] -pytest = ["pytest (>=4.6)"] -pytz = ["pytz (>=2014.1)"] -redis = ["redis (>=3.0.0)"] -zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2023.4)"] - [[package]] name = "identify" version = "2.5.33" @@ -2525,24 +2435,6 @@ files = [ {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, ] -[[package]] -name = "minio" -version = "7.2.15" -description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" -optional = false -python-versions = ">=3.9" -files = [ - {file = "minio-7.2.15-py3-none-any.whl", hash = "sha256:c06ef7a43e5d67107067f77b6c07ebdd68733e5aa7eed03076472410ca19d876"}, - {file = "minio-7.2.15.tar.gz", hash = "sha256:5247df5d4dca7bfa4c9b20093acd5ad43e82d8710ceb059d79c6eea970f49f79"}, -] - -[package.dependencies] -argon2-cffi = "*" -certifi = "*" -pycryptodome = "*" -typing-extensions = "*" -urllib3 = "*" - [[package]] name = "multidict" version = "6.1.0" @@ -4043,17 +3935,6 @@ files = [ {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, ] -[[package]] -name = "sortedcontainers" -version = "2.4.0" -description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" -optional = false -python-versions = "*" -files = [ - {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, - {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, -] - [[package]] name = "sqlalchemy" version = "2.0.25" @@ -4755,4 +4636,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10, <3.13" -content-hash = "82157c082fb82718a5e092b999f14d3c18ca27a2acfb198101e5dcae6c94e42a" +content-hash = "7b255d62870b43bd42bf9b686e94e5dbbda9929f4a503d6b5e6e91c8ca2627f4" diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index e332a910e5..b51dab8f05 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -14,7 +14,6 @@ python-dotenv = "^1.0.0" SQLAlchemy = "^2.0.17" psycopg2 = "^2.9.6" APScheduler = "^3.10.1" -pytest = "^7.4.0" alembic = "^1.11.1" pydantic = ">=2.6.1,<2.7.0" httpx = "^0.24.1" @@ -26,12 +25,10 @@ hexbytes = ">=1.2.0" # required for to_0x_hex() function starlette = ">=0.40.0" # avoid the vulnerability with multipart/form-data cvat-sdk = "2.37.0" cryptography = "<44.0.0" # human-protocol-sdk -> pgpy dep requires cryptography < 45 -human-protocol-sdk = "^4.3.0" +human-protocol-sdk = "^7.3.1" [tool.poetry.group.dev.dependencies] -hypothesis = "^6.82.6" - -[tool.poetry.group.lint.dependencies] +pytest = "^7.4.0" pre-commit = "^3.3.3" ruff = "^0.6.0" diff --git a/packages/examples/cvat/recording-oracle/src/chain/escrow.py b/packages/examples/cvat/recording-oracle/src/chain/escrow.py index bd21e2ff18..e0955548a7 100644 --- a/packages/examples/cvat/recording-oracle/src/chain/escrow.py +++ b/packages/examples/cvat/recording-oracle/src/chain/escrow.py @@ -1,15 +1,34 @@ import json +import httpx from human_protocol_sdk.constants import ChainId, Status from human_protocol_sdk.encryption import Encryption, EncryptionUtils from human_protocol_sdk.escrow import EscrowClient, EscrowData, EscrowUtils -from human_protocol_sdk.storage import StorageUtils +from human_protocol_sdk.utils import validate_url from src.chain.web3 import get_web3 from src.core.config import Config from src.core.types import OracleWebhookTypes +class ManifestNotAvailableError(Exception): + """Raised when the escrow manifest cannot be retrieved.""" + + +def _get_manifest_content(manifest: str) -> str: + if validate_url(manifest): + try: + response = httpx.get(manifest, follow_redirects=True) + response.raise_for_status() + except Exception as e: + raise ManifestNotAvailableError( + f"failed to download manifest from {manifest}: {e}" + ) from e + return response.text + + return manifest + + def get_escrow(chain_id: int, escrow_address: str) -> EscrowData: escrow = EscrowUtils.get_escrow(ChainId(chain_id), escrow_address) if not escrow: @@ -46,7 +65,7 @@ def validate_escrow( def get_escrow_manifest(chain_id: int, escrow_address: str) -> dict: escrow = get_escrow(chain_id, escrow_address) - manifest_content = StorageUtils.download_file_from_url(escrow.manifest).decode("utf-8") + manifest_content = _get_manifest_content(escrow.manifest) if EncryptionUtils.is_encrypted(manifest_content): encryption = Encryption( diff --git a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py index fb5f09b418..f380a3874f 100644 --- a/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py +++ b/packages/examples/cvat/recording-oracle/src/core/tasks/skeletons_from_boxes.py @@ -22,7 +22,7 @@ class RoiInfo: # RoI is centered on the bbox center # Coordinates can be out of image boundaries. - # In this case RoI includes extra margins to be centered on bbox center + # In this case RoI includes extra margins to be centered on bbox center. roi_x: int roi_y: int roi_w: int diff --git a/packages/examples/cvat/recording-oracle/tests/conftest.py b/packages/examples/cvat/recording-oracle/tests/conftest.py index 306d4f3368..32972c10f8 100644 --- a/packages/examples/cvat/recording-oracle/tests/conftest.py +++ b/packages/examples/cvat/recording-oracle/tests/conftest.py @@ -2,17 +2,48 @@ import pytest from fastapi.testclient import TestClient +from sqlalchemy import inspect, text from sqlalchemy.orm import Session from src import app from src.db import Base, SessionLocal, engine -@pytest.fixture(autouse=True) -def db(): +@pytest.fixture(scope="session", autouse=True) +def _create_schema() -> None: + """Create the schema once for the whole session; per-test cleanup only clears data (below).""" Base.metadata.drop_all(bind=engine) Base.metadata.create_all(bind=engine) + # The per-test cleanup only DELETEs rows; it does not reset sequences. Guard that assumption so + # a future model adding a serial/identity column fails loudly here rather than silently leaking + # sequence state (e.g. auto-increment ids) across tests. + sequences = inspect(engine).get_sequence_names() + assert not sequences, ( + f"Found DB sequences {sequences}: the per-test cleanup DELETEs rows without resetting " + "sequences, so ids would leak between tests. Reset them per test (e.g. TRUNCATE ... " + "RESTART IDENTITY, or ALTER SEQUENCE) and update the cleanup." + ) + + +# Clear every table between tests. DELETE in reverse dependency order is ~0 on empty tables, versus +# recreating the whole schema (drop_all + create_all, ~140ms) on every test. No autoincrement PKs, +# so there are no sequences to reset (guarded in _create_schema). Each DELETE is a single SQL +# expression. +_CLEANUP_SQL = text( + ";\n".join( + str(table.delete().compile(dialect=engine.dialect)) + for table in reversed(Base.metadata.sorted_tables) + ) +) + + +@pytest.fixture(autouse=True) +def db(_create_schema) -> None: + """Reset the database to an empty schema before each test.""" + with engine.begin() as connection: + connection.execute(_CLEANUP_SQL) + @pytest.fixture(scope="module") def client() -> Generator: diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py index a06702b6ac..742c7addb6 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_escrow.py @@ -3,123 +3,113 @@ from unittest.mock import MagicMock, patch import pytest +from human_protocol_sdk.constants import ChainId, Status from human_protocol_sdk.encryption import EncryptionUtils -from human_protocol_sdk.escrow import EscrowClientError -from web3 import Web3 -from web3.middleware import SignAndSendRawMiddlewareBuilder -from web3.providers.rpc import HTTPProvider +from human_protocol_sdk.escrow import EscrowClientError, EscrowData from src.chain.escrow import ( get_escrow_manifest, store_results, validate_escrow, ) -from src.core.config import Config from tests.utils.constants import ( - DEFAULT_GAS_PAYER_PRIV, DEFAULT_HASH, DEFAULT_MANIFEST_URL, + ESCROW_ADDRESS, + EXCHANGE_ORACLE_ADDRESS, + FACTORY_ADDRESS, + JOB_LAUNCHER_ADDRESS, PGP_PASSPHRASE, PGP_PRIVATE_KEY1, PGP_PUBLIC_KEY1, PGP_PUBLIC_KEY2, + RECORDING_ORACLE_ADDRESS, REPUTATION_ORACLE_ADDRESS, + TOKEN_ADDRESS, ) -from tests.utils.setup_escrow import ( - amount, - bulk_payout, - create_escrow, - fund_escrow, - get_intermediate_results_url, - setup_escrow, -) + +escrow_address = ESCROW_ADDRESS +chain_id = ChainId.LOCALHOST.value class ServiceIntegrationTest(unittest.TestCase): def setUp(self): - self.w3 = Web3(HTTPProvider()) - - # Set default gas payer - self.gas_payer = self.w3.eth.account.from_key(DEFAULT_GAS_PAYER_PRIV) - self.w3.middleware_onion.inject( - SignAndSendRawMiddlewareBuilder.build(DEFAULT_GAS_PAYER_PRIV), - "SignAndSendRawMiddlewareBuilder", - layer=0, + self.escrow_data = EscrowData( + chain_id=ChainId.LOCALHOST.name, + id=1, + address=escrow_address, + amount_paid=100, + balance=100, + count=0, + factory_address=FACTORY_ADDRESS, + launcher=JOB_LAUNCHER_ADDRESS, + job_requester_id=JOB_LAUNCHER_ADDRESS, + status=Status.Pending.name, + token=TOKEN_ADDRESS, + total_funded_amount=1000, + created_at="0", + manifest=DEFAULT_MANIFEST_URL, + recording_oracle=RECORDING_ORACLE_ADDRESS, + exchange_oracle=EXCHANGE_ORACLE_ADDRESS, + reputation_oracle=REPUTATION_ORACLE_ADDRESS, ) - self.w3.eth.default_account = self.gas_payer.address - self.network_config = Config.localhost - self.escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, self.escrow_address) - setup_escrow(self.w3, self.escrow_address) + def test_validate_escrow(self): + with patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function: + mock_function.return_value = self.escrow_data + assert validate_escrow(chain_id, escrow_address) is None - def escrow(self, status: str = "Pending", balance: float = amount): - mock_escrow = MagicMock() - mock_escrow.status = status - mock_escrow.balance = balance - mock_escrow.reputation_oracle = REPUTATION_ORACLE_ADDRESS - mock_escrow.manifest = "http://s3.amazonaws.com" - return mock_escrow + def test_validate_escrow_invalid_address(self): + with pytest.raises(EscrowClientError, match="Invalid escrow address: invalid_address"): + validate_escrow(chain_id, "invalid_address") - def test_validate_escrow(self): - with patch("src.chain.escrow.get_escrow") as mock_get_escrow: - mock_get_escrow.return_value = self.escrow("Pending", amount) - # should not throw an exception - validate_escrow(self.network_config.chain_id, self.escrow_address) + def test_validate_escrow_invalid_status(self): + with patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function: + self.escrow_data.status = Status.Partial.name + mock_function.return_value = self.escrow_data + with pytest.raises( + ValueError, + match=r"Escrow is not in any of the accepted states \(Pending\)", + ): + validate_escrow(chain_id, escrow_address) def test_validate_escrow_without_funds(self): - with patch("src.chain.escrow.get_escrow") as mock_get_escrow: - mock_get_escrow.return_value = self.escrow("Pending", 0) - + with patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function: + self.escrow_data.balance = 0 + mock_function.return_value = self.escrow_data with pytest.raises(ValueError, match="Escrow doesn't have funds"): - validate_escrow(-1, "", allow_no_funds=False) + validate_escrow(chain_id, escrow_address) - # should not throw an exception - validate_escrow(self.network_config.chain_id, self.escrow_address, allow_no_funds=True) - - def test_validate_escrow_invalid_status(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address) - bulk_payout( - self.w3, - escrow_address, - self.gas_payer.address, - Web3.to_wei(50, "milliether"), - ) - with patch("src.chain.escrow.get_escrow") as mock_get_escrow: - mock_get_escrow.return_value = self.escrow("Partial", 0.95) - - with pytest.raises(ValueError, match="Escrow is not in any of the accepted states"): - validate_escrow(self.w3.eth.chain_id, escrow_address) + assert validate_escrow(chain_id, escrow_address, allow_no_funds=True) is None def test_get_escrow_manifest(self): with ( - patch("src.chain.escrow.get_escrow") as mock_get_escrow, - patch("src.chain.escrow.StorageUtils.download_file_from_url") as mock_download, + patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function, + patch("src.chain.escrow._get_manifest_content") as mock_download, ): - mock_download.return_value = json.dumps({"title": "test"}).encode() - - mock_get_escrow.return_value = self.escrow() - manifest = get_escrow_manifest(self.network_config.chain_id, self.escrow_address) + mock_download.return_value = json.dumps({"title": "test"}) + mock_function.return_value = self.escrow_data + manifest = get_escrow_manifest(chain_id, escrow_address) assert isinstance(manifest, dict) assert manifest is not None + def test_get_escrow_manifest_invalid_address(self): + with pytest.raises(EscrowClientError, match="Invalid escrow address: invalid_address"): + get_escrow_manifest(chain_id, "invalid_address") + def test_get_encrypted_escrow_manifest(self): with ( patch("src.chain.escrow.EscrowUtils.get_escrow") as mock_function, - patch("src.chain.escrow.StorageUtils.download_file_from_url") as mock_download, + patch("src.chain.escrow._get_manifest_content") as mock_download, patch("src.core.config.Config.encryption_config.pgp_private_key", PGP_PRIVATE_KEY1), patch("src.core.config.Config.encryption_config.pgp_passphrase", PGP_PASSPHRASE), patch( "src.core.config.Config.encryption_config.pgp_public_key_url", "http:///some-url" ), ): - mock_function.return_value = self.escrow() - original_manifest_content = { - "title": "test", - } + mock_function.return_value = self.escrow_data + original_manifest_content = {"title": "test"} original_manifest = json.dumps(original_manifest_content) encrypted_manifest = EncryptionUtils.encrypt( @@ -127,39 +117,36 @@ def test_get_encrypted_escrow_manifest(self): ) assert encrypted_manifest != original_manifest - mock_download.return_value = encrypted_manifest.encode() - downloaded_manifest_content = get_escrow_manifest( - self.network_config.chain_id, self.escrow_address - ) + mock_download.return_value = encrypted_manifest + downloaded_manifest_content = get_escrow_manifest(chain_id, escrow_address) assert downloaded_manifest_content == original_manifest_content def test_store_results(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address) - with patch("src.chain.escrow.get_web3") as mock_function: - mock_function.return_value = self.w3 - results = store_results( - self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, DEFAULT_HASH + with ( + patch("src.chain.escrow.get_web3"), + patch("src.chain.escrow.EscrowClient") as mock_client_cls, + ): + assert ( + store_results(chain_id, escrow_address, DEFAULT_MANIFEST_URL, DEFAULT_HASH) is None + ) + mock_client_cls.return_value.store_results.assert_called_once_with( + escrow_address, DEFAULT_MANIFEST_URL, DEFAULT_HASH ) - assert results is None - intermediate_results_url = get_intermediate_results_url(self.w3, escrow_address) - assert intermediate_results_url == DEFAULT_MANIFEST_URL def test_store_results_invalid_url(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address) - with patch("src.chain.escrow.get_web3") as mock_function: - mock_function.return_value = self.w3 - with pytest.raises(EscrowClientError, match="Invalid URL: invalid_url"): - store_results(self.w3.eth.chain_id, escrow_address, "invalid_url", DEFAULT_HASH) + w3 = MagicMock() + w3.eth.chain_id = chain_id + with ( + patch("src.chain.escrow.get_web3", return_value=w3), + pytest.raises(EscrowClientError, match="Invalid URL: invalid_url"), + ): + store_results(chain_id, escrow_address, "invalid_url", DEFAULT_HASH) def test_store_results_invalid_hash(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address) - with patch("src.chain.escrow.get_web3") as mock_function: - mock_function.return_value = self.w3 - with pytest.raises(EscrowClientError, match="Invalid empty hash"): - store_results(self.w3.eth.chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") + w3 = MagicMock() + w3.eth.chain_id = chain_id + with ( + patch("src.chain.escrow.get_web3", return_value=w3), + pytest.raises(EscrowClientError, match="Invalid empty hash"), + ): + store_results(chain_id, escrow_address, DEFAULT_MANIFEST_URL, "") diff --git a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py index a8ffa538ff..ee0756ff02 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/chain/test_kvstore.py @@ -39,10 +39,11 @@ def setup(self): count=0, factory_address=FACTORY_ADDRESS, launcher=JOB_LAUNCHER_ADDRESS, + job_requester_id=JOB_LAUNCHER_ADDRESS, status=Status.Pending.name, token=TOKEN_ADDRESS, total_funded_amount=1000, - created_at="", + created_at="0", manifest=DEFAULT_MANIFEST_URL, exchange_oracle=EXCHANGE_ORACLE_ADDRESS, reputation_oracle=REPUTATION_ORACLE_ADDRESS, diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py index 50e2d18705..81dfa6fe95 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_exchange_oracle_webhooks.py @@ -4,11 +4,7 @@ from unittest.mock import MagicMock, call, patch from sqlalchemy.sql import select -from web3 import Web3 -from web3.middleware import SignAndSendRawMiddlewareBuilder -from web3.providers.rpc import HTTPProvider -from src.core.config import Config from src.core.storage import compose_data_bucket_prefix, compose_results_bucket_prefix from src.core.types import ( ExchangeOracleEventTypes, @@ -26,22 +22,12 @@ from src.services.cloud import StorageClient from src.services.webhook import OracleWebhookDirectionTags -from tests.utils.constants import DEFAULT_GAS_PAYER_PRIV, ESCROW_ADDRESS, SIGNATURE -from tests.utils.setup_escrow import create_escrow, fund_escrow, setup_escrow +from tests.utils.constants import ESCROW_ADDRESS, SIGNATURE class ServiceIntegrationTest(unittest.TestCase): def setUp(self): self.session = SessionLocal() - self.w3 = Web3(HTTPProvider(Config.localhost.rpc_api)) - - self.gas_payer = self.w3.eth.account.from_key(DEFAULT_GAS_PAYER_PRIV) - self.w3.middleware_onion.inject( - SignAndSendRawMiddlewareBuilder.build(DEFAULT_GAS_PAYER_PRIV), - "SignAndSendRawMiddlewareBuilder", - layer=0, - ) - self.w3.eth.default_account = self.gas_payer.address def tearDown(self): self.session.close() @@ -60,9 +46,7 @@ def make_webhook(self, escrow_address, event_type=ExchangeOracleEventTypes.job_f ) def test_process_exchange_oracle_webhook(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address) + escrow_address = ESCROW_ADDRESS webhook = self.make_webhook(escrow_address) self.session.add(webhook) @@ -175,7 +159,7 @@ def test_process_recording_oracle_webhooks_invalid_escrow_address(self): assert updated_webhook.attempts == 1 def test_process_recording_oracle_webhooks_invalid_escrow_balance(self): - escrow_address = create_escrow(self.w3) + escrow_address = ESCROW_ADDRESS webhook = self.make_webhook(escrow_address) @@ -195,16 +179,21 @@ def test_process_recording_oracle_webhooks_invalid_escrow_balance(self): assert updated_webhook.attempts == 1 def test_process_job_launcher_webhooks_invalid_manifest_url(self): - escrow_address = create_escrow(self.w3) - fund_escrow(self.w3, escrow_address) - setup_escrow(self.w3, escrow_address, manifest="http://localhost/invalid/url") + escrow_address = ESCROW_ADDRESS webhook = self.make_webhook(escrow_address) self.session.add(webhook) self.session.commit() - process_incoming_exchange_oracle_webhooks() + with ( + patch("src.crons.process_exchange_oracle_webhooks.validate_escrow"), + patch( + "src.crons.process_exchange_oracle_webhooks.validate_results", + side_effect=Exception("failed to download manifest"), + ), + ): + process_incoming_exchange_oracle_webhooks() updated_webhook = ( self.session.execute(select(Webhook).where(Webhook.id == webhook.id)).scalars().first() diff --git a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py index 254016415f..11bdf6cd7e 100644 --- a/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py +++ b/packages/examples/cvat/recording-oracle/tests/integration/cron/test_process_reputation_oracle_webhooks.py @@ -3,9 +3,6 @@ from unittest.mock import MagicMock, patch from sqlalchemy.sql import select -from web3 import Web3 -from web3.middleware import SignAndSendRawMiddlewareBuilder -from web3.providers.rpc import HTTPProvider from src.core.types import ( Networks, @@ -18,23 +15,12 @@ from src.models.webhook import Webhook from src.services.webhook import OracleWebhookDirectionTags -from tests.utils.constants import DEFAULT_GAS_PAYER_PRIV, SIGNATURE -from tests.utils.setup_escrow import create_escrow -from tests.utils.setup_kvstore import store_kvstore_value +from tests.utils.constants import ESCROW_ADDRESS, SIGNATURE class ServiceIntegrationTest(unittest.TestCase): def setUp(self): self.session = SessionLocal() - self.w3 = Web3(HTTPProvider()) - - self.gas_payer = self.w3.eth.account.from_key(DEFAULT_GAS_PAYER_PRIV) - self.w3.middleware_onion.inject( - SignAndSendRawMiddlewareBuilder.build(DEFAULT_GAS_PAYER_PRIV), - "SignAndSendRawMiddlewareBuilder", - layer=0, - ) - self.w3.eth.default_account = self.gas_payer.address def tearDown(self): self.session.close() @@ -67,8 +53,7 @@ def test_process_reputation_oracle_webhooks(self): mock_get_repo_url.return_value = expected_url chain_id = Networks.localhost.value - escrow_address = create_escrow(self.w3) - store_kvstore_value("webhook_url", expected_url) + escrow_address = ESCROW_ADDRESS event_data = {} mock_signature.return_value = (None, SIGNATURE) @@ -122,7 +107,7 @@ def test_process_reputation_oracle_webhooks_invalid_reputation_oracle_url(self): ) as mock_get_repo_url: mock_get_repo_url.return_value = "https://not.a.real/url/existing.somewhere" - webhook = self.get_webhook(create_escrow(self.w3), Networks.localhost.value, {}) + webhook = self.get_webhook(ESCROW_ADDRESS, Networks.localhost.value, {}) self.session.add(webhook) self.session.commit() process_outgoing_reputation_oracle_webhooks() diff --git a/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py b/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py deleted file mode 100644 index f289738487..0000000000 --- a/packages/examples/cvat/recording-oracle/tests/utils/setup_escrow.py +++ /dev/null @@ -1,68 +0,0 @@ -from decimal import Decimal - -from human_protocol_sdk.constants import NETWORKS, ChainId -from human_protocol_sdk.escrow import EscrowClient, EscrowConfig -from human_protocol_sdk.staking import StakingClient -from web3 import Web3 - -from tests.utils.constants import ( - DEFAULT_HASH, - DEFAULT_MANIFEST_URL, - EXCHANGE_ORACLE_ADDRESS, - EXCHANGE_ORACLE_FEE, - JOB_REQUESTER_ID, - RECORDING_ORACLE_ADDRESS, - RECORDING_ORACLE_FEE, - REPUTATION_ORACLE_ADDRESS, - REPUTATION_ORACLE_FEE, -) - -amount = Web3.to_wei(1, "ether") - - -def create_escrow(web3: Web3): - staking_client = StakingClient(web3) - escrow_client = EscrowClient(web3) - staking_client.approve_stake(amount) - staking_client.stake(amount) - return escrow_client.create_escrow( - token_address=NETWORKS[ChainId.LOCALHOST]["hmt_address"], - trusted_handlers=[web3.eth.default_account], - job_requester_id=JOB_REQUESTER_ID, - ) - - -def setup_escrow(web3: Web3, escrow_address: str, *, manifest: str = DEFAULT_MANIFEST_URL): - escrow_client = EscrowClient(web3) - escrow_client.setup( - escrow_address=escrow_address, - escrow_config=EscrowConfig( - exchange_oracle_address=EXCHANGE_ORACLE_ADDRESS, - exchange_oracle_fee=EXCHANGE_ORACLE_FEE, - recording_oracle_address=RECORDING_ORACLE_ADDRESS, - recording_oracle_fee=RECORDING_ORACLE_FEE, - reputation_oracle_address=REPUTATION_ORACLE_ADDRESS, - reputation_oracle_fee=REPUTATION_ORACLE_FEE, - manifest=manifest, - hash=DEFAULT_HASH, - ), - ) - - -def fund_escrow(web3: Web3, escrow_address: str): - escrow_client = EscrowClient(web3) - escrow_client.fund(escrow_address, amount) - - -def bulk_payout(web3: Web3, escrow_address: str, recipient: str, amount: Decimal): - escrow_client = EscrowClient(web3) - escrow_client.bulk_payout( - escrow_address, [recipient], [amount], DEFAULT_MANIFEST_URL, DEFAULT_HASH, 1 - ) - - -def get_intermediate_results_url(web3: Web3, escrow_address: str): - escrow_client = EscrowClient(web3) - return ( - escrow_client._get_escrow_contract(escrow_address).functions.intermediateResultsUrl().call() - ) diff --git a/packages/examples/cvat/recording-oracle/tests/utils/setup_kvstore.py b/packages/examples/cvat/recording-oracle/tests/utils/setup_kvstore.py deleted file mode 100644 index ec9bd88a70..0000000000 --- a/packages/examples/cvat/recording-oracle/tests/utils/setup_kvstore.py +++ /dev/null @@ -1,22 +0,0 @@ -from human_protocol_sdk.kvstore import KVStoreClient -from web3 import HTTPProvider, Web3 -from web3.middleware import SignAndSendRawMiddlewareBuilder - -from tests.utils.constants import REPUTATION_ORACLE_PRIV - -amount = Web3.to_wei(1, "ether") - - -def store_kvstore_value(key: str, value: str): - w3 = Web3(HTTPProvider()) - - # Set default gas payer - reputation_oracle = w3.eth.account.from_key(REPUTATION_ORACLE_PRIV) - w3.middleware_onion.inject( - SignAndSendRawMiddlewareBuilder.build(REPUTATION_ORACLE_PRIV), - "SignAndSendRawMiddlewareBuilder", - layer=0, - ) - w3.eth.default_account = reputation_oracle.address - kvstore_client = KVStoreClient(w3) - kvstore_client.set(key, value)