diff --git a/packages/examples/cvat/exchange-oracle/poetry.lock b/packages/examples/cvat/exchange-oracle/poetry.lock index f1970962d5..8af6c9bc2d 100644 --- a/packages/examples/cvat/exchange-oracle/poetry.lock +++ b/packages/examples/cvat/exchange-oracle/poetry.lock @@ -945,13 +945,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "cvat-sdk" -version = "2.31.0" +version = "2.37.0" description = "CVAT REST API" optional = false python-versions = ">=3.9" files = [ - {file = "cvat_sdk-2.31.0-py3-none-any.whl", hash = "sha256:b33e8526dad8c481f82e445badfced5d69747eaf7e5660b0d176cf86d394a02e"}, - {file = "cvat_sdk-2.31.0.tar.gz", hash = "sha256:aaeff833c32bfe711f418c62bdab135e0746eff0e89757e8b61cfad14a42ef23"}, + {file = "cvat_sdk-2.37.0-py3-none-any.whl", hash = "sha256:faa94cfd6678089814179a8da828761dfa3daf08eb752490ee85551a1045dac5"}, + {file = "cvat_sdk-2.37.0.tar.gz", hash = "sha256:e990908a473c499eb6d7b84f7f2e640ea729ef027d4c4cc32a5a925752532689"}, ] [package.dependencies] @@ -5047,4 +5047,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10,<3.13" -content-hash = "c643f28ae7113ae0b8051952c0adfcb74a0ae182ba5039133f57f52b78608007" +content-hash = "8bf7f09b99af5cd8b02a36fc0a1b5ad4af28d5d17d7c0275afa38edb281c3cfc" diff --git a/packages/examples/cvat/exchange-oracle/pyproject.toml b/packages/examples/cvat/exchange-oracle/pyproject.toml index 187d9c81e3..5dd04c1813 100644 --- a/packages/examples/cvat/exchange-oracle/pyproject.toml +++ b/packages/examples/cvat/exchange-oracle/pyproject.toml @@ -16,7 +16,7 @@ sqlalchemy-utils = "^0.41.1" alembic = "^1.11.1" httpx = "^0.24.1" pytest = "^7.2.2" -cvat-sdk = "2.31.0" +cvat-sdk = "2.37.0" sqlalchemy = "^2.0.16" apscheduler = "^3.10.1" xmltodict = "^0.13.0" diff --git a/packages/examples/cvat/exchange-oracle/src/.env.template b/packages/examples/cvat/exchange-oracle/src/.env.template index b07c93515c..88a546cff2 100644 --- a/packages/examples/cvat/exchange-oracle/src/.env.template +++ b/packages/examples/cvat/exchange-oracle/src/.env.template @@ -77,6 +77,8 @@ CVAT_IOU_THRESHOLD= CVAT_OKS_SIGMA= CVAT_EXPORT_TIMEOUT= CVAT_IMPORT_TIMEOUT= +CVAT_PROJECTS_PAGE_SIZE= +CVAT_JOBS_PAGE_SIZE= # Storage Config (S3/GCS) diff --git a/packages/examples/cvat/exchange-oracle/src/core/config.py b/packages/examples/cvat/exchange-oracle/src/core/config.py index 9e40bb1265..245380cc58 100644 --- a/packages/examples/cvat/exchange-oracle/src/core/config.py +++ b/packages/examples/cvat/exchange-oracle/src/core/config.py @@ -146,7 +146,7 @@ class CronConfig: "Maximum number of downloading attempts per job or project during results downloading" track_completed_escrows_jobs_downloading_batch_size = int( - getenv("TRACK_COMPLETED_ESCROWS_JOBS_DOWNLOADING_BATCH_SIZE", 500) + getenv("TRACK_COMPLETED_ESCROWS_JOBS_DOWNLOADING_BATCH_SIZE", 10) ) "Maximum number of parallel downloading requests during results downloading" @@ -183,6 +183,9 @@ class CvatConfig: incoming_webhooks_url = getenv("CVAT_INCOMING_WEBHOOKS_URL") webhook_secret = getenv("CVAT_WEBHOOK_SECRET", "thisisasamplesecret") + projects_page_size = int(getenv("CVAT_PROJECTS_PAGE_SIZE", 100)) + jobs_page_size = int(getenv("CVAT_JOBS_PAGE_SIZE", 100)) + class StorageConfig: provider: ClassVar[str] = os.environ["STORAGE_PROVIDER"].lower() diff --git a/packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py b/packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py index 27c869c863..06934225b4 100644 --- a/packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py +++ b/packages/examples/cvat/exchange-oracle/src/cvat/api_calls.py @@ -47,15 +47,23 @@ def _request_annotations(endpoint: Endpoint, cvat_id: int, format_name: str) -> _get_annotations(request_id, ...) """ - (_, response) = endpoint.call_with_http_info( - id=cvat_id, - format=format_name, - save_images=False, - _parse_response=False, - ) + try: + (_, response) = endpoint.call_with_http_info( + id=cvat_id, + format=format_name, + save_images=False, + _parse_response=False, + ) + + assert response.status in [HTTPStatus.ACCEPTED, HTTPStatus.CREATED] + rq_id = response.json()["rq_id"] + except exceptions.ApiException as e: + if e.status == HTTPStatus.CONFLICT: + rq_id = json.loads(e.body)["rq_id"] + else: + raise - assert response.status in [HTTPStatus.ACCEPTED, HTTPStatus.CREATED] - return response.json()["rq_id"] + return rq_id def _get_annotations( @@ -462,6 +470,7 @@ def fetch_task_jobs(task_id: int) -> list[models.JobRead]: api_client.jobs_api.list_endpoint, task_id=task_id, type="annotation", + page_size=Config.cvat_config.jobs_page_size, ) except exceptions.ApiException as e: logger.exception(f"Exception when calling JobsApi.list: {e}\n") @@ -535,6 +544,7 @@ def fetch_projects(assignee: str = "") -> list[models.ProjectRead]: return get_paginated_collection( api_client.projects_api.list_endpoint, **({"assignee": assignee} if assignee else {}), + page_size=Config.cvat_config.projects_page_size, ) except exceptions.ApiException as e: logger.exception(f"Exception when calling ProjectsApi.list(): {e}\n") @@ -711,6 +721,7 @@ def update_quality_control_settings( logger = logging.getLogger("app") params = { + "inherit": False, "max_validations_per_job": max_validations_per_job, "target_metric": target_metric, "target_metric_threshold": target_metric_threshold, diff --git a/packages/examples/cvat/recording-oracle/poetry.lock b/packages/examples/cvat/recording-oracle/poetry.lock index 4b99cada57..52dd95468a 100644 --- a/packages/examples/cvat/recording-oracle/poetry.lock +++ b/packages/examples/cvat/recording-oracle/poetry.lock @@ -914,13 +914,13 @@ test-randomorder = ["pytest-randomly"] [[package]] name = "cvat-sdk" -version = "2.31.0" +version = "2.37.0" description = "CVAT REST API" optional = false python-versions = ">=3.9" files = [ - {file = "cvat_sdk-2.31.0-py3-none-any.whl", hash = "sha256:b33e8526dad8c481f82e445badfced5d69747eaf7e5660b0d176cf86d394a02e"}, - {file = "cvat_sdk-2.31.0.tar.gz", hash = "sha256:aaeff833c32bfe711f418c62bdab135e0746eff0e89757e8b61cfad14a42ef23"}, + {file = "cvat_sdk-2.37.0-py3-none-any.whl", hash = "sha256:faa94cfd6678089814179a8da828761dfa3daf08eb752490ee85551a1045dac5"}, + {file = "cvat_sdk-2.37.0.tar.gz", hash = "sha256:e990908a473c499eb6d7b84f7f2e640ea729ef027d4c4cc32a5a925752532689"}, ] [package.dependencies] @@ -4732,4 +4732,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.0" python-versions = "^3.10, <3.13" -content-hash = "5f830a339a6f870a60e94be16dc742280e0ec9002fb4a51404fca9e18a6f399f" +content-hash = "3f4ce0cc7668a0c9ffaa02c1306404603d41215e39199c212dabef36ab112a7c" diff --git a/packages/examples/cvat/recording-oracle/pyproject.toml b/packages/examples/cvat/recording-oracle/pyproject.toml index fa03eb2769..194543a267 100644 --- a/packages/examples/cvat/recording-oracle/pyproject.toml +++ b/packages/examples/cvat/recording-oracle/pyproject.toml @@ -24,7 +24,7 @@ google-cloud-storage = "^2.14.0" datumaro = {git = "https://github.com/cvat-ai/datumaro.git", rev = "ff83c00c2c1bc4b8fdfcc55067fcab0a9b5b6b11"} hexbytes = ">=1.2.0" # required for to_0x_hex() function starlette = ">=0.40.0" # avoid the vulnerability with multipart/form-data -cvat-sdk = "2.31.0" +cvat-sdk = "2.37.0" cryptography = "<44.0.0" # human-protocol-sdk -> pgpy dep requires cryptography < 45 human-protocol-sdk = "^4.0.3" diff --git a/packages/examples/cvat/recording-oracle/src/.env.template b/packages/examples/cvat/recording-oracle/src/.env.template index 6abb5d245f..a5851b0fb6 100644 --- a/packages/examples/cvat/recording-oracle/src/.env.template +++ b/packages/examples/cvat/recording-oracle/src/.env.template @@ -61,6 +61,7 @@ CVAT_ADMIN_PASS= CVAT_ORG_SLUG= CVAT_QUALITY_RETRIEVAL_TIMEOUT= CVAT_QUALITY_CHECK_INTERVAL= +CVAT_QUALITY_REPORTS_PAGE_SIZE= # Localhost diff --git a/packages/examples/cvat/recording-oracle/src/core/config.py b/packages/examples/cvat/recording-oracle/src/core/config.py index fa262b8b0b..248f51cc6e 100644 --- a/packages/examples/cvat/recording-oracle/src/core/config.py +++ b/packages/examples/cvat/recording-oracle/src/core/config.py @@ -195,6 +195,7 @@ class ValidationConfig: warmup_iterations = int(getenv("WARMUP_ITERATIONS", "1")) """ The first escrow iterations where the annotation speed is checked to be big enough. + Set to 0 to disable. """ min_warmup_progress = float(getenv("MIN_WARMUP_PROGRESS", "10")) @@ -234,6 +235,7 @@ class CvatConfig: quality_retrieval_timeout = int(getenv("CVAT_QUALITY_RETRIEVAL_TIMEOUT", 60 * 60)) quality_check_interval = int(getenv("CVAT_QUALITY_CHECK_INTERVAL", 5)) + quality_reports_page_size = int(getenv("CVAT_QUALITY_REPORTS_PAGE_SIZE", 100)) class Config: diff --git a/packages/examples/cvat/recording-oracle/src/cvat/api_calls.py b/packages/examples/cvat/recording-oracle/src/cvat/api_calls.py index 0b51e7986c..574e44c918 100644 --- a/packages/examples/cvat/recording-oracle/src/cvat/api_calls.py +++ b/packages/examples/cvat/recording-oracle/src/cvat/api_calls.py @@ -134,7 +134,10 @@ def get_jobs_quality_reports(parent_id: int) -> list[models.QualityReport]: with get_api_client() as api_client: try: return get_paginated_collection( - api_client.quality_api.list_reports_endpoint, parent_id=parent_id, target="job" + api_client.quality_api.list_reports_endpoint, + parent_id=parent_id, + target="job", + page_size=Config.cvat_config.quality_reports_page_size, ) except exceptions.ApiException as e: