From c5a507422a0386ea3aaa317c776c7b1e42538917 Mon Sep 17 00:00:00 2001 From: Lino Giger <68745352+LinoGiger@users.noreply.github.com> Date: Tue, 24 Feb 2026 10:28:00 +0100 Subject: [PATCH 1/4] added the new benchmark features --- docs/mri.md | 41 +++++ docs/mri_advanced.md | 31 ++++ .../create_benchmark_participant_result.py | 30 ++-- .../{_participant.py => participant.py} | 49 +++++- .../benchmark/rapidata_benchmark.py | 156 +++++++++++++++++- src/rapidata/types/__init__.py | 4 + 6 files changed, 294 insertions(+), 17 deletions(-) rename src/rapidata/rapidata_client/benchmark/participant/{_participant.py => participant.py} (76%) diff --git a/docs/mri.md b/docs/mri.md index d6edf690c..196b6c68f 100644 --- a/docs/mri.md +++ b/docs/mri.md @@ -87,6 +87,47 @@ benchmark.evaluate_model( ) ``` +### 3b. Adding Models Without Immediate Submission + +If you want to add a model and control when it is submitted for evaluation, use `add_model` instead of `evaluate_model`. This lets you upload media, inspect the participant, and submit on your own schedule. + +```python +# Add a model without submitting +participant = benchmark.add_model( + name="MyAIModel_v2.1", + media=[ + "https://assets.rapidata.ai/mountain_sunset1.png", + "https://assets.rapidata.ai/futuristic_city.png", + "https://assets.rapidata.ai/wizard_portrait.png" + ], + prompts=[ + "A serene mountain landscape at sunset", + "A futuristic city with flying cars", + "A portrait of a wise old wizard" + ] +) + +# Upload additional media to the same participant +participant.upload_media( + assets=["https://assets.rapidata.ai/mountain_sunset2.png"], + identifiers=["A serene mountain landscape at sunset"] +) + +# Submit the individual participant +participant.run() + +# Or submit all unsubmitted participants at once +benchmark.run() +``` + +You can also inspect existing participants: + +```python +# List all participants +for p in benchmark.participants: + print(p.name, p.status) +``` + ### 4. Matchmaking and Ranking MRI creates fair comparisons by: diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index 47e1023f6..ff1b2f839 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -132,8 +132,39 @@ leaderboard = benchmark.create_leaderboard( ) ``` +## Participant Management + +### Listing Participants + +You can list all participants in a benchmark using the `participants` property: + +```python +for participant in benchmark.participants: + print(f"{participant.name} - {participant.status}") +``` + +### Submitting Participants + +When using `add_model`, participants are created in the `CREATED` state and are not yet submitted for evaluation. You can submit them individually or in bulk: + +```python +# Submit a single participant +participant = benchmark.add_model( + name="ModelA", + media=["https://example.com/img1.png"], + identifiers=["scene_1"] +) +participant.run() + +# Or add multiple models and submit them all at once +benchmark.add_model(name="ModelB", media=["https://example.com/img2.png"], identifiers=["scene_1"]) +benchmark.add_model(name="ModelC", media=["https://example.com/img3.png"], identifiers=["scene_1"]) +benchmark.run() # Submits all participants in CREATED state +``` + ## References - [RapidataBenchmarkManager](/reference/rapidata/rapidata_client/benchmark/rapidata_benchmark_manager/) - [RapidataBenchmark](/reference/rapidata/rapidata_client/benchmark/rapidata_benchmark/) - [RapidataLeaderboard](/reference/rapidata/rapidata_client/benchmark/leaderboard/rapidata_leaderboard/) +- [BenchmarkParticipant](/reference/rapidata/rapidata_client/benchmark/participant/participant/) diff --git a/src/rapidata/api_client/models/create_benchmark_participant_result.py b/src/rapidata/api_client/models/create_benchmark_participant_result.py index 456ae5bb3..f11793b0d 100644 --- a/src/rapidata/api_client/models/create_benchmark_participant_result.py +++ b/src/rapidata/api_client/models/create_benchmark_participant_result.py @@ -1,14 +1,14 @@ # coding: utf-8 """ - Rapidata Asset API +Rapidata Asset API - The API for the Rapidata Asset service +The API for the Rapidata Asset service - The version of the OpenAPI document: v1 - Generated by OpenAPI Generator (https://openapi-generator.tech) +The version of the OpenAPI document: v1 +Generated by OpenAPI Generator (https://openapi-generator.tech) - Do not edit the class manually. +Do not edit the class manually. """ # noqa: E501 @@ -22,10 +22,12 @@ from typing import Optional, Set from typing_extensions import Self + class CreateBenchmarkParticipantResult(BaseModel): """ CreateBenchmarkParticipantResult - """ # noqa: E501 + """ # noqa: E501 + participant_id: StrictStr = Field(alias="participantId") dataset_id: Optional[StrictStr] = Field(default=None, alias="datasetId") __properties: ClassVar[List[str]] = ["participantId", "datasetId"] @@ -36,7 +38,6 @@ class CreateBenchmarkParticipantResult(BaseModel): protected_namespaces=(), ) - def to_str(self) -> str: """Returns the string representation of the model using alias""" return pprint.pformat(self.model_dump(by_alias=True)) @@ -61,8 +62,7 @@ def to_dict(self) -> Dict[str, Any]: were set at model initialization. Other fields with value `None` are ignored. """ - excluded_fields: Set[str] = set([ - ]) + excluded_fields: Set[str] = set([]) _dict = self.model_dump( by_alias=True, @@ -80,10 +80,10 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: if not isinstance(obj, dict): return cls.model_validate(obj) - _obj = cls.model_validate({ - "participantId": obj.get("participantId"), - "datasetId": obj.get("datasetId") - }) + _obj = cls.model_validate( + { + "participantId": obj.get("participantId"), + "datasetId": obj.get("datasetId"), + } + ) return _obj - - diff --git a/src/rapidata/rapidata_client/benchmark/participant/_participant.py b/src/rapidata/rapidata_client/benchmark/participant/participant.py similarity index 76% rename from src/rapidata/rapidata_client/benchmark/participant/_participant.py rename to src/rapidata/rapidata_client/benchmark/participant/participant.py index 999aff00a..b6af67376 100644 --- a/src/rapidata/rapidata_client/benchmark/participant/_participant.py +++ b/src/rapidata/rapidata_client/benchmark/participant/participant.py @@ -2,6 +2,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import time +from typing import TYPE_CHECKING from tqdm.auto import tqdm from rapidata.rapidata_client.config import logger @@ -16,13 +17,59 @@ from rapidata.service.openapi_service import OpenAPIService +if TYPE_CHECKING: + from rapidata.api_client.models.participant_status import ParticipantStatus + class BenchmarkParticipant: - def __init__(self, name: str, id: str, openapi_service: OpenAPIService): + """A participant (model) in a benchmark evaluation. + + Represents a model that has been added to a benchmark for evaluation. + Provides methods to upload media and submit the participant for evaluation. + + Args: + name: The name of the participant/model. + id: The unique identifier of the participant. + openapi_service: The OpenAPI service for API communication. + status: The current status of the participant. + """ + + def __init__( + self, + name: str, + id: str, + openapi_service: OpenAPIService, + status: ParticipantStatus = ParticipantStatus.CREATED, + ): self.name = name self.id = id self._openapi_service = openapi_service self._asset_uploader = AssetUploader(openapi_service) + self._status = status + + @property + def status(self) -> ParticipantStatus | None: + """The current status of the participant.""" + return self._status + + def run(self) -> None: + """Submits the participant for evaluation. + + After uploading media, call this method to submit the participant + so that it enters the evaluation pipeline. + """ + from rapidata.api_client.models.participant_status import ParticipantStatus + + self._openapi_service.participant_api.participants_participant_id_submit_post( + participant_id=self.id + ) + self._status = ParticipantStatus.SUBMITTED + + def __str__(self) -> str: + return f"BenchmarkParticipant(name={self.name}, id={self.id}, status={self._status})" + + def __repr__(self) -> str: + return self.__str__() def _process_single_sample_upload( self, diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 2d6993189..9195beb8c 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -11,6 +11,9 @@ from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import ( RapidataLeaderboard, ) + from rapidata.rapidata_client.benchmark.participant.participant import ( + BenchmarkParticipant, + ) from rapidata.rapidata_client.filter import RapidataFilter from rapidata.rapidata_client.settings import RapidataSetting from rapidata.service.openapi_service import OpenAPIService @@ -39,6 +42,7 @@ def __init__(self, name: str, id: str, openapi_service: OpenAPIService): self.__leaderboards: list["RapidataLeaderboard"] = [] self.__identifiers: list[str] = [] self.__tags: list[list[str]] = [] + self.__participants: list[BenchmarkParticipant] = [] self.__benchmark_page: str = ( f"https://app.{self._openapi_service.environment}/mri/benchmarks/{self.id}" ) @@ -184,6 +188,31 @@ def leaderboards(self) -> list[RapidataLeaderboard]: return self.__leaderboards + @property + def participants(self) -> list[BenchmarkParticipant]: + """Returns the participants that are registered for the benchmark.""" + from rapidata.rapidata_client.benchmark.participant.participant import ( + BenchmarkParticipant, + ) + + with tracer.start_as_current_span("RapidataBenchmark.participants"): + if not self.__participants: + result = self._openapi_service.benchmark_api.benchmark_benchmark_id_participants_get( + benchmark_id=self.id, + ) + + self.__participants = [ + BenchmarkParticipant( + name=p.name, + id=p.id, + openapi_service=self._openapi_service, + status=p.status, + ) + for p in result.items + ] + + return self.__participants + def add_prompt( self, identifier: str | None = None, @@ -419,7 +448,7 @@ def evaluate_model( from rapidata.api_client.models.create_benchmark_participant_model import ( CreateBenchmarkParticipantModel, ) - from rapidata.rapidata_client.benchmark.participant._participant import ( + from rapidata.rapidata_client.benchmark.participant.participant import ( BenchmarkParticipant, ) @@ -497,6 +526,131 @@ def evaluate_model( participant_id=participant_result.participant_id ) + def add_model( + self, + name: str, + media: list[str], + identifiers: list[str] | None = None, + prompts: list[str] | None = None, + ) -> BenchmarkParticipant: + """Adds a model to the benchmark without immediately submitting it for evaluation. + + This method creates a participant, uploads media, but does NOT submit the participant. + Use `participant.run()` or `benchmark.run()` to submit afterwards. + + Args: + name: The name of the model. + media: The generated images/videos that will be used to evaluate the model. + identifiers: The identifiers that correspond to the media. The order of the identifiers must match the order of the media.\n + The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property. + prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media. + + Returns: + The created BenchmarkParticipant instance. + """ + from rapidata.api_client.models.create_benchmark_participant_model import ( + CreateBenchmarkParticipantModel, + ) + from rapidata.api_client.models.participant_status import ParticipantStatus + from rapidata.rapidata_client.benchmark.participant.participant import ( + BenchmarkParticipant, + ) + + with tracer.start_as_current_span("add_model"): + if not media: + raise ValueError("Media must be a non-empty list of strings") + + if not identifiers and not prompts: + raise ValueError("Identifiers or prompts must be provided.") + + if identifiers and prompts: + raise ValueError( + "Identifiers and prompts cannot be provided at the same time. Use one or the other." + ) + + if not identifiers: + assert prompts is not None + identifiers = prompts + + if len(media) != len(identifiers): + raise ValueError( + "Media and identifiers/prompts must have the same length" + ) + + if not all(identifier in self.identifiers for identifier in identifiers): + raise ValueError( + "All identifiers/prompts must be in the registered identifiers/prompts list. To see the registered identifiers/prompts, use the identifiers/prompts property." + ) + + participant_result = self._openapi_service.benchmark_api.benchmark_benchmark_id_participants_post( + benchmark_id=self.id, + create_benchmark_participant_model=CreateBenchmarkParticipantModel( + name=name, + ), + ) + + logger.info(f"Participant created: {participant_result.participant_id}") + + participant = BenchmarkParticipant( + name, + participant_result.participant_id, + self._openapi_service, + ) + + with tracer.start_as_current_span("upload_media_for_participant"): + logger.info( + f"Uploading {len(media)} media assets to participant {participant.id}" + ) + + successful_uploads, failed_uploads = participant.upload_media( + media, + identifiers, + ) + + total_uploads = len(media) + success_rate = ( + (len(successful_uploads) / total_uploads * 100) + if total_uploads > 0 + else 0 + ) + logger.info( + f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)" + ) + + if failed_uploads: + logger.error(f"Failed uploads for media: {failed_uploads}") + logger.warning( + "Some uploads failed. The model evaluation may be incomplete." + ) + + if len(successful_uploads) == 0: + raise RuntimeError( + "No uploads were successful. The model evaluation will not be completed." + ) + + # Clear cache so next access re-fetches + self.__participants = [] + + return participant + + def run(self) -> None: + """Submits all participants that are in `CREATED` state. + + This is a convenience method to submit all unsubmitted participants at once. + """ + from rapidata.api_client.models.participant_status import ParticipantStatus + + with tracer.start_as_current_span("RapidataBenchmark.run"): + created = [ + p for p in self.participants if p.status == ParticipantStatus.CREATED + ] + logger.info(f"Submitting {len(created)} participants in CREATED state") + for participant in created: + participant.run() + + # Clear cache so next access re-fetches + self.__participants = [] + def view(self) -> None: """ Views the benchmark. diff --git a/src/rapidata/types/__init__.py b/src/rapidata/types/__init__.py index 481e03455..acf18b6d3 100644 --- a/src/rapidata/types/__init__.py +++ b/src/rapidata/types/__init__.py @@ -15,6 +15,9 @@ from rapidata.rapidata_client.benchmark.leaderboard.rapidata_leaderboard import ( RapidataLeaderboard, ) +from rapidata.rapidata_client.benchmark.participant.participant import ( + BenchmarkParticipant, +) # Selection Types from rapidata.rapidata_client.selection.ab_test_selection import AbTestSelection @@ -97,6 +100,7 @@ "RapidataBenchmark", "RapidataBenchmarkManager", "RapidataLeaderboard", + "BenchmarkParticipant", # Selection Types "AbTestSelection", "CappedSelection", From dd1acf7d33d0419d26aeb87c7bbca94b48c08bed Mon Sep 17 00:00:00 2001 From: Lino Giger <68745352+LinoGiger@users.noreply.github.com> Date: Tue, 24 Feb 2026 10:33:51 +0100 Subject: [PATCH 2/4] fixed bug --- .../rapidata_client/benchmark/participant/participant.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/rapidata/rapidata_client/benchmark/participant/participant.py b/src/rapidata/rapidata_client/benchmark/participant/participant.py index b6af67376..33e1209b6 100644 --- a/src/rapidata/rapidata_client/benchmark/participant/participant.py +++ b/src/rapidata/rapidata_client/benchmark/participant/participant.py @@ -2,7 +2,6 @@ from concurrent.futures import ThreadPoolExecutor, as_completed import time -from typing import TYPE_CHECKING from tqdm.auto import tqdm from rapidata.rapidata_client.config import logger @@ -16,9 +15,7 @@ from rapidata.service.openapi_service import OpenAPIService - -if TYPE_CHECKING: - from rapidata.api_client.models.participant_status import ParticipantStatus +from rapidata.api_client.models.participant_status import ParticipantStatus class BenchmarkParticipant: @@ -58,8 +55,6 @@ def run(self) -> None: After uploading media, call this method to submit the participant so that it enters the evaluation pipeline. """ - from rapidata.api_client.models.participant_status import ParticipantStatus - self._openapi_service.participant_api.participants_participant_id_submit_post( participant_id=self.id ) From 17fa29097c174996f3fd07ef74aa47540db224c9 Mon Sep 17 00:00:00 2001 From: Lino Giger <68745352+LinoGiger@users.noreply.github.com> Date: Tue, 24 Feb 2026 10:47:31 +0100 Subject: [PATCH 3/4] small fixes --- docs/mri_advanced.md | 8 +- .../benchmark/participant/participant.py | 2 +- .../benchmark/rapidata_benchmark.py | 84 ++----------------- 3 files changed, 11 insertions(+), 83 deletions(-) diff --git a/docs/mri_advanced.md b/docs/mri_advanced.md index ff1b2f839..995a3a187 100644 --- a/docs/mri_advanced.md +++ b/docs/mri_advanced.md @@ -12,7 +12,7 @@ In the MRI quickstart we used the prompts to identify the media and create the a ```python # Example 1: Explicit identifiers -benchmark = benchmark_manager.create_new_benchmark( +benchmark = client.mri.create_new_benchmark( name="Preference Benchmark", identifiers=["scene_1", "scene_2", "scene_3"], prompts=[ @@ -28,7 +28,7 @@ benchmark = benchmark_manager.create_new_benchmark( ) # Example 2: Identifiers used for the same prompts but different seeding -benchmark = benchmark_manager.create_new_benchmark( +benchmark = client.mri.create_new_benchmark( name="Preference Benchmark", identifiers=["seed_1", "seed_2", "seed_3"], prompts=["prompt_1", "prompt_1", "prompt_1"], @@ -36,7 +36,7 @@ benchmark = benchmark_manager.create_new_benchmark( ) # Example 3: Using only prompt assets -benchmark = benchmark_manager.create_new_benchmark( +benchmark = client.mri.create_new_benchmark( name="Preference Benchmark", identifiers=["image_1", "image_2", "image_3"], prompt_assets=["https://example.com/asset1.jpg", "https://example.com/asset2.jpg", "https://example.com/asset3.jpg"] @@ -58,7 +58,7 @@ tags = [ ["indoor", "vehicle"] ] -benchmark = benchmark_manager.create_new_benchmark( +benchmark = client.mri.create_new_benchmark( name="Tagged Benchmark", identifiers=["scene_1", "scene_2", "scene_3", "scene_4"], prompts=["A sunny beach", "A mountain landscape", "A city skyline", "A car in a garage"], diff --git a/src/rapidata/rapidata_client/benchmark/participant/participant.py b/src/rapidata/rapidata_client/benchmark/participant/participant.py index 33e1209b6..c74e4ace3 100644 --- a/src/rapidata/rapidata_client/benchmark/participant/participant.py +++ b/src/rapidata/rapidata_client/benchmark/participant/participant.py @@ -45,7 +45,7 @@ def __init__( self._status = status @property - def status(self) -> ParticipantStatus | None: + def status(self) -> ParticipantStatus: """The current status of the participant.""" return self._status diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 9195beb8c..053fbe022 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -445,86 +445,14 @@ def evaluate_model( The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property. prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media. """ - from rapidata.api_client.models.create_benchmark_participant_model import ( - CreateBenchmarkParticipantModel, - ) - from rapidata.rapidata_client.benchmark.participant.participant import ( - BenchmarkParticipant, - ) - with tracer.start_as_current_span("evaluate_model"): - if not media: - raise ValueError("Media must be a non-empty list of strings") - - if not identifiers and not prompts: - raise ValueError("Identifiers or prompts must be provided.") - - if identifiers and prompts: - raise ValueError( - "Identifiers and prompts cannot be provided at the same time. Use one or the other." - ) - - if not identifiers: - assert prompts is not None - identifiers = prompts - - if len(media) != len(identifiers): - raise ValueError( - "Media and identifiers/prompts must have the same length" - ) - - if not all(identifier in self.identifiers for identifier in identifiers): - raise ValueError( - "All identifiers/prompts must be in the registered identifiers/prompts list. To see the registered identifiers/prompts, use the identifiers/prompts property." - ) - - participant_result = self._openapi_service.benchmark_api.benchmark_benchmark_id_participants_post( - benchmark_id=self.id, - create_benchmark_participant_model=CreateBenchmarkParticipantModel( - name=name, - ), + participant = self.add_model( + name=name, + media=media, + identifiers=identifiers, + prompts=prompts, ) - - logger.info(f"Participant created: {participant_result.participant_id}") - - participant = BenchmarkParticipant( - name, participant_result.participant_id, self._openapi_service - ) - - with tracer.start_as_current_span("upload_media_for_participant"): - logger.info( - f"Uploading {len(media)} media assets to participant {participant.id}" - ) - - successful_uploads, failed_uploads = participant.upload_media( - media, - identifiers, - ) - - total_uploads = len(media) - success_rate = ( - (len(successful_uploads) / total_uploads * 100) - if total_uploads > 0 - else 0 - ) - logger.info( - f"Upload complete: {len(successful_uploads)} successful, {len(failed_uploads)} failed ({success_rate:.1f}% success rate)" - ) - - if failed_uploads: - logger.error(f"Failed uploads for media: {failed_uploads}") - logger.warning( - "Some uploads failed. The model evaluation may be incomplete." - ) - - if len(successful_uploads) == 0: - raise RuntimeError( - "No uploads were successful. The model evaluation will not be completed." - ) - - self._openapi_service.participant_api.participants_participant_id_submit_post( - participant_id=participant_result.participant_id - ) + participant.run() def add_model( self, From 066b30decfd4dcbdb758807c8024a118976fc2cf Mon Sep 17 00:00:00 2001 From: Lino Giger <68745352+LinoGiger@users.noreply.github.com> Date: Tue, 24 Feb 2026 10:53:31 +0100 Subject: [PATCH 4/4] minor removal of import --- .../rapidata_client/benchmark/rapidata_benchmark.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py index 053fbe022..e218b86a8 100644 --- a/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py +++ b/src/rapidata/rapidata_client/benchmark/rapidata_benchmark.py @@ -236,7 +236,7 @@ def add_prompt( from rapidata.api_client.models.i_asset_input import IAssetInput - with tracer.start_as_current_span("RapidataBenchmark.add_prompt"): + with tracer.start_as_current_span("RapidataBenchmark.add_prompt_to_benchmark"): if tags is None: tags = [] @@ -350,7 +350,7 @@ def create_leaderboard( RapidataLeaderboard, ) - with tracer.start_as_current_span("create_leaderboard"): + with tracer.start_as_current_span("RapidataBenchmark.create_leaderboard"): if level_of_detail is not None and ( not isinstance(level_of_detail, str) or level_of_detail not in ["low", "medium", "high", "very high"] @@ -445,7 +445,7 @@ def evaluate_model( The identifiers that are used must be registered for the benchmark. To see the registered identifiers, use the identifiers property. prompts: The prompts that correspond to the media. The order of the prompts must match the order of the media. """ - with tracer.start_as_current_span("evaluate_model"): + with tracer.start_as_current_span("RapidataBenchmark.evaluate_model"): participant = self.add_model( name=name, media=media, @@ -479,12 +479,11 @@ def add_model( from rapidata.api_client.models.create_benchmark_participant_model import ( CreateBenchmarkParticipantModel, ) - from rapidata.api_client.models.participant_status import ParticipantStatus from rapidata.rapidata_client.benchmark.participant.participant import ( BenchmarkParticipant, ) - with tracer.start_as_current_span("add_model"): + with tracer.start_as_current_span("RapidataBenchmark.add_model"): if not media: raise ValueError("Media must be a non-empty list of strings")