From 9bc1b5ad599cc8f056e7927b92b9fbd841616253 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 30 Apr 2025 16:58:34 -0700 Subject: [PATCH 1/5] fix more fields in declarative_component_schema --- .../declarative_component_schema.yaml | 62 ++++++++++++++++--- 1 file changed, 53 insertions(+), 9 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 9d2b6d6b6..bdc33d992 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1937,8 +1937,8 @@ definitions: - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" url: - title: The URL of an API endpoint - description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + title: API Endpoint URL + description: The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. type: string interpolation_context: - config @@ -2003,6 +2003,46 @@ definitions: title: Fetch Properties from Endpoint description: Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields. "$ref": "#/definitions/PropertiesFromEndpoint" + request_parameters: + title: Query Parameters + description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. + anyOf: + - type: object + title: Key/Value Pairs + additionalProperties: + anyOf: + - type: string + - $ref: "#/definitions/QueryProperties" + - type: string + title: Interpolated Value + interpolation_context: + - next_page_token + - stream_interval + - stream_partition + - stream_slice + examples: + - unit: "day" + - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + - searchIn: "{{ ','.join(config.get('search_in', [])) }}" + - sort_by[asc]: updated_at + request_headers: + title: Request Headers + description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. + anyOf: + - type: object + title: Key/Value Pairs + additionalProperties: + type: string + - type: string + title: Interpolated Value + interpolation_context: + - next_page_token + - stream_interval + - stream_partition + - stream_slice + examples: + - Output-Format: JSON + - Version: "{{ config['version'] }}" request_body_data: deprecated: true deprecation_message: "Use `request_body` field instead." @@ -2045,7 +2085,7 @@ definitions: field: "updated_at" order: "ascending" request_body: - title: Request Body Payload to be send as a part of the API request. + title: Request Body description: Specifies how to populate the body of the request with a payload. Can contain nested objects. anyOf: - "$ref": "#/definitions/RequestBodyPlainText" @@ -2076,6 +2116,7 @@ definitions: value: param1: "value1" param2: "{{ config['param2_value'] }}" +<<<<<<< Updated upstream - type: RequestBodyGraphQL value: query: @@ -2121,6 +2162,8 @@ definitions: - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - searchIn: "{{ ','.join(config.get('search_in', [])) }}" - sort_by[asc]: updated_at +======= +>>>>>>> Stashed changes error_handler: title: Error Handler description: Error handler component that defines how to handle errors. @@ -3100,7 +3143,7 @@ definitions: type: array default: [] items: - - type: string + type: string interpolation_context: - config parent_key: @@ -3194,8 +3237,8 @@ definitions: retriever: description: Requester component that describes how to fetch the properties to query from a remote API endpoint. anyOf: - - "$ref": "#/definitions/CustomRetriever" - "$ref": "#/definitions/SimpleRetriever" + - "$ref": "#/definitions/CustomRetriever" $parameters: type: object additionalProperties: true @@ -3426,6 +3469,7 @@ definitions: additionalProperties: true LegacySessionTokenAuthenticator: title: Session Token Authenticator + deprecated: true description: Deprecated - use SessionTokenAuthenticator instead. Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session. type: object required: @@ -3534,15 +3578,15 @@ definitions: - "$ref": "#/definitions/HttpRequester" - "$ref": "#/definitions/CustomRequester" decoder: - title: Decoder + title: HTTP Response Format description: Component decoding the response so records can be extracted. anyOf: - - "$ref": "#/definitions/CsvDecoder" - - "$ref": "#/definitions/GzipDecoder" - "$ref": "#/definitions/JsonDecoder" + - "$ref": "#/definitions/XmlDecoder" + - "$ref": "#/definitions/CsvDecoder" - "$ref": "#/definitions/JsonlDecoder" + - "$ref": "#/definitions/GzipDecoder" - "$ref": "#/definitions/IterableDecoder" - - "$ref": "#/definitions/XmlDecoder" - "$ref": "#/definitions/ZipfileDecoder" - "$ref": "#/definitions/CustomDecoder" record_selector: From 9dc835b273ded4f79ba7d4737e725f7be1c1d09f Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 30 Apr 2025 17:00:07 -0700 Subject: [PATCH 2/5] fix merge conflict --- .../declarative_component_schema.yaml | 45 +------------------ 1 file changed, 1 insertion(+), 44 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index bdc33d992..9b6b27dd2 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2085,7 +2085,7 @@ definitions: field: "updated_at" order: "ascending" request_body: - title: Request Body + title: Request Body Payload to be send as a part of the API request. description: Specifies how to populate the body of the request with a payload. Can contain nested objects. anyOf: - "$ref": "#/definitions/RequestBodyPlainText" @@ -2116,54 +2116,11 @@ definitions: value: param1: "value1" param2: "{{ config['param2_value'] }}" -<<<<<<< Updated upstream - type: RequestBodyGraphQL value: query: param1: "value1" param2: "{{ config['param2_value'] }}" - request_headers: - title: Request Headers - description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. - anyOf: - - type: object - title: Key/Value Pairs - additionalProperties: - type: string - - type: string - title: Interpolated Value - interpolation_context: - - next_page_token - - stream_interval - - stream_partition - - stream_slice - examples: - - Output-Format: JSON - - Version: "{{ config['version'] }}" - request_parameters: - title: Query Parameters - description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. - anyOf: - - type: object - title: Key/Value Pairs - additionalProperties: - anyOf: - - type: string - - $ref: "#/definitions/QueryProperties" - - type: string - title: Interpolated Value - interpolation_context: - - next_page_token - - stream_interval - - stream_partition - - stream_slice - examples: - - unit: "day" - - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - - searchIn: "{{ ','.join(config.get('search_in', [])) }}" - - sort_by[asc]: updated_at -======= ->>>>>>> Stashed changes error_handler: title: Error Handler description: Error handler component that defines how to handle errors. From e315dcb53d015ea08be2720d7ef3c363598e5293 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 1 May 2025 16:22:18 -0700 Subject: [PATCH 3/5] fix primary key and request body titles --- .../sources/declarative/declarative_component_schema.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 9b6b27dd2..09c32126a 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -1417,7 +1417,6 @@ definitions: - "$ref": "#/definitions/CustomIncrementalSync" primary_key: title: Primary Key - description: The primary key of the stream. "$ref": "#/definitions/PrimaryKey" default: "" schema_loader: @@ -2085,7 +2084,7 @@ definitions: field: "updated_at" order: "ascending" request_body: - title: Request Body Payload to be send as a part of the API request. + title: Request Body description: Specifies how to populate the body of the request with a payload. Can contain nested objects. anyOf: - "$ref": "#/definitions/RequestBodyPlainText" @@ -3155,14 +3154,19 @@ definitions: anyOf: - type: string title: Single Key + description: The single top-level field to use as the primary key. - type: array title: Composite Key + description: An array of top-level fields representing a composite primary key. items: type: string - type: array title: Composite Key of Nested Fields + description: An array of arrays representing a composite primary key where the fields are nested fields. items: type: array + title: Nested Field Path + description: Path to the nested field in the record. items: type: string default: "" From 38c08e7eb6428cdc221cb90e12ddbe51e3f7dfa5 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 1 May 2025 18:50:53 -0700 Subject: [PATCH 4/5] reorder and fix descriptions --- .../declarative_component_schema.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 09c32126a..8e247f199 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -299,8 +299,8 @@ definitions: - "$ref": "#/definitions/CustomAuthenticator" - "$ref": "#/definitions/OAuthAuthenticator" - "$ref": "#/definitions/JwtAuthenticator" - - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" examples: - authenticators: @@ -1467,18 +1467,18 @@ definitions: requester: description: Requester component that describes how to prepare HTTP requests to send to the source API. anyOf: - - "$ref": "#/definitions/CustomRequester" - "$ref": "#/definitions/HttpRequester" + - "$ref": "#/definitions/CustomRequester" download_target_extractor: description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response anyOf: - - "$ref": "#/definitions/CustomRecordExtractor" - "$ref": "#/definitions/DpathExtractor" + - "$ref": "#/definitions/CustomRecordExtractor" file_extractor: description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content anyOf: - - "$ref": "#/definitions/CustomRecordExtractor" - "$ref": "#/definitions/DpathExtractor" + - "$ref": "#/definitions/CustomRecordExtractor" filename_extractor: description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided. type: string @@ -1918,7 +1918,7 @@ definitions: deprecated: true deprecation_message: "Use `url` field instead." title: API Base URL - description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. linkable: true type: string interpolation_context: @@ -1937,7 +1937,7 @@ definitions: - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" url: title: API Endpoint URL - description: The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. type: string interpolation_context: - config @@ -1957,7 +1957,7 @@ definitions: deprecated: true deprecation_message: "Use `url` field instead." title: URL Path - description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. type: string interpolation_context: - config @@ -1988,7 +1988,6 @@ definitions: linkable: true description: Authentication method to use for requests sent to the API. anyOf: - - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/ApiKeyAuthenticator" - "$ref": "#/definitions/BasicHttpAuthenticator" - "$ref": "#/definitions/BearerAuthenticator" @@ -1997,6 +1996,7 @@ definitions: - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/SelectiveAuthenticator" - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" fetch_properties_from_endpoint: title: Fetch Properties from Endpoint From 38c9e349841654977462515f2ecf68bff7d41b91 Mon Sep 17 00:00:00 2001 From: Oleksandr Bazarnov Date: Fri, 2 May 2025 11:07:38 +0300 Subject: [PATCH 5/5] generated models from schema changes --- .../models/declarative_component_schema.py | 72 +++++++++---------- 1 file changed, 35 insertions(+), 37 deletions(-) diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 720fa00ed..df8a4d08b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2080,8 +2080,8 @@ class Config: CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, - NoAuth, SessionTokenAuthenticator, + NoAuth, LegacySessionTokenAuthenticator, ], ] = Field( @@ -2103,15 +2103,15 @@ class Config: class FileUploader(BaseModel): type: Literal["FileUploader"] - requester: Union[CustomRequester, HttpRequester] = Field( + requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) - download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( + download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", ) - file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field( + file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( None, description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", ) @@ -2144,9 +2144,7 @@ class Config: description="Component used to fetch data incrementally based on a time field in the data.", title="Incremental Sync", ) - primary_key: Optional[PrimaryKey] = Field( - "", description="The primary key of the stream.", title="Primary Key" - ) + primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ Union[ InlineSchemaLoader, @@ -2242,7 +2240,7 @@ class HttpRequester(BaseModelWithDeprecations): None, deprecated=True, deprecation_message="Use `url` field instead.", - description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api", @@ -2253,20 +2251,20 @@ class HttpRequester(BaseModelWithDeprecations): ) url: Optional[str] = Field( None, - description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "https://connect.squareup.com/v2", "{{ config['url'] or 'https://app.posthog.com'}}/api", "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", ], - title="The URL of an API endpoint", + title="API Endpoint URL", ) path: Optional[str] = Field( None, deprecated=True, deprecation_message="Use `url` field instead.", - description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", @@ -2282,7 +2280,6 @@ class HttpRequester(BaseModelWithDeprecations): ) authenticator: Optional[ Union[ - NoAuth, ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, @@ -2291,6 +2288,7 @@ class HttpRequester(BaseModelWithDeprecations): SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, + NoAuth, LegacySessionTokenAuthenticator, ] ] = Field( @@ -2303,6 +2301,25 @@ class HttpRequester(BaseModelWithDeprecations): description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", title="Fetch Properties from Endpoint", ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) + request_headers: Optional[Union[Dict[str, str], str]] = Field( + None, + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", + ) request_body_data: Optional[Union[Dict[str, str], str]] = Field( None, deprecated=True, @@ -2363,26 +2380,7 @@ class HttpRequester(BaseModelWithDeprecations): }, }, ], - title="Request Body Payload to be send as a part of the API request.", - ) - request_headers: Optional[Union[Dict[str, str], str]] = Field( - None, - description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", - examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], - title="Request Headers", - ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + title="Request Body", ) error_handler: Optional[ Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] @@ -2475,7 +2473,7 @@ class PropertiesFromEndpoint(BaseModel): description="Describes the path to the field that should be extracted", examples=[["name"]], ) - retriever: Union[CustomRetriever, SimpleRetriever] = Field( + retriever: Union[SimpleRetriever, CustomRetriever] = Field( ..., description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", ) @@ -2526,19 +2524,19 @@ class SimpleRetriever(BaseModel): ) decoder: Optional[ Union[ - CsvDecoder, - GzipDecoder, JsonDecoder, + XmlDecoder, + CsvDecoder, JsonlDecoder, + GzipDecoder, IterableDecoder, - XmlDecoder, ZipfileDecoder, CustomDecoder, ] ] = Field( None, description="Component decoding the response so records can be extracted.", - title="Decoder", + title="HTTP Response Format", ) record_selector: RecordSelector = Field( ...,