diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index 9d2b6d6b6..8e247f199 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -299,8 +299,8 @@ definitions: - "$ref": "#/definitions/CustomAuthenticator" - "$ref": "#/definitions/OAuthAuthenticator" - "$ref": "#/definitions/JwtAuthenticator" - - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/SessionTokenAuthenticator" + - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" examples: - authenticators: @@ -1417,7 +1417,6 @@ definitions: - "$ref": "#/definitions/CustomIncrementalSync" primary_key: title: Primary Key - description: The primary key of the stream. "$ref": "#/definitions/PrimaryKey" default: "" schema_loader: @@ -1468,18 +1467,18 @@ definitions: requester: description: Requester component that describes how to prepare HTTP requests to send to the source API. anyOf: - - "$ref": "#/definitions/CustomRequester" - "$ref": "#/definitions/HttpRequester" + - "$ref": "#/definitions/CustomRequester" download_target_extractor: description: Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response anyOf: - - "$ref": "#/definitions/CustomRecordExtractor" - "$ref": "#/definitions/DpathExtractor" + - "$ref": "#/definitions/CustomRecordExtractor" file_extractor: description: Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content anyOf: - - "$ref": "#/definitions/CustomRecordExtractor" - "$ref": "#/definitions/DpathExtractor" + - "$ref": "#/definitions/CustomRecordExtractor" filename_extractor: description: Defines the name to store the file. Stream name is automatically added to the file path. File unique ID can be used to avoid overwriting files. Random UUID will be used if the extractor is not provided. type: string @@ -1919,7 +1918,7 @@ definitions: deprecated: true deprecation_message: "Use `url` field instead." title: API Base URL - description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. linkable: true type: string interpolation_context: @@ -1937,8 +1936,8 @@ definitions: - "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups" - "https://example.com/api/v1/resource/{{ next_page_token['id'] }}" url: - title: The URL of an API endpoint - description: The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + title: API Endpoint URL + description: The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. type: string interpolation_context: - config @@ -1958,7 +1957,7 @@ definitions: deprecated: true deprecation_message: "Use `url` field instead." title: URL Path - description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this. + description: Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this. type: string interpolation_context: - config @@ -1989,7 +1988,6 @@ definitions: linkable: true description: Authentication method to use for requests sent to the API. anyOf: - - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/ApiKeyAuthenticator" - "$ref": "#/definitions/BasicHttpAuthenticator" - "$ref": "#/definitions/BearerAuthenticator" @@ -1998,11 +1996,52 @@ definitions: - "$ref": "#/definitions/SessionTokenAuthenticator" - "$ref": "#/definitions/SelectiveAuthenticator" - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" fetch_properties_from_endpoint: title: Fetch Properties from Endpoint description: Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields. "$ref": "#/definitions/PropertiesFromEndpoint" + request_parameters: + title: Query Parameters + description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. + anyOf: + - type: object + title: Key/Value Pairs + additionalProperties: + anyOf: + - type: string + - $ref: "#/definitions/QueryProperties" + - type: string + title: Interpolated Value + interpolation_context: + - next_page_token + - stream_interval + - stream_partition + - stream_slice + examples: + - unit: "day" + - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + - searchIn: "{{ ','.join(config.get('search_in', [])) }}" + - sort_by[asc]: updated_at + request_headers: + title: Request Headers + description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. + anyOf: + - type: object + title: Key/Value Pairs + additionalProperties: + type: string + - type: string + title: Interpolated Value + interpolation_context: + - next_page_token + - stream_interval + - stream_partition + - stream_slice + examples: + - Output-Format: JSON + - Version: "{{ config['version'] }}" request_body_data: deprecated: true deprecation_message: "Use `request_body` field instead." @@ -2045,7 +2084,7 @@ definitions: field: "updated_at" order: "ascending" request_body: - title: Request Body Payload to be send as a part of the API request. + title: Request Body description: Specifies how to populate the body of the request with a payload. Can contain nested objects. anyOf: - "$ref": "#/definitions/RequestBodyPlainText" @@ -2081,46 +2120,6 @@ definitions: query: param1: "value1" param2: "{{ config['param2_value'] }}" - request_headers: - title: Request Headers - description: Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. - anyOf: - - type: object - title: Key/Value Pairs - additionalProperties: - type: string - - type: string - title: Interpolated Value - interpolation_context: - - next_page_token - - stream_interval - - stream_partition - - stream_slice - examples: - - Output-Format: JSON - - Version: "{{ config['version'] }}" - request_parameters: - title: Query Parameters - description: Specifies the query parameters that should be set on an outgoing HTTP request given the inputs. - anyOf: - - type: object - title: Key/Value Pairs - additionalProperties: - anyOf: - - type: string - - $ref: "#/definitions/QueryProperties" - - type: string - title: Interpolated Value - interpolation_context: - - next_page_token - - stream_interval - - stream_partition - - stream_slice - examples: - - unit: "day" - - query: 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - - searchIn: "{{ ','.join(config.get('search_in', [])) }}" - - sort_by[asc]: updated_at error_handler: title: Error Handler description: Error handler component that defines how to handle errors. @@ -3100,7 +3099,7 @@ definitions: type: array default: [] items: - - type: string + type: string interpolation_context: - config parent_key: @@ -3155,14 +3154,19 @@ definitions: anyOf: - type: string title: Single Key + description: The single top-level field to use as the primary key. - type: array title: Composite Key + description: An array of top-level fields representing a composite primary key. items: type: string - type: array title: Composite Key of Nested Fields + description: An array of arrays representing a composite primary key where the fields are nested fields. items: type: array + title: Nested Field Path + description: Path to the nested field in the record. items: type: string default: "" @@ -3194,8 +3198,8 @@ definitions: retriever: description: Requester component that describes how to fetch the properties to query from a remote API endpoint. anyOf: - - "$ref": "#/definitions/CustomRetriever" - "$ref": "#/definitions/SimpleRetriever" + - "$ref": "#/definitions/CustomRetriever" $parameters: type: object additionalProperties: true @@ -3426,6 +3430,7 @@ definitions: additionalProperties: true LegacySessionTokenAuthenticator: title: Session Token Authenticator + deprecated: true description: Deprecated - use SessionTokenAuthenticator instead. Authenticator for requests authenticated using session tokens. A session token is a random value generated by a server to identify a specific user for the duration of one interaction session. type: object required: @@ -3534,15 +3539,15 @@ definitions: - "$ref": "#/definitions/HttpRequester" - "$ref": "#/definitions/CustomRequester" decoder: - title: Decoder + title: HTTP Response Format description: Component decoding the response so records can be extracted. anyOf: - - "$ref": "#/definitions/CsvDecoder" - - "$ref": "#/definitions/GzipDecoder" - "$ref": "#/definitions/JsonDecoder" + - "$ref": "#/definitions/XmlDecoder" + - "$ref": "#/definitions/CsvDecoder" - "$ref": "#/definitions/JsonlDecoder" + - "$ref": "#/definitions/GzipDecoder" - "$ref": "#/definitions/IterableDecoder" - - "$ref": "#/definitions/XmlDecoder" - "$ref": "#/definitions/ZipfileDecoder" - "$ref": "#/definitions/CustomDecoder" record_selector: diff --git a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py index 720fa00ed..df8a4d08b 100644 --- a/airbyte_cdk/sources/declarative/models/declarative_component_schema.py +++ b/airbyte_cdk/sources/declarative/models/declarative_component_schema.py @@ -2080,8 +2080,8 @@ class Config: CustomAuthenticator, OAuthAuthenticator, JwtAuthenticator, - NoAuth, SessionTokenAuthenticator, + NoAuth, LegacySessionTokenAuthenticator, ], ] = Field( @@ -2103,15 +2103,15 @@ class Config: class FileUploader(BaseModel): type: Literal["FileUploader"] - requester: Union[CustomRequester, HttpRequester] = Field( + requester: Union[HttpRequester, CustomRequester] = Field( ..., description="Requester component that describes how to prepare HTTP requests to send to the source API.", ) - download_target_extractor: Union[CustomRecordExtractor, DpathExtractor] = Field( + download_target_extractor: Union[DpathExtractor, CustomRecordExtractor] = Field( ..., description="Responsible for fetching the url where the file is located. This is applied on each records and not on the HTTP response", ) - file_extractor: Optional[Union[CustomRecordExtractor, DpathExtractor]] = Field( + file_extractor: Optional[Union[DpathExtractor, CustomRecordExtractor]] = Field( None, description="Responsible for fetching the content of the file. If not defined, the assumption is that the whole response body is the file content", ) @@ -2144,9 +2144,7 @@ class Config: description="Component used to fetch data incrementally based on a time field in the data.", title="Incremental Sync", ) - primary_key: Optional[PrimaryKey] = Field( - "", description="The primary key of the stream.", title="Primary Key" - ) + primary_key: Optional[PrimaryKey] = Field("", title="Primary Key") schema_loader: Optional[ Union[ InlineSchemaLoader, @@ -2242,7 +2240,7 @@ class HttpRequester(BaseModelWithDeprecations): None, deprecated=True, deprecation_message="Use `url` field instead.", - description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="Deprecated, use the `url` instead. Base URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "https://connect.squareup.com/v2", "{{ config['base_url'] or 'https://app.posthog.com'}}/api", @@ -2253,20 +2251,20 @@ class HttpRequester(BaseModelWithDeprecations): ) url: Optional[str] = Field( None, - description="The URL of the API source. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="The URL of the source API endpoint. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "https://connect.squareup.com/v2", "{{ config['url'] or 'https://app.posthog.com'}}/api", "https://connect.squareup.com/v2/quotes/{{ stream_partition['id'] }}/quote_line_groups", "https://example.com/api/v1/resource/{{ next_page_token['id'] }}", ], - title="The URL of an API endpoint", + title="API Endpoint URL", ) path: Optional[str] = Field( None, deprecated=True, deprecation_message="Use `url` field instead.", - description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authentication component for this.", + description="Deprecated, use the `url` instead. Path the specific API endpoint that this stream represents. Do not put sensitive information (e.g. API tokens) into this field - Use the Authenticator component for this.", examples=[ "/products", "/quotes/{{ stream_partition['id'] }}/quote_line_groups", @@ -2282,7 +2280,6 @@ class HttpRequester(BaseModelWithDeprecations): ) authenticator: Optional[ Union[ - NoAuth, ApiKeyAuthenticator, BasicHttpAuthenticator, BearerAuthenticator, @@ -2291,6 +2288,7 @@ class HttpRequester(BaseModelWithDeprecations): SessionTokenAuthenticator, SelectiveAuthenticator, CustomAuthenticator, + NoAuth, LegacySessionTokenAuthenticator, ] ] = Field( @@ -2303,6 +2301,25 @@ class HttpRequester(BaseModelWithDeprecations): description="Allows for retrieving a dynamic set of properties from an API endpoint which can be injected into outbound request using the stream_partition.extra_fields.", title="Fetch Properties from Endpoint", ) + request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( + None, + description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", + examples=[ + {"unit": "day"}, + { + "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' + }, + {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, + {"sort_by[asc]": "updated_at"}, + ], + title="Query Parameters", + ) + request_headers: Optional[Union[Dict[str, str], str]] = Field( + None, + description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", + examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], + title="Request Headers", + ) request_body_data: Optional[Union[Dict[str, str], str]] = Field( None, deprecated=True, @@ -2363,26 +2380,7 @@ class HttpRequester(BaseModelWithDeprecations): }, }, ], - title="Request Body Payload to be send as a part of the API request.", - ) - request_headers: Optional[Union[Dict[str, str], str]] = Field( - None, - description="Return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method.", - examples=[{"Output-Format": "JSON"}, {"Version": "{{ config['version'] }}"}], - title="Request Headers", - ) - request_parameters: Optional[Union[Dict[str, Union[str, QueryProperties]], str]] = Field( - None, - description="Specifies the query parameters that should be set on an outgoing HTTP request given the inputs.", - examples=[ - {"unit": "day"}, - { - "query": 'last_event_time BETWEEN TIMESTAMP "{{ stream_interval.start_time }}" AND TIMESTAMP "{{ stream_interval.end_time }}"' - }, - {"searchIn": "{{ ','.join(config.get('search_in', [])) }}"}, - {"sort_by[asc]": "updated_at"}, - ], - title="Query Parameters", + title="Request Body", ) error_handler: Optional[ Union[DefaultErrorHandler, CompositeErrorHandler, CustomErrorHandler] @@ -2475,7 +2473,7 @@ class PropertiesFromEndpoint(BaseModel): description="Describes the path to the field that should be extracted", examples=[["name"]], ) - retriever: Union[CustomRetriever, SimpleRetriever] = Field( + retriever: Union[SimpleRetriever, CustomRetriever] = Field( ..., description="Requester component that describes how to fetch the properties to query from a remote API endpoint.", ) @@ -2526,19 +2524,19 @@ class SimpleRetriever(BaseModel): ) decoder: Optional[ Union[ - CsvDecoder, - GzipDecoder, JsonDecoder, + XmlDecoder, + CsvDecoder, JsonlDecoder, + GzipDecoder, IterableDecoder, - XmlDecoder, ZipfileDecoder, CustomDecoder, ] ] = Field( None, description="Component decoding the response so records can be extracted.", - title="Decoder", + title="HTTP Response Format", ) record_selector: RecordSelector = Field( ...,