From 3b3a5333454d8555979b4b432259e8933aec4e87 Mon Sep 17 00:00:00 2001 From: lmossman Date: Thu, 15 May 2025 13:47:21 -0700 Subject: [PATCH 1/2] reorder more anyOf options and adjust titles --- .../sources/declarative/declarative_component_schema.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index ca595d92e..e4abba544 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -2315,9 +2315,9 @@ definitions: title: Retriever description: Component used to coordinate how records are extracted across stream slices and request pages. anyOf: + - "$ref": "#/definitions/SimpleRetriever" - "$ref": "#/definitions/AsyncRetriever" - "$ref": "#/definitions/CustomRetriever" - - "$ref": "#/definitions/SimpleRetriever" schema_transformations: title: Schema Transformations description: A list of transformations to be applied to the schema. @@ -3751,7 +3751,7 @@ definitions: - "$ref": "#/definitions/GroupingPartitionRouter" - "$ref": "#/definitions/CustomPartitionRouter" decoder: - title: Decoder + title: HTTP Response Format description: Component decoding the response so records can be extracted. anyOf: - "$ref": "#/definitions/CsvDecoder" @@ -3763,7 +3763,7 @@ definitions: - "$ref": "#/definitions/ZipfileDecoder" - "$ref": "#/definitions/CustomDecoder" download_decoder: - title: Download Decoder + title: Download HTTP Response Format description: Component decoding the download response so records can be extracted. anyOf: - "$ref": "#/definitions/CsvDecoder" From 38f8d189ea97cedb836ab671df094b4ae7729e21 Mon Sep 17 00:00:00 2001 From: lmossman Date: Wed, 28 May 2025 16:44:07 -0700 Subject: [PATCH 2/2] more reordering, updating titles, and added deprecation --- .../declarative_component_schema.yaml | 107 ++++++++++-------- 1 file changed, 57 insertions(+), 50 deletions(-) diff --git a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml index e4abba544..224f7f6a2 100644 --- a/airbyte_cdk/sources/declarative/declarative_component_schema.yaml +++ b/airbyte_cdk/sources/declarative/declarative_component_schema.yaml @@ -296,12 +296,12 @@ definitions: - "$ref": "#/definitions/ApiKeyAuthenticator" - "$ref": "#/definitions/BasicHttpAuthenticator" - "$ref": "#/definitions/BearerAuthenticator" - - "$ref": "#/definitions/CustomAuthenticator" - "$ref": "#/definitions/OAuthAuthenticator" - "$ref": "#/definitions/JwtAuthenticator" - "$ref": "#/definitions/SessionTokenAuthenticator" - - "$ref": "#/definitions/NoAuth" - "$ref": "#/definitions/LegacySessionTokenAuthenticator" + - "$ref": "#/definitions/CustomAuthenticator" + - "$ref": "#/definitions/NoAuth" examples: - authenticators: token: "#/definitions/ApiKeyAuthenticator" @@ -907,6 +907,49 @@ definitions: examples: - "created_at" - "{{ config['record_cursor'] }}" + cursor_datetime_formats: + title: Cursor Datetime Formats + description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used. + type: array + items: + type: string + examples: + - "%Y-%m-%dT%H:%M:%S.%f%z" + - "%Y-%m-%d" + - "%s" + start_datetime: + title: Start Datetime + description: The datetime that determines the earliest record that should be synced. + anyOf: + - "$ref": "#/definitions/MinMaxDatetime" + - type: string + title: Interpolated Value + interpolation_context: + - config + examples: + - "2020-01-1T00:00:00Z" + - "{{ config['start_time'] }}" + start_time_option: + title: Inject Start Time Into Outgoing HTTP Request + description: Optionally configures how the start datetime will be sent in requests to the source API. + "$ref": "#/definitions/RequestOption" + end_datetime: + title: End Datetime + description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used. + anyOf: + - "$ref": "#/definitions/MinMaxDatetime" + - type: string + title: Interpolated Value + interpolation_context: + - config + examples: + - "2021-01-1T00:00:00Z" + - "{{ now_utc() }}" + - "{{ day_delta(-1) }}" + end_time_option: + title: Inject End Time Into Outgoing HTTP Request + description: Optionally configures how the end datetime will be sent in requests to the source API. + "$ref": "#/definitions/RequestOption" datetime_format: title: Outgoing Datetime Format description: | @@ -948,27 +991,6 @@ definitions: - "%s" - "%ms" - "%s_as_float" - start_datetime: - title: Start Datetime - description: The datetime that determines the earliest record that should be synced. - anyOf: - - type: string - - "$ref": "#/definitions/MinMaxDatetime" - interpolation_context: - - config - examples: - - "2020-01-1T00:00:00Z" - - "{{ config['start_time'] }}" - cursor_datetime_formats: - title: Cursor Datetime Formats - description: The possible formats for the cursor field, in order of preference. The first format that matches the cursor field value will be used to parse it. If not provided, the `datetime_format` will be used. - type: array - items: - type: string - examples: - - "%Y-%m-%dT%H:%M:%S.%f%z" - - "%Y-%m-%d" - - "%s" cursor_granularity: title: Cursor Granularity description: @@ -977,22 +999,6 @@ definitions: type: string examples: - "PT1S" - end_datetime: - title: End Datetime - description: The datetime that determines the last record that should be synced. If not provided, `{{ now_utc() }}` will be used. - anyOf: - - type: string - - "$ref": "#/definitions/MinMaxDatetime" - interpolation_context: - - config - examples: - - "2021-01-1T00:00:00Z" - - "{{ now_utc() }}" - - "{{ day_delta(-1) }}" - end_time_option: - title: Inject End Time Into Outgoing HTTP Request - description: Optionally configures how the end datetime will be sent in requests to the source API. - "$ref": "#/definitions/RequestOption" is_data_feed: title: Whether the target API is formatted as a data feed description: A data feed API is an API that does not allow filtering and paginates the content from the most recent to the least recent. Given this, the CDK needs to know when to stop paginating and this field will generate a stop condition for pagination. @@ -1032,10 +1038,6 @@ definitions: type: string examples: - "starting_time" - start_time_option: - title: Inject Start Time Into Outgoing HTTP Request - description: Optionally configures how the start datetime will be sent in requests to the source API. - "$ref": "#/definitions/RequestOption" step: title: Step description: The size of the time window (ISO8601 duration). Given this field is provided, `cursor_granularity` needs to be provided as well. @@ -1429,14 +1431,15 @@ definitions: - "$ref": "#/definitions/InlineSchemaLoader" - "$ref": "#/definitions/DynamicSchemaLoader" - "$ref": "#/definitions/JsonFileSchemaLoader" - - "$ref": "#/definitions/CustomSchemaLoader" - - type: array + - title: Multiple Schema Loaders + type: array items: anyOf: - "$ref": "#/definitions/InlineSchemaLoader" - "$ref": "#/definitions/DynamicSchemaLoader" - "$ref": "#/definitions/JsonFileSchemaLoader" - "$ref": "#/definitions/CustomSchemaLoader" + - "$ref": "#/definitions/CustomSchemaLoader" # TODO we have move the transformation to the RecordSelector level in the code but kept this here for # compatibility reason. We should eventually move this to align with the code. transformations: @@ -1446,13 +1449,13 @@ definitions: items: anyOf: - "$ref": "#/definitions/AddFields" - - "$ref": "#/definitions/CustomTransformation" - "$ref": "#/definitions/RemoveFields" - "$ref": "#/definitions/KeysToLower" - "$ref": "#/definitions/KeysToSnakeCase" - "$ref": "#/definitions/FlattenFields" - "$ref": "#/definitions/DpathFlattenFields" - "$ref": "#/definitions/KeysReplace" + - "$ref": "#/definitions/CustomTransformation" state_migrations: title: State Migrations description: Array of state migrations to be applied on the input state @@ -1689,10 +1692,10 @@ definitions: items: anyOf: - "$ref": "#/definitions/ConstantBackoffStrategy" - - "$ref": "#/definitions/CustomBackoffStrategy" - "$ref": "#/definitions/ExponentialBackoffStrategy" - "$ref": "#/definitions/WaitTimeFromHeader" - "$ref": "#/definitions/WaitUntilTimeFromHeader" + - "$ref": "#/definitions/CustomBackoffStrategy" max_retries: title: Max Retry Count description: The maximum number of time to retry a retryable request before giving up and failing. @@ -2325,13 +2328,13 @@ definitions: items: anyOf: - "$ref": "#/definitions/AddFields" - - "$ref": "#/definitions/CustomTransformation" - "$ref": "#/definitions/RemoveFields" - "$ref": "#/definitions/KeysToLower" - "$ref": "#/definitions/KeysToSnakeCase" - "$ref": "#/definitions/FlattenFields" - "$ref": "#/definitions/DpathFlattenFields" - "$ref": "#/definitions/KeysReplace" + - "$ref": "#/definitions/CustomTransformation" schema_type_identifier: "$ref": "#/definitions/SchemaTypeIdentifier" $parameters: @@ -2353,6 +2356,8 @@ definitions: type: object additionalProperties: true JsonFileSchemaLoader: + deprecated: true + deprecation_message: "Use `InlineSchemaLoader` instead." title: Json File Schema Loader description: Loads the schema from a json file. type: object @@ -3893,9 +3898,9 @@ definitions: title: Underlying Partition Router description: The partition router whose output will be grouped. This can be any valid partition router component. anyOf: - - "$ref": "#/definitions/CustomPartitionRouter" - "$ref": "#/definitions/ListPartitionRouter" - "$ref": "#/definitions/SubstreamPartitionRouter" + - "$ref": "#/definitions/CustomPartitionRouter" deduplicate: title: Deduplicate Partitions description: If true, ensures that partitions are unique within each group by removing duplicates based on the partition key. @@ -3999,6 +4004,7 @@ definitions: additionalProperties: true HttpComponentsResolver: type: object + title: Http Components Resolver description: (This component is experimental. Use at your own risk.) Component resolve and populates stream templates with components fetched via an HTTP retriever. properties: type: @@ -4008,9 +4014,9 @@ definitions: title: Retriever description: Component used to coordinate how records are extracted across stream slices and request pages. anyOf: + - "$ref": "#/definitions/SimpleRetriever" - "$ref": "#/definitions/AsyncRetriever" - "$ref": "#/definitions/CustomRetriever" - - "$ref": "#/definitions/SimpleRetriever" components_mapping: type: array items: @@ -4050,6 +4056,7 @@ definitions: additionalProperties: true ConfigComponentsResolver: type: object + title: Config Components Resolver description: (This component is experimental. Use at your own risk.) Resolves and populates stream templates with components fetched from the source config. properties: type: