diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 82b843ff62..db84dc8b40 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -75,7 +75,7 @@ resources.apps.*.compute_status *apps.ComputeStatus ALL resources.apps.*.compute_status.message string ALL resources.apps.*.compute_status.state apps.ComputeState ALL resources.apps.*.config map[string]any INPUT -resources.apps.*.config[*] any INPUT +resources.apps.*.config.* any INPUT resources.apps.*.create_time string ALL resources.apps.*.creator string ALL resources.apps.*.default_source_code_path string ALL @@ -293,7 +293,7 @@ resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes *compute.V resources.jobs.*.job_clusters[*].new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.cluster_name string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -344,9 +344,9 @@ resources.jobs.*.job_clusters[*].new_cluster.remote_disk_throughput int INPUT ST resources.jobs.*.job_clusters[*].new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.single_user_name string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.job_clusters[*].new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.spark_version string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.job_clusters[*].new_cluster.ssh_public_keys[*] string INPUT STATE @@ -480,7 +480,7 @@ resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_log_conf.volumes * resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.cluster_name string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -531,9 +531,9 @@ resources.jobs.*.settings.job_clusters[*].new_cluster.remote_disk_throughput int resources.jobs.*.settings.job_clusters[*].new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.single_user_name string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.job_clusters[*].new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.spark_version string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.job_clusters[*].new_cluster.ssh_public_keys[*] string REMOTE @@ -563,14 +563,14 @@ resources.jobs.*.settings.schedule.pause_status jobs.PauseStatus REMOTE resources.jobs.*.settings.schedule.quartz_cron_expression string REMOTE resources.jobs.*.settings.schedule.timezone_id string REMOTE resources.jobs.*.settings.tags map[string]string REMOTE -resources.jobs.*.settings.tags[*] string REMOTE +resources.jobs.*.settings.tags.* string REMOTE resources.jobs.*.settings.tasks []jobs.Task REMOTE resources.jobs.*.settings.tasks[*] jobs.Task REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task *jobs.CleanRoomsNotebookTask REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.clean_room_name string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.etag string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].clean_rooms_notebook_task.notebook_name string REMOTE resources.jobs.*.settings.tasks[*].condition_task *jobs.ConditionTask REMOTE resources.jobs.*.settings.tasks[*].condition_task.left string REMOTE @@ -629,7 +629,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.clean_room_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.etag string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.condition_task *jobs.ConditionTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.condition_task.left string REMOTE @@ -761,7 +761,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_log_co resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.cluster_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -812,9 +812,9 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.remote_disk_th resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.single_user_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.spark_version string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.ssh_public_keys[*] string REMOTE @@ -826,7 +826,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.workload_type. resources.jobs.*.settings.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks bool REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task *jobs.NotebookTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.notebook_path string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.notebook_task.warehouse_id string REMOTE @@ -856,7 +856,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.power_bi_task.warehouse_id resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task *jobs.PythonWheelTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.entry_point string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.named_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.package_name string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.parameters []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.python_wheel_task.parameters[*] string REMOTE @@ -869,19 +869,19 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.jar_params [] resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.jar_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_id int64 REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.job_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.notebook_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.pipeline_params *jobs.PipelineParams REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh bool REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_named_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_params []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.python_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.spark_submit_params []string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.spark_submit_params[*] string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.run_job_task.sql_params.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task *jobs.SparkJarTask REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task.jar_uri string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.spark_jar_task.main_class_name string REMOTE @@ -916,7 +916,7 @@ resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file *jobs.SqlTas resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file.path string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.file.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.parameters.* string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.query *jobs.SqlTaskQuery REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.query.query_id string REMOTE resources.jobs.*.settings.tasks[*].for_each_task.task.sql_task.warehouse_id string REMOTE @@ -1015,7 +1015,7 @@ resources.jobs.*.settings.tasks[*].new_cluster.cluster_log_conf.volumes *compute resources.jobs.*.settings.tasks[*].new_cluster.cluster_log_conf.volumes.destination string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.cluster_name string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.custom_tags map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.custom_tags[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.custom_tags.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.data_security_mode compute.DataSecurityMode REMOTE resources.jobs.*.settings.tasks[*].new_cluster.docker_image *compute.DockerImage REMOTE resources.jobs.*.settings.tasks[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth REMOTE @@ -1066,9 +1066,9 @@ resources.jobs.*.settings.tasks[*].new_cluster.remote_disk_throughput int REMOTE resources.jobs.*.settings.tasks[*].new_cluster.runtime_engine compute.RuntimeEngine REMOTE resources.jobs.*.settings.tasks[*].new_cluster.single_user_name string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_conf map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.spark_conf[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.spark_conf.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars map[string]string REMOTE -resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars[*] string REMOTE +resources.jobs.*.settings.tasks[*].new_cluster.spark_env_vars.* string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.spark_version string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.ssh_public_keys []string REMOTE resources.jobs.*.settings.tasks[*].new_cluster.ssh_public_keys[*] string REMOTE @@ -1080,7 +1080,7 @@ resources.jobs.*.settings.tasks[*].new_cluster.workload_type.clients.jobs bool R resources.jobs.*.settings.tasks[*].new_cluster.workload_type.clients.notebooks bool REMOTE resources.jobs.*.settings.tasks[*].notebook_task *jobs.NotebookTask REMOTE resources.jobs.*.settings.tasks[*].notebook_task.base_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].notebook_task.base_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].notebook_task.base_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].notebook_task.notebook_path string REMOTE resources.jobs.*.settings.tasks[*].notebook_task.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].notebook_task.warehouse_id string REMOTE @@ -1110,7 +1110,7 @@ resources.jobs.*.settings.tasks[*].power_bi_task.warehouse_id string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task *jobs.PythonWheelTask REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.entry_point string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].python_wheel_task.named_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.package_name string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.parameters []string REMOTE resources.jobs.*.settings.tasks[*].python_wheel_task.parameters[*] string REMOTE @@ -1123,19 +1123,19 @@ resources.jobs.*.settings.tasks[*].run_job_task.jar_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.jar_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.job_id int64 REMOTE resources.jobs.*.settings.tasks[*].run_job_task.job_parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.job_parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.job_parameters.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.notebook_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.notebook_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.notebook_params.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.pipeline_params *jobs.PipelineParams REMOTE resources.jobs.*.settings.tasks[*].run_job_task.pipeline_params.full_refresh bool REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_named_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.python_named_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.python_named_params.* string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.python_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.spark_submit_params []string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.spark_submit_params[*] string REMOTE resources.jobs.*.settings.tasks[*].run_job_task.sql_params map[string]string REMOTE -resources.jobs.*.settings.tasks[*].run_job_task.sql_params[*] string REMOTE +resources.jobs.*.settings.tasks[*].run_job_task.sql_params.* string REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task *jobs.SparkJarTask REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task.jar_uri string REMOTE resources.jobs.*.settings.tasks[*].spark_jar_task.main_class_name string REMOTE @@ -1170,7 +1170,7 @@ resources.jobs.*.settings.tasks[*].sql_task.file *jobs.SqlTaskFile REMOTE resources.jobs.*.settings.tasks[*].sql_task.file.path string REMOTE resources.jobs.*.settings.tasks[*].sql_task.file.source jobs.Source REMOTE resources.jobs.*.settings.tasks[*].sql_task.parameters map[string]string REMOTE -resources.jobs.*.settings.tasks[*].sql_task.parameters[*] string REMOTE +resources.jobs.*.settings.tasks[*].sql_task.parameters.* string REMOTE resources.jobs.*.settings.tasks[*].sql_task.query *jobs.SqlTaskQuery REMOTE resources.jobs.*.settings.tasks[*].sql_task.query.query_id string REMOTE resources.jobs.*.settings.tasks[*].sql_task.warehouse_id string REMOTE @@ -1232,14 +1232,14 @@ resources.jobs.*.settings.webhook_notifications.on_success []jobs.Webhook REMOTE resources.jobs.*.settings.webhook_notifications.on_success[*] jobs.Webhook REMOTE resources.jobs.*.settings.webhook_notifications.on_success[*].id string REMOTE resources.jobs.*.tags map[string]string INPUT STATE -resources.jobs.*.tags[*] string INPUT STATE +resources.jobs.*.tags.* string INPUT STATE resources.jobs.*.tasks []jobs.Task INPUT STATE resources.jobs.*.tasks[*] jobs.Task INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task *jobs.CleanRoomsNotebookTask INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.clean_room_name string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.etag string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].clean_rooms_notebook_task.notebook_name string INPUT STATE resources.jobs.*.tasks[*].condition_task *jobs.ConditionTask INPUT STATE resources.jobs.*.tasks[*].condition_task.left string INPUT STATE @@ -1298,7 +1298,7 @@ resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task *jobs.Cle resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.clean_room_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.etag string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.clean_rooms_notebook_task.notebook_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.condition_task *jobs.ConditionTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.condition_task.left string INPUT STATE @@ -1430,7 +1430,7 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volume resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.cluster_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -1481,9 +1481,9 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.remote_disk_throughput resources.jobs.*.tasks[*].for_each_task.task.new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.single_user_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.spark_version string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.new_cluster.ssh_public_keys[*] string INPUT STATE @@ -1495,7 +1495,7 @@ resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.j resources.jobs.*.tasks[*].for_each_task.task.new_cluster.workload_type.clients.notebooks bool INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task *jobs.NotebookTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.notebook_task.base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.notebook_path string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.notebook_task.warehouse_id string INPUT STATE @@ -1525,7 +1525,7 @@ resources.jobs.*.tasks[*].for_each_task.task.power_bi_task.warehouse_id string I resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task *jobs.PythonWheelTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.entry_point string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.named_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.package_name string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.python_wheel_task.parameters[*] string INPUT STATE @@ -1538,19 +1538,19 @@ resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params []string IN resources.jobs.*.tasks[*].for_each_task.task.run_job_task.jar_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_id int64 INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.job_parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.notebook_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params *jobs.PipelineParams INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.pipeline_params.full_refresh bool INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_named_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.python_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params []string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.spark_submit_params[*] string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.run_job_task.sql_params.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task *jobs.SparkJarTask INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.jar_uri string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.spark_jar_task.main_class_name string INPUT STATE @@ -1585,7 +1585,7 @@ resources.jobs.*.tasks[*].for_each_task.task.sql_task.file *jobs.SqlTaskFile INP resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.path string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.file.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].for_each_task.task.sql_task.parameters.* string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.query *jobs.SqlTaskQuery INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.query.query_id string INPUT STATE resources.jobs.*.tasks[*].for_each_task.task.sql_task.warehouse_id string INPUT STATE @@ -1684,7 +1684,7 @@ resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes *compute.VolumesS resources.jobs.*.tasks[*].new_cluster.cluster_log_conf.volumes.destination string INPUT STATE resources.jobs.*.tasks[*].new_cluster.cluster_name string INPUT STATE resources.jobs.*.tasks[*].new_cluster.custom_tags map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.custom_tags[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.custom_tags.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.data_security_mode compute.DataSecurityMode INPUT STATE resources.jobs.*.tasks[*].new_cluster.docker_image *compute.DockerImage INPUT STATE resources.jobs.*.tasks[*].new_cluster.docker_image.basic_auth *compute.DockerBasicAuth INPUT STATE @@ -1735,9 +1735,9 @@ resources.jobs.*.tasks[*].new_cluster.remote_disk_throughput int INPUT STATE resources.jobs.*.tasks[*].new_cluster.runtime_engine compute.RuntimeEngine INPUT STATE resources.jobs.*.tasks[*].new_cluster.single_user_name string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_conf map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.spark_conf[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.spark_conf.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_env_vars map[string]string INPUT STATE -resources.jobs.*.tasks[*].new_cluster.spark_env_vars[*] string INPUT STATE +resources.jobs.*.tasks[*].new_cluster.spark_env_vars.* string INPUT STATE resources.jobs.*.tasks[*].new_cluster.spark_version string INPUT STATE resources.jobs.*.tasks[*].new_cluster.ssh_public_keys []string INPUT STATE resources.jobs.*.tasks[*].new_cluster.ssh_public_keys[*] string INPUT STATE @@ -1749,7 +1749,7 @@ resources.jobs.*.tasks[*].new_cluster.workload_type.clients.jobs bool INPUT STAT resources.jobs.*.tasks[*].new_cluster.workload_type.clients.notebooks bool INPUT STATE resources.jobs.*.tasks[*].notebook_task *jobs.NotebookTask INPUT STATE resources.jobs.*.tasks[*].notebook_task.base_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].notebook_task.base_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].notebook_task.base_parameters.* string INPUT STATE resources.jobs.*.tasks[*].notebook_task.notebook_path string INPUT STATE resources.jobs.*.tasks[*].notebook_task.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].notebook_task.warehouse_id string INPUT STATE @@ -1779,7 +1779,7 @@ resources.jobs.*.tasks[*].power_bi_task.warehouse_id string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task *jobs.PythonWheelTask INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.entry_point string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.named_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].python_wheel_task.named_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].python_wheel_task.named_parameters.* string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.package_name string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.parameters []string INPUT STATE resources.jobs.*.tasks[*].python_wheel_task.parameters[*] string INPUT STATE @@ -1792,19 +1792,19 @@ resources.jobs.*.tasks[*].run_job_task.jar_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.jar_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.job_id int64 INPUT STATE resources.jobs.*.tasks[*].run_job_task.job_parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.job_parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.job_parameters.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.notebook_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.notebook_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.notebook_params.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.pipeline_params *jobs.PipelineParams INPUT STATE resources.jobs.*.tasks[*].run_job_task.pipeline_params.full_refresh bool INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_named_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.python_named_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.python_named_params.* string INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.python_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.spark_submit_params []string INPUT STATE resources.jobs.*.tasks[*].run_job_task.spark_submit_params[*] string INPUT STATE resources.jobs.*.tasks[*].run_job_task.sql_params map[string]string INPUT STATE -resources.jobs.*.tasks[*].run_job_task.sql_params[*] string INPUT STATE +resources.jobs.*.tasks[*].run_job_task.sql_params.* string INPUT STATE resources.jobs.*.tasks[*].spark_jar_task *jobs.SparkJarTask INPUT STATE resources.jobs.*.tasks[*].spark_jar_task.jar_uri string INPUT STATE resources.jobs.*.tasks[*].spark_jar_task.main_class_name string INPUT STATE @@ -1839,7 +1839,7 @@ resources.jobs.*.tasks[*].sql_task.file *jobs.SqlTaskFile INPUT STATE resources.jobs.*.tasks[*].sql_task.file.path string INPUT STATE resources.jobs.*.tasks[*].sql_task.file.source jobs.Source INPUT STATE resources.jobs.*.tasks[*].sql_task.parameters map[string]string INPUT STATE -resources.jobs.*.tasks[*].sql_task.parameters[*] string INPUT STATE +resources.jobs.*.tasks[*].sql_task.parameters.* string INPUT STATE resources.jobs.*.tasks[*].sql_task.query *jobs.SqlTaskQuery INPUT STATE resources.jobs.*.tasks[*].sql_task.query.query_id string INPUT STATE resources.jobs.*.tasks[*].sql_task.warehouse_id string INPUT STATE @@ -1955,7 +1955,7 @@ resources.pipelines.*.clusters[*].cluster_log_conf.s3.region string INPUT STATE resources.pipelines.*.clusters[*].cluster_log_conf.volumes *compute.VolumesStorageInfo INPUT STATE resources.pipelines.*.clusters[*].cluster_log_conf.volumes.destination string INPUT STATE resources.pipelines.*.clusters[*].custom_tags map[string]string INPUT STATE -resources.pipelines.*.clusters[*].custom_tags[*] string INPUT STATE +resources.pipelines.*.clusters[*].custom_tags.* string INPUT STATE resources.pipelines.*.clusters[*].driver_instance_pool_id string INPUT STATE resources.pipelines.*.clusters[*].driver_node_type_id string INPUT STATE resources.pipelines.*.clusters[*].enable_local_disk_encryption bool INPUT STATE @@ -1995,13 +1995,13 @@ resources.pipelines.*.clusters[*].node_type_id string INPUT STATE resources.pipelines.*.clusters[*].num_workers int INPUT STATE resources.pipelines.*.clusters[*].policy_id string INPUT STATE resources.pipelines.*.clusters[*].spark_conf map[string]string INPUT STATE -resources.pipelines.*.clusters[*].spark_conf[*] string INPUT STATE +resources.pipelines.*.clusters[*].spark_conf.* string INPUT STATE resources.pipelines.*.clusters[*].spark_env_vars map[string]string INPUT STATE -resources.pipelines.*.clusters[*].spark_env_vars[*] string INPUT STATE +resources.pipelines.*.clusters[*].spark_env_vars.* string INPUT STATE resources.pipelines.*.clusters[*].ssh_public_keys []string INPUT STATE resources.pipelines.*.clusters[*].ssh_public_keys[*] string INPUT STATE resources.pipelines.*.configuration map[string]string INPUT STATE -resources.pipelines.*.configuration[*] string INPUT STATE +resources.pipelines.*.configuration.* string INPUT STATE resources.pipelines.*.continuous bool INPUT STATE resources.pipelines.*.creator_user_name string REMOTE resources.pipelines.*.deployment *pipelines.PipelineDeployment INPUT STATE @@ -2220,7 +2220,7 @@ resources.pipelines.*.spec.clusters[*].cluster_log_conf.s3.region string REMOTE resources.pipelines.*.spec.clusters[*].cluster_log_conf.volumes *compute.VolumesStorageInfo REMOTE resources.pipelines.*.spec.clusters[*].cluster_log_conf.volumes.destination string REMOTE resources.pipelines.*.spec.clusters[*].custom_tags map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].custom_tags[*] string REMOTE +resources.pipelines.*.spec.clusters[*].custom_tags.* string REMOTE resources.pipelines.*.spec.clusters[*].driver_instance_pool_id string REMOTE resources.pipelines.*.spec.clusters[*].driver_node_type_id string REMOTE resources.pipelines.*.spec.clusters[*].enable_local_disk_encryption bool REMOTE @@ -2260,13 +2260,13 @@ resources.pipelines.*.spec.clusters[*].node_type_id string REMOTE resources.pipelines.*.spec.clusters[*].num_workers int REMOTE resources.pipelines.*.spec.clusters[*].policy_id string REMOTE resources.pipelines.*.spec.clusters[*].spark_conf map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].spark_conf[*] string REMOTE +resources.pipelines.*.spec.clusters[*].spark_conf.* string REMOTE resources.pipelines.*.spec.clusters[*].spark_env_vars map[string]string REMOTE -resources.pipelines.*.spec.clusters[*].spark_env_vars[*] string REMOTE +resources.pipelines.*.spec.clusters[*].spark_env_vars.* string REMOTE resources.pipelines.*.spec.clusters[*].ssh_public_keys []string REMOTE resources.pipelines.*.spec.clusters[*].ssh_public_keys[*] string REMOTE resources.pipelines.*.spec.configuration map[string]string REMOTE -resources.pipelines.*.spec.configuration[*] string REMOTE +resources.pipelines.*.spec.configuration.* string REMOTE resources.pipelines.*.spec.continuous bool REMOTE resources.pipelines.*.spec.deployment *pipelines.PipelineDeployment REMOTE resources.pipelines.*.spec.deployment.kind pipelines.DeploymentKind REMOTE @@ -2420,7 +2420,7 @@ resources.pipelines.*.spec.schema string REMOTE resources.pipelines.*.spec.serverless bool REMOTE resources.pipelines.*.spec.storage string REMOTE resources.pipelines.*.spec.tags map[string]string REMOTE -resources.pipelines.*.spec.tags[*] string REMOTE +resources.pipelines.*.spec.tags.* string REMOTE resources.pipelines.*.spec.target string REMOTE resources.pipelines.*.spec.trigger *pipelines.PipelineTrigger REMOTE resources.pipelines.*.spec.trigger.cron *pipelines.CronTrigger REMOTE @@ -2430,7 +2430,7 @@ resources.pipelines.*.spec.trigger.manual *pipelines.ManualTrigger REMOTE resources.pipelines.*.state pipelines.PipelineState REMOTE resources.pipelines.*.storage string INPUT STATE resources.pipelines.*.tags map[string]string INPUT STATE -resources.pipelines.*.tags[*] string INPUT STATE +resources.pipelines.*.tags.* string INPUT STATE resources.pipelines.*.target string INPUT STATE resources.pipelines.*.trigger *pipelines.PipelineTrigger INPUT STATE resources.pipelines.*.trigger.cron *pipelines.CronTrigger INPUT STATE @@ -2463,7 +2463,7 @@ resources.schemas.*.modified_status string INPUT resources.schemas.*.name string ALL resources.schemas.*.owner string REMOTE resources.schemas.*.properties map[string]string ALL -resources.schemas.*.properties[*] string ALL +resources.schemas.*.properties.* string ALL resources.schemas.*.schema_id string REMOTE resources.schemas.*.storage_location string REMOTE resources.schemas.*.storage_root string ALL @@ -2483,7 +2483,7 @@ resources.sql_warehouses.*.health.details string REMOTE resources.sql_warehouses.*.health.failure_reason *sql.TerminationReason REMOTE resources.sql_warehouses.*.health.failure_reason.code sql.TerminationReasonCode REMOTE resources.sql_warehouses.*.health.failure_reason.parameters map[string]string REMOTE -resources.sql_warehouses.*.health.failure_reason.parameters[*] string REMOTE +resources.sql_warehouses.*.health.failure_reason.parameters.* string REMOTE resources.sql_warehouses.*.health.failure_reason.type sql.TerminationReasonType REMOTE resources.sql_warehouses.*.health.message string REMOTE resources.sql_warehouses.*.health.status sql.Status REMOTE diff --git a/bundle/config/mutator/log_resource_references.go b/bundle/config/mutator/log_resource_references.go index 546293ffde..14c6beed49 100644 --- a/bundle/config/mutator/log_resource_references.go +++ b/bundle/config/mutator/log_resource_references.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" ) // Longest field name: @@ -115,7 +116,14 @@ func truncate(s string, n int, suffix string) string { } func censorValue(ctx context.Context, v any, path dyn.Path) (string, error) { - v, err := structaccess.Get(v, path) + pathString := path.String() + pathNode, err := structpath.Parse(pathString) + if err != nil { + log.Warnf(ctx, "internal error: parsing %q: %s", pathString, err) + return "err", err + } + + v, err = structaccess.Get(v, pathNode) if err != nil { log.Infof(ctx, "internal error: path=%s: %s", path, err) return "err", err diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 18af7bf4e9..9ffd43329a 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/deployplan" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structwalk" @@ -155,9 +154,9 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) require.NoError(t, structwalk.Walk(newState, func(path *structpath.PathNode, val any, field *reflect.StructField) { - remoteValue, err := structaccess.Get(remappedState, dyn.MustPathFromString(path.DynPath())) + remoteValue, err := structaccess.Get(remappedState, path) if err != nil { - t.Errorf("Failed to read %s from remapped remote state %#v", path.DynPath(), remappedState) + t.Errorf("Failed to read %s from remapped remote state %#v", path.String(), remappedState) } if val == nil { // t.Logf("Ignoring %s nil, remoteValue=%#v", path.String(), remoteValue) @@ -172,7 +171,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W // t.Logf("Testing %s v=%#v, remoteValue=%#v", path.String(), val, remoteValue) // We expect fields set explicitly to be preserved by testserver, which is true for all resources as of today. // If not true for your resource, add exception here: - assert.Equal(t, val, remoteValue, path.DynPath()) + assert.Equal(t, val, remoteValue, path.String()) })) err = adapter.DoDelete(ctx, createdID) diff --git a/bundle/direct/plan.go b/bundle/direct/plan.go index bff9cf92d0..6b051068b7 100644 --- a/bundle/direct/plan.go +++ b/bundle/direct/plan.go @@ -9,13 +9,12 @@ import ( "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" - "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/databricks-sdk-go" - "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" ) func (d *DeploymentUnit) Plan(ctx context.Context, client *databricks.WorkspaceClient, db *dstate.DeploymentState, inputConfig any, localOnly, refresh bool) (deployplan.ActionType, error) { @@ -68,12 +67,12 @@ func (d *DeploymentUnit) refreshRemoteState(ctx context.Context, id string) erro var ErrDelayed = errors.New("must be resolved after apply") func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db *dstate.DeploymentState, reference string, actionType deployplan.ActionType, config any) (any, error) { - path, ok := dynvar.PureReferenceToPath(reference) - if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey { + path, ok := structpath.PureReferenceToPath(reference) + if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey { return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference) } - fieldPath := path[3:] + fieldPath := path.SkipPrefix(3) if fieldPath.String() == "id" { if actionType.KeepsID() { @@ -136,12 +135,12 @@ func (d *DeploymentUnit) ResolveReferenceLocalOrRemote(ctx context.Context, db * } func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate.DeploymentState, reference string) (any, error) { - path, ok := dynvar.PureReferenceToPath(reference) - if !ok || len(path) <= 3 || path[0:3].String() != d.ResourceKey { + path, ok := structpath.PureReferenceToPath(reference) + if !ok || path.Len() <= 3 || path.Prefix(3).String() != d.ResourceKey { return nil, fmt.Errorf("internal error: expected reference to %q, got %q", d.ResourceKey, reference) } - fieldPath := path[3:] + fieldPath := path.SkipPrefix(3) // Handle "id" field separately - read from state, not remote state if fieldPath.String() == "id" { @@ -157,7 +156,7 @@ func (d *DeploymentUnit) ResolveReferenceRemote(ctx context.Context, db *dstate. } // ReadRemoteStateField reads a field from remote state with refresh if needed. -func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath dyn.Path) (any, error) { +func (d *DeploymentUnit) ReadRemoteStateField(ctx context.Context, db *dstate.DeploymentState, fieldPath *structpath.PathNode) (any, error) { // We have options: // 1) Rely on the cached value; refresh if not cached. // 2) Always refresh, read the value. diff --git a/bundle/internal/validation/enum.go b/bundle/internal/validation/enum.go index cd5bfc2044..4b1704da97 100644 --- a/bundle/internal/validation/enum.go +++ b/bundle/internal/validation/enum.go @@ -132,8 +132,7 @@ func extractEnumFields(typ reflect.Type) ([]EnumPatternInfo, error) { return true } - fieldPath := path.DynPath() - fieldsByPattern[fieldPath] = enumValues + fieldsByPattern[path.String()] = enumValues } return true }) diff --git a/bundle/internal/validation/required.go b/bundle/internal/validation/required.go index 72e8f15fab..e0fd6270c3 100644 --- a/bundle/internal/validation/required.go +++ b/bundle/internal/validation/required.go @@ -63,12 +63,12 @@ func extractRequiredFields(typ reflect.Type) ([]RequiredPatternInfo, error) { return true } - fieldName, ok := path.Field() + fieldName, ok := path.StringKey() if !ok { return true } - parentPath := path.Parent().DynPath() + parentPath := path.Parent().String() fieldsByPattern[parentPath] = append(fieldsByPattern[parentPath], fieldName) return true }) diff --git a/libs/structs/structaccess/bundle_test.go b/libs/structs/structaccess/bundle_test.go index 27b868e9bd..a93ff2be95 100644 --- a/libs/structs/structaccess/bundle_test.go +++ b/libs/structs/structaccess/bundle_test.go @@ -52,14 +52,6 @@ func TestGet_ConfigRoot_JobTagsAccess(t *testing.T) { require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags.env.inner")) require.Error(t, ValidateByString(reflect.TypeOf(root), "resources.jobs.my_job.tags1")) - // Leading dot is allowed - v, err = GetByString(root, ".resources.jobs.my_job.tags.team") - require.NoError(t, err) - require.Equal(t, "platform", v) - require.NoError(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team")) - require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags.team.inner")) - require.Error(t, ValidateByString(reflect.TypeOf(root), ".resources.jobs.my_job.tags1")) - // Array indexing test (1) v, err = GetByString(root, "resources.jobs.my_job.tasks[0].task_key") require.NoError(t, err) diff --git a/libs/structs/structaccess/get.go b/libs/structs/structaccess/get.go index 4e298e24fe..18ba74082a 100644 --- a/libs/structs/structaccess/get.go +++ b/libs/structs/structaccess/get.go @@ -1,87 +1,74 @@ package structaccess import ( + "errors" "fmt" "reflect" - "strconv" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" ) // GetByString returns the value at the given path inside v. // This is a convenience function that parses the path string and calls Get. -// -// Path grammar (compatible with dyn path): -// - Struct field names and map keys separated by '.' (e.g., connection.id) -// - (Note, this prevents maps keys that are not id-like from being referenced, but this general problem with references today.) -// - Numeric indices in brackets for arrays/slices (e.g., items[0].name) -// - Leading '.' is allowed (e.g., .connection.id) -// -// Behavior: -// - For structs: a key matches a field by its json tag name (if present and not "-"). -// Embedded anonymous structs are searched. -// - For maps: a key indexes map[string]T (or string alias key types). -// - For slices/arrays: an index [N] selects the N-th element. -// - Wildcards ("*" or "[*]") are not supported and return an error. func GetByString(v any, path string) (any, error) { if path == "" { return v, nil } - dynPath, err := dyn.NewPathFromString(path) + pathNode, err := structpath.Parse(path) if err != nil { return nil, err } - return Get(v, dynPath) + return Get(v, pathNode) } // Get returns the value at the given path inside v. -func Get(v any, path dyn.Path) (any, error) { - if len(path) == 0 { +// Wildcards ("*" or "[*]") are not supported and return an error. +func Get(v any, path *structpath.PathNode) (any, error) { + if path.IsRoot() { return v, nil } + // Convert path to slice for easier iteration + pathSegments := path.AsSlice() + cur := reflect.ValueOf(v) - prefix := "" - for _, c := range path { + for _, node := range pathSegments { var ok bool cur, ok = deref(cur) if !ok { // cannot proceed further due to nil encountered at current location - return nil, fmt.Errorf("%s: cannot access nil value", prefix) + return nil, fmt.Errorf("%s: cannot access nil value", node.Parent().String()) } - if c.Key() != "" { - // Key access: struct field (by json tag) or map key. - newPrefix := prefix - if newPrefix == "" { - newPrefix = c.Key() - } else { - newPrefix = newPrefix + "." + c.Key() + if idx, isIndex := node.Index(); isIndex { + kind := cur.Kind() + if kind != reflect.Slice && kind != reflect.Array { + return nil, fmt.Errorf("%s: cannot index %s", node.String(), kind) } - nv, err := accessKey(cur, c.Key(), newPrefix) - if err != nil { - return nil, err + if idx < 0 || idx >= cur.Len() { + return nil, fmt.Errorf("%s: index out of range, length is %d", node.String(), cur.Len()) } - cur = nv - prefix = newPrefix + cur = cur.Index(idx) continue } - // Index access: slice/array - idx := c.Index() - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" - kind := cur.Kind() - if kind != reflect.Slice && kind != reflect.Array { - return nil, fmt.Errorf("%s: cannot index %s", newPrefix, kind) + if node.DotStar() || node.BracketStar() { + return nil, fmt.Errorf("wildcards not supported: %s", path.String()) } - if idx < 0 || idx >= cur.Len() { - return nil, fmt.Errorf("%s: index out of range, length is %d", newPrefix, cur.Len()) + + key, ok := node.StringKey() + if !ok { + return nil, errors.New("unsupported path node type") + } + + nv, err := accessKey(cur, key, node) + if err != nil { + return nil, err } - cur = cur.Index(idx) - prefix = newPrefix + cur = nv } // If the current value is invalid (e.g., omitted due to omitempty), return nil. @@ -100,12 +87,12 @@ func Get(v any, path dyn.Path) (any, error) { // accessKey returns the field or map entry value selected by key from v. // v must be non-pointer, non-interface reflect.Value. -func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { +func accessKey(v reflect.Value, key string, path *structpath.PathNode) (reflect.Value, error) { switch v.Kind() { case reflect.Struct: fv, sf, owner, ok := findStructFieldByKey(v, key) if !ok { - return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", prefix, key, v.Type()) + return reflect.Value{}, fmt.Errorf("%s: field %q not found in %s", path.String(), key, v.Type()) } // Evaluate ForceSendFields on both the current struct and the declaring owner force := containsForceSendField(v, sf.Name) || containsForceSendField(owner, sf.Name) @@ -129,7 +116,7 @@ func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { case reflect.Map: kt := v.Type().Key() if kt.Kind() != reflect.String { - return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", prefix, kt) + return reflect.Value{}, fmt.Errorf("%s: map key must be string, got %s", path.String(), kt) } mk := reflect.ValueOf(key) if kt != mk.Type() { @@ -137,11 +124,11 @@ func accessKey(v reflect.Value, key, prefix string) (reflect.Value, error) { } mv := v.MapIndex(mk) if !mv.IsValid() { - return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", prefix, key) + return reflect.Value{}, fmt.Errorf("%s: key %q not found in map", path.String(), key) } return mv, nil default: - return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", prefix, key, v.Kind()) + return reflect.Value{}, fmt.Errorf("%s: cannot access key %q on %s", path.String(), key, v.Kind()) } } diff --git a/libs/structs/structaccess/get_test.go b/libs/structs/structaccess/get_test.go index 70dd6ed803..dec25b87b8 100644 --- a/libs/structs/structaccess/get_test.go +++ b/libs/structs/structaccess/get_test.go @@ -138,10 +138,15 @@ func runCommonTests(t *testing.T, obj any) { want: "bar", }, { - name: "leading dot allowed", - path: ".connection.id", + name: "struct field with bracket notation", + path: "['connection']['id']", want: "abc", }, + { + name: "map key with bracket notation", + path: "labels['env']", + want: "dev", + }, // Regular scalar fields - always return zero values { @@ -169,7 +174,7 @@ func runCommonTests(t *testing.T, obj any) { { name: "wildcard not supported", path: "items[*].id", - errFmt: "invalid path: items[*].id", + errFmt: "wildcards not supported: items[*].id", }, { name: "missing field", diff --git a/libs/structs/structaccess/typecheck.go b/libs/structs/structaccess/typecheck.go index a7b804c748..3f851571be 100644 --- a/libs/structs/structaccess/typecheck.go +++ b/libs/structs/structaccess/typecheck.go @@ -1,11 +1,11 @@ package structaccess import ( + "errors" "fmt" "reflect" - "strconv" - "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/cli/libs/structs/structtag" ) @@ -17,77 +17,82 @@ func ValidateByString(t reflect.Type, path string) error { return nil } - p, err := dyn.NewPathFromString(path) + pathNode, err := structpath.Parse(path) if err != nil { return err } - return Validate(t, p) + return Validate(t, pathNode) } // Validate reports whether the given path is valid for the provided type. // It returns nil if the path resolves fully, or an error indicating where resolution failed. -func Validate(t reflect.Type, path dyn.Path) error { - if len(path) == 0 { +func Validate(t reflect.Type, path *structpath.PathNode) error { + if path.IsRoot() { return nil } + // Convert path to slice for easier iteration + pathSegments := path.AsSlice() + cur := t - prefix := "" - for _, c := range path { + for _, node := range pathSegments { // Always dereference pointers at the type level. for cur.Kind() == reflect.Pointer { cur = cur.Elem() } - if c.Key() != "" { - // Key access: struct field (by json tag) or map key. - newPrefix := prefix - if newPrefix == "" { - newPrefix = c.Key() - } else { - newPrefix = newPrefix + "." + c.Key() - } - - switch cur.Kind() { - case reflect.Struct: - sf, _, ok := findStructFieldByKeyType(cur, c.Key()) - if !ok { - return fmt.Errorf("%s: field %q not found in %s", newPrefix, c.Key(), cur.String()) - } - cur = sf.Type - case reflect.Map: - kt := cur.Key() - if kt.Kind() != reflect.String { - return fmt.Errorf("%s: map key must be string, got %s", newPrefix, kt) - } - cur = cur.Elem() - default: - return fmt.Errorf("%s: cannot access key %q on %s", newPrefix, c.Key(), cur.Kind()) + if _, isIndex := node.Index(); isIndex { + // Index access: slice/array + kind := cur.Kind() + if kind != reflect.Slice && kind != reflect.Array { + return fmt.Errorf("%s: cannot index %s", node.String(), kind) } - prefix = newPrefix + cur = cur.Elem() continue } - // Index access: slice/array - idx := c.Index() - newPrefix := prefix + "[" + strconv.Itoa(idx) + "]" - kind := cur.Kind() - if kind != reflect.Slice && kind != reflect.Array { - return fmt.Errorf("%s: cannot index %s", newPrefix, kind) + // Handle wildcards + if node.DotStar() || node.BracketStar() { + return fmt.Errorf("wildcards not supported: %s", path.String()) + } + + key, ok := node.StringKey() + + if !ok { + return errors.New("unsupported path node type") + } + + switch cur.Kind() { + case reflect.Struct: + sf, _, ok := FindStructFieldByKeyType(cur, key) + if !ok { + return fmt.Errorf("%s: field %q not found in %s", node.String(), key, cur.String()) + } + cur = sf.Type + case reflect.Map: + kt := cur.Key() + if kt.Kind() != reflect.String { + return fmt.Errorf("%s: map key must be string, got %s", node.String(), kt) + } + cur = cur.Elem() + default: + return fmt.Errorf("%s: cannot access key %q on %s", node.String(), key, cur.Kind()) } - cur = cur.Elem() - prefix = newPrefix } return nil } -// findStructFieldByKeyType searches exported fields of struct type t for a field matching key. +// FindStructFieldByKeyType searches exported fields of struct type t for a field matching key. // It matches json tag name (when present and not "-") only. // It also searches embedded anonymous structs (pointer or value) recursively. // Returns the StructField, the declaring owner type, and whether it was found. -func findStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, reflect.Type, bool) { +func FindStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, reflect.Type, bool) { + if t.Kind() != reflect.Struct { + return reflect.StructField{}, reflect.TypeOf(nil), false + } + // First pass: direct fields for i := range t.NumField() { sf := t.Field(i) @@ -121,7 +126,7 @@ func findStructFieldByKeyType(t reflect.Type, key string) (reflect.StructField, if ft.Kind() != reflect.Struct { continue } - if osf, owner, ok := findStructFieldByKeyType(ft, key); ok { + if osf, owner, ok := FindStructFieldByKeyType(ft, key); ok { // Skip fields marked as internal/readonly btag := structtag.BundleTag(osf.Tag.Get("bundle")) if btag.Internal() || btag.ReadOnly() { diff --git a/libs/structs/structdiff/diff.go b/libs/structs/structdiff/diff.go index 473e8b26de..91be5a05c4 100644 --- a/libs/structs/structdiff/diff.go +++ b/libs/structs/structdiff/diff.go @@ -127,7 +127,15 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) + jsonTag := structtag.JSONTag(sf.Tag.Get("json")) + + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTag.Name() + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStringKey(path, fieldName) + v1Field := s1.Field(i) v2Field := s2.Field(i) @@ -135,7 +143,6 @@ func diffStruct(path *structpath.PathNode, s1, s2 reflect.Value, changes *[]Chan zero2 := v2Field.IsZero() if zero1 || zero2 { - jsonTag := structtag.JSONTag(sf.Tag.Get("json")) if jsonTag.OmitEmpty() { if zero1 { if !slices.Contains(forced1, sf.Name) { @@ -176,7 +183,7 @@ func diffMapStringKey(path *structpath.PathNode, m1, m2 reflect.Value, changes * k := keySet[ks] v1 := m1.MapIndex(k) v2 := m2.MapIndex(k) - node := structpath.NewMapKey(path, ks) + node := structpath.NewStringKey(path, ks) diffValues(node, v1, v2, changes) } } diff --git a/libs/structs/structdiff/diff_test.go b/libs/structs/structdiff/diff_test.go index 794e0954ff..3c69eefdcd 100644 --- a/libs/structs/structdiff/diff_test.go +++ b/libs/structs/structdiff/diff_test.go @@ -132,7 +132,7 @@ func TestGetStructDiff(t *testing.T) { name: "map diff", a: A{M: map[string]int{"a": 1}}, b: A{M: map[string]int{"a": 2}}, - want: []ResolvedChange{{Field: "m['a']", Old: 1, New: 2}}, + want: []ResolvedChange{{Field: "m.a", Old: 1, New: 2}}, }, { name: "slice diff", @@ -243,9 +243,9 @@ func TestGetStructDiff(t *testing.T) { a: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}}, b: map[string]C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}}, want: []ResolvedChange{ - {Field: "['key1'].name", Old: "", New: nil}, - {Field: "['key1'].age", Old: nil, New: 0}, - {Field: "['key1'].is_enabled", Old: false, New: nil}, + {Field: "key1.name", Old: "", New: nil}, + {Field: "key1.age", Old: nil, New: 0}, + {Field: "key1.is_enabled", Old: false, New: nil}, }, }, @@ -255,9 +255,9 @@ func TestGetStructDiff(t *testing.T) { a: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Name", "IsEnabled", "Title"}}}, b: map[string]*C{"key1": {Title: "title", ForceSendFields: []string{"Age"}}}, want: []ResolvedChange{ - {Field: "['key1'].name", Old: "", New: nil}, - {Field: "['key1'].age", Old: nil, New: 0}, - {Field: "['key1'].is_enabled", Old: false, New: nil}, + {Field: "key1.name", Old: "", New: nil}, + {Field: "key1.age", Old: nil, New: 0}, + {Field: "key1.is_enabled", Old: false, New: nil}, }, }, } diff --git a/libs/structs/structpath/path.go b/libs/structs/structpath/path.go index dd12b62972..a656565b2b 100644 --- a/libs/structs/structpath/path.go +++ b/libs/structs/structpath/path.go @@ -3,18 +3,22 @@ package structpath import ( "errors" "fmt" - "reflect" "strconv" "strings" - "github.com/databricks/cli/libs/structs/structtag" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/dynvar" ) const ( - tagStruct = -1 - tagMapKey = -2 - tagAnyKey = -4 - tagAnyIndex = -5 + // Encodes string key, which is encoded as .field or as ['spark.conf'] + tagStringKey = -1 + + // Encodes wildcard after a dot: foo.* + tagDotStar = -2 + + // Encodes wildcard in brackets: foo[*] + tagBracketStar = -3 ) // PathNode represents a node in a path for struct diffing. @@ -23,7 +27,7 @@ type PathNode struct { prev *PathNode key string // Computed key (JSON key for structs, string key for maps, or Go field name for fallback) // If index >= 0, the node specifies a slice/array index in index. - // If index < 0, this describes the type of node (see tagStruct and other consts above) + // If index < 0, this describes the type of node index int } @@ -41,35 +45,26 @@ func (p *PathNode) Index() (int, bool) { return -1, false } -func (p *PathNode) MapKey() (string, bool) { - if p == nil { - return "", false - } - if p.index == tagMapKey { - return p.key, true - } - return "", false -} - -func (p *PathNode) AnyKey() bool { +func (p *PathNode) DotStar() bool { if p == nil { return false } - return p.index == tagAnyKey + return p.index == tagDotStar } -func (p *PathNode) AnyIndex() bool { +func (p *PathNode) BracketStar() bool { if p == nil { return false } - return p.index == tagAnyIndex + return p.index == tagBracketStar } -func (p *PathNode) Field() (string, bool) { +// StringKey returns either Field() or MapKey() if either is available +func (p *PathNode) StringKey() (string, bool) { if p == nil { return "", false } - if p.index == tagStruct { + if p.index == tagStringKey { return p.key, true } return "", false @@ -82,10 +77,26 @@ func (p *PathNode) Parent() *PathNode { return p.prev } +// AsSlice returns the path as a slice of PathNodes from root to current. +// Efficiently pre-allocates the exact length and fills in reverse order. +func (p *PathNode) AsSlice() []*PathNode { + length := p.Len() + segments := make([]*PathNode, length) + + // Fill in reverse order + current := p + for i := length - 1; i >= 0; i-- { + segments[i] = current + current = current.Parent() + } + + return segments +} + // NewIndex creates a new PathNode for an array/slice index. func NewIndex(prev *PathNode, index int) *PathNode { if index < 0 { - panic("index msut be non-negative") + panic("index must be non-negative") } return &PathNode{ prev: prev, @@ -93,48 +104,33 @@ func NewIndex(prev *PathNode, index int) *PathNode { } } -// NewMapKey creates a new PathNode for a map key. -func NewMapKey(prev *PathNode, key string) *PathNode { - return &PathNode{ - prev: prev, - key: key, - index: tagMapKey, - } -} - -// NewStructField creates a new PathNode for a struct field. -// The jsonTag is used for JSON key resolution, and fieldName is used as fallback. -func NewStructField(prev *PathNode, tag reflect.StructTag, fieldName string) *PathNode { - jsonTag := structtag.JSONTag(tag.Get("json")) - - key := fieldName - if name := jsonTag.Name(); name != "" { - key = name - } - +// NewStringKey creates either StructField or MapKey +// The fieldName should be the resolved field name (e.g., from JSON tag or Go field name). +func NewStringKey(prev *PathNode, fieldName string) *PathNode { return &PathNode{ prev: prev, - key: key, - index: tagStruct, + key: fieldName, + index: tagStringKey, } } -func NewAnyKey(prev *PathNode) *PathNode { +func NewDotStar(prev *PathNode) *PathNode { return &PathNode{ prev: prev, - index: tagAnyKey, + index: tagDotStar, } } -func NewAnyIndex(prev *PathNode) *PathNode { +func NewBracketStar(prev *PathNode) *PathNode { return &PathNode{ prev: prev, - index: tagAnyIndex, + index: tagBracketStar, } } // String returns the string representation of the path. -// The map keys are encoded in single quotes: tags['name']. Single quote can escaped by placing two single quotes: tags[””] (map key is one single quote). +// The string keys are encoded in dot syntax (foo.bar) if they don't have any reserved characters (so can be parsed as fields). +// Otherwise they are encoded in brackets + single quotes: tags['name']. Single quote can escaped by placing two single quotes. // This encoding is chosen over traditional double quotes because when encoded in JSON it does not need to be escaped: // // { @@ -145,25 +141,45 @@ func (p *PathNode) String() string { return "" } - if p.index >= 0 { - return p.prev.String() + "[" + strconv.Itoa(p.index) + "]" - } + // Get all path components from root to current + components := p.AsSlice() - if p.index == tagAnyKey || p.index == tagAnyIndex { - return p.prev.String() + "[*]" - } + var result strings.Builder - if p.index == tagStruct { - prev := p.prev.String() - if prev == "" { - return p.key + for i, node := range components { + if node.index >= 0 { + // Array/slice index + result.WriteString("[") + result.WriteString(strconv.Itoa(node.index)) + result.WriteString("]") + } else if node.index == tagDotStar { + if i == 0 { + result.WriteString("*") + } else { + result.WriteString(".*") + } + } else if node.index == tagBracketStar { + result.WriteString("[*]") + } else if isValidField(node.key) { + // Valid field name + if i != 0 { + result.WriteString(".") + } + result.WriteString(node.key) + } else { + // Map key with single quotes + result.WriteString("[") + result.WriteString(EncodeMapKey(node.key)) + result.WriteString("]") } - return prev + "." + p.key } - // Format map key with single quotes, escaping single quotes by doubling them - escapedKey := strings.ReplaceAll(p.key, "'", "''") - return fmt.Sprintf("%s['%s']", p.prev.String(), escapedKey) + return result.String() +} + +func EncodeMapKey(s string) string { + escaped := strings.ReplaceAll(s, "'", "''") + return "'" + escaped + "'" } // Parse parses a string representation of a path using a state machine. @@ -171,9 +187,10 @@ func (p *PathNode) String() string { // State Machine for Path Parsing: // // States: -// - START: Beginning of parsing, expects field name or "[" -// - FIELD_START: After a dot, expects field name only +// - START: Beginning of parsing, expects field name, "[", or "*" +// - FIELD_START: After a dot, expects field name or "*" // - FIELD: Reading field name characters +// - DOT_STAR: Encountered "*" (at start or after dot), expects ".", "[", or EOF // - BRACKET_OPEN: Just encountered "[", expects digit, "'" or "*" // - INDEX: Reading array index digits, expects more digits or "]" // - MAP_KEY: Reading map key content, expects any char or "'" @@ -183,9 +200,10 @@ func (p *PathNode) String() string { // - END: Successfully completed parsing // // Transitions: -// - START: [a-zA-Z_-] -> FIELD, "[" -> BRACKET_OPEN, EOF -> END -// - FIELD_START: [a-zA-Z_-] -> FIELD, other -> ERROR +// - START: [a-zA-Z_-] -> FIELD, "[" -> BRACKET_OPEN, "*" -> DOT_STAR, EOF -> END +// - FIELD_START: [a-zA-Z_-] -> FIELD, "*" -> DOT_STAR, other -> ERROR // - FIELD: [a-zA-Z0-9_-] -> FIELD, "." -> FIELD_START, "[" -> BRACKET_OPEN, EOF -> END +// - DOT_STAR: "." -> FIELD_START, "[" -> BRACKET_OPEN, EOF -> END, other -> ERROR // - BRACKET_OPEN: [0-9] -> INDEX, "'" -> MAP_KEY, "*" -> WILDCARD // - INDEX: [0-9] -> INDEX, "]" -> EXPECT_DOT_OR_END // - MAP_KEY: (any except "'") -> MAP_KEY, "'" -> MAP_KEY_QUOTE @@ -202,6 +220,7 @@ func Parse(s string) (*PathNode, error) { stateStart = iota stateFieldStart stateField + stateDotStar stateBracketOpen stateIndex stateMapKey @@ -223,6 +242,8 @@ func Parse(s string) (*PathNode, error) { case stateStart: if ch == '[' { state = stateBracketOpen + } else if ch == '*' { + state = stateDotStar } else if !isReservedFieldChar(ch) { currentToken.WriteByte(ch) state = stateField @@ -231,7 +252,9 @@ func Parse(s string) (*PathNode, error) { } case stateFieldStart: - if !isReservedFieldChar(ch) { + if ch == '*' { + state = stateDotStar + } else if !isReservedFieldChar(ch) { currentToken.WriteByte(ch) state = stateField } else { @@ -240,11 +263,11 @@ func Parse(s string) (*PathNode, error) { case stateField: if ch == '.' { - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateFieldStart } else if ch == '[' { - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateBracketOpen } else if !isReservedFieldChar(ch) { @@ -253,6 +276,18 @@ func Parse(s string) (*PathNode, error) { return nil, fmt.Errorf("invalid character '%c' in field name at position %d", ch, pos) } + case stateDotStar: + switch ch { + case '.': + result = NewDotStar(result) + state = stateFieldStart + case '[': + result = NewDotStar(result) + state = stateBracketOpen + default: + return nil, fmt.Errorf("unexpected character '%c' after '.*' at position %d", ch, pos) + } + case stateBracketOpen: if ch >= '0' && ch <= '9' { currentToken.WriteByte(ch) @@ -296,7 +331,7 @@ func Parse(s string) (*PathNode, error) { state = stateMapKey case ']': // End of map key - result = NewMapKey(result, currentToken.String()) + result = NewStringKey(result, currentToken.String()) currentToken.Reset() state = stateExpectDotOrEnd default: @@ -305,9 +340,7 @@ func Parse(s string) (*PathNode, error) { case stateWildcard: if ch == ']' { - // Note, since we're parsing this without type info present, we don't know if it's AnyKey or AnyIndex - // Perhaps structpath should be simplified to have Wildcard as merged representation of AnyKey/AnyIndex - result = NewAnyKey(result) + result = NewBracketStar(result) state = stateExpectDotOrEnd } else { return nil, fmt.Errorf("unexpected character '%c' after '*' at position %d", ch, pos) @@ -338,7 +371,10 @@ func Parse(s string) (*PathNode, error) { case stateStart: return result, nil // Empty path, result is nil case stateField: - result = NewStructField(result, reflect.StructTag(""), currentToken.String()) + result = NewStringKey(result, currentToken.String()) + return result, nil + case stateDotStar: + result = NewDotStar(result) return result, nil case stateExpectDotOrEnd: return result, nil @@ -376,39 +412,120 @@ func isReservedFieldChar(ch byte) bool { return true case ']': // Bracket notation end return true + case '\'': + return true + case ' ': + return true + case '}': + return true + case '{': + return true default: return false } } -// Path in libs/dyn format -func (p *PathNode) DynPath() string { - if p == nil { - return "" +func isValidField(s string) bool { + for ind := range s { + if isReservedFieldChar(s[ind]) { + return false + } } + return len(s) > 0 +} - if p.index >= 0 { - return p.prev.DynPath() + "[" + strconv.Itoa(p.index) + "]" +// PureReferenceToPath returns a PathNode if s is a pure variable reference, otherwise false. +// This function is similar to dynvar.PureReferenceToPath but returns a *PathNode instead of dyn.Path. +func PureReferenceToPath(s string) (*PathNode, bool) { + ref, ok := dynvar.NewRef(dyn.V(s)) + if !ok { + return nil, false } - if p.index == tagAnyKey { - prev := p.prev.DynPath() - if prev == "" { - return "*" - } else { - return prev + ".*" + if !ref.IsPure() { + return nil, false + } + + pathNode, err := Parse(ref.References()[0]) + if err != nil { + return nil, false + } + + return pathNode, true +} + +// SkipPrefix returns a new PathNode that skips the first n components of the path. +// If n is greater than or equal to the path length, returns nil (root). +func (p *PathNode) SkipPrefix(n int) *PathNode { + if p.IsRoot() || n <= 0 { + return p + } + + length := p.Len() + if n >= length { + return nil // Return root + } + + startNode := p.Prefix(n) + + var result *PathNode + current := p + for current != startNode { + result = &PathNode{ + prev: result, + key: current.key, + index: current.index, } + current = current.Parent() } - if p.index == tagAnyIndex { - return p.prev.DynPath() + "[*]" + return result.ReverseInPlace() +} + +// ReverseInPlace returns a new PathNode with the order of components reversed. +func (p *PathNode) ReverseInPlace() *PathNode { + var result *PathNode + current := p + for current != nil { + next := current.prev + current.prev = result + result = current + current = next } + return result +} - // Both struct fields and map keys use dot notation in DynPath - prev := p.prev.DynPath() - if prev == "" { - return p.key - } else { - return prev + "." + p.key +// Len returns the number of components in the path. +func (p *PathNode) Len() int { + length := 0 + current := p + for current != nil { + length++ + current = current.Parent() } + return length +} + +// Prefix returns the PathNode at the nth position (1-indexed from root). +// If n is greater than the path length, returns the entire path. +// If n <= 0, returns nil (root). +func (p *PathNode) Prefix(n int) *PathNode { + if p.IsRoot() || n <= 0 { + return nil // Return root + } + + // Find the path length first to handle edge cases + length := p.Len() + if n >= length { + return p // Return entire path + } + + // Traverse from root to find the nth node (1-indexed) + current := p + // Move to root first + for range length - n { + current = current.Parent() + } + + return current } diff --git a/libs/structs/structpath/path_test.go b/libs/structs/structpath/path_test.go index a2be306a94..a9e19501dc 100644 --- a/libs/structs/structpath/path_test.go +++ b/libs/structs/structpath/path_test.go @@ -1,7 +1,6 @@ package structpath import ( - "reflect" "testing" "github.com/stretchr/testify/assert" @@ -9,17 +8,14 @@ import ( func TestPathNode(t *testing.T) { tests := []struct { - name string - node *PathNode - String string - DynPath string // Only set when different from String - IgnoreDynPath bool // Do not test DynPath - Index any - MapKey any - Field any - Root any - AnyKey bool - AnyIndex bool + name string + node *PathNode + String string + Index any + StringKey any + Root any + DotStar bool + BracketStar bool }{ // Single node tests { @@ -35,183 +31,149 @@ func TestPathNode(t *testing.T) { Index: 5, }, { - name: "map key", - node: NewMapKey(nil, "mykey"), - String: `['mykey']`, - DynPath: "mykey", - MapKey: "mykey", + name: "map key", + node: NewStringKey(nil, "mykey"), + String: `mykey`, + StringKey: "mykey", }, { - name: "struct field with JSON tag", - node: NewStructField(nil, reflect.StructTag(`json:"json_name"`), "GoFieldName"), - String: "json_name", - Field: "json_name", + name: "dot star", + node: NewDotStar(nil), + String: "*", + DotStar: true, }, { - name: "struct field without JSON tag (fallback to Go name)", - node: NewStructField(nil, reflect.StructTag(""), "GoFieldName"), - String: "GoFieldName", - Field: "GoFieldName", - }, - { - name: "struct field with dash JSON tag", - node: NewStructField(nil, reflect.StructTag(`json:"-"`), "GoFieldName"), - String: "-", - Field: "-", - }, - { - name: "struct field with JSON tag options", - node: NewStructField(nil, reflect.StructTag(`json:"lazy_field,omitempty"`), "LazyField"), - String: "lazy_field", - Field: "lazy_field", - }, - { - name: "any key", - node: NewAnyKey(nil), - String: "[*]", - DynPath: "*", - AnyKey: true, - }, - { - name: "any index", - node: NewAnyIndex(nil), - String: "[*]", - AnyIndex: true, + name: "bracket star", + node: NewBracketStar(nil), + String: "[*]", + BracketStar: true, }, // Two node tests { name: "struct field -> array index", - node: NewIndex(NewStructField(nil, reflect.StructTag(`json:"items"`), "Items"), 3), + node: NewIndex(NewStringKey(nil, "items"), 3), String: "items[3]", Index: 3, }, { - name: "struct field -> map key", - node: NewMapKey(NewStructField(nil, reflect.StructTag(`json:"config"`), "Config"), "database"), - String: `config['database']`, - DynPath: "config.database", - MapKey: "database", - }, - { - name: "struct field -> struct field", - node: NewStructField(NewStructField(nil, reflect.StructTag(`json:"user"`), "User"), reflect.StructTag(`json:"name"`), "Name"), - String: "user.name", - Field: "name", - }, - { - name: "map key -> array index", - node: NewIndex(NewMapKey(nil, "servers"), 0), - String: `['servers'][0]`, - DynPath: "servers[0]", - Index: 0, + name: "struct field -> map key", + node: NewStringKey(NewStringKey(nil, "config"), "database.name"), + String: `config['database.name']`, + StringKey: "database.name", }, { - name: "map key -> struct field", - node: NewStructField(NewMapKey(nil, "primary"), reflect.StructTag(`json:"host"`), "Host"), - String: `['primary'].host`, - DynPath: `primary.host`, - Field: "host", + name: "struct field -> struct field", + node: NewStringKey(NewStringKey(nil, "user"), "name"), + String: "user.name", + StringKey: "name", }, { - name: "array index -> struct field", - node: NewStructField(NewIndex(nil, 2), reflect.StructTag(`json:"id"`), "ID"), - String: "[2].id", - Field: "id", + name: "map key -> array index", + node: NewIndex(NewStringKey(nil, "servers list"), 0), + String: `['servers list'][0]`, + Index: 0, }, { - name: "array index -> map key", - node: NewMapKey(NewIndex(nil, 1), "status"), - String: `[1]['status']`, - DynPath: "[1].status", - MapKey: "status", + name: "array index -> struct field", + node: NewStringKey(NewIndex(nil, 2), "id"), + String: "[2].id", + StringKey: "id", }, { - name: "struct field without JSON tag -> struct field with JSON tag", - node: NewStructField(NewStructField(nil, reflect.StructTag(""), "Parent"), reflect.StructTag(`json:"child_name"`), "ChildName"), - String: "Parent.child_name", - Field: "child_name", + name: "array index -> map key", + node: NewStringKey(NewIndex(nil, 1), "status{}"), + String: `[1]['status{}']`, + StringKey: "status{}", }, { - name: "any key", - node: NewAnyKey(NewStructField(nil, reflect.StructTag(""), "Parent")), - String: "Parent[*]", - DynPath: "Parent.*", - AnyKey: true, + name: "dot star with parent", + node: NewDotStar(NewStringKey(nil, "Parent")), + String: "Parent.*", + DotStar: true, }, { - name: "any index", - node: NewAnyIndex(NewStructField(nil, reflect.StructTag(""), "Parent")), - String: "Parent[*]", - AnyIndex: true, + name: "bracket star with parent", + node: NewBracketStar(NewStringKey(nil, "Parent")), + String: "Parent[*]", + BracketStar: true, }, + // Edge cases with special characters in map keys { - name: "map key with single quote", - node: NewMapKey(nil, "key's"), - String: `['key''s']`, - DynPath: "key's", - MapKey: "key's", + name: "map key with single quote", + node: NewStringKey(nil, "key's"), + String: `['key''s']`, + StringKey: "key's", }, { - name: "map key with multiple single quotes", - node: NewMapKey(nil, "''"), - String: `['''''']`, - DynPath: "''", - MapKey: "''", + name: "map key with multiple single quotes", + node: NewStringKey(nil, "''"), + String: `['''''']`, + StringKey: "''", }, { - name: "empty map key", - node: NewMapKey(nil, ""), - String: `['']`, - IgnoreDynPath: true, - MapKey: "", + name: "empty map key", + node: NewStringKey(nil, ""), + String: `['']`, + StringKey: "", }, { name: "complex path", - node: NewStructField( + node: NewStringKey( NewIndex( - NewMapKey( - NewStructField( - NewStructField(nil, reflect.StructTag(`json:"user"`), "User"), - reflect.StructTag(`json:"settings"`), "Settings"), - "theme"), + NewStringKey( + NewStringKey( + NewStringKey(nil, "user"), + "settings"), + "theme.list"), 0), - reflect.StructTag(`json:"color"`), "Color"), - String: "user.settings['theme'][0].color", - DynPath: "user.settings.theme[0].color", - Field: "color", + "color"), + String: "user.settings['theme.list'][0].color", + StringKey: "color", + }, + { + name: "field with special characters", + node: NewStringKey(nil, "field@name:with#symbols!"), + String: "field@name:with#symbols!", + StringKey: "field@name:with#symbols!", }, { - name: "field with special characters", - node: NewStructField(nil, reflect.StructTag(""), "field@name:with#symbols!"), - String: "field@name:with#symbols!", - Field: "field@name:with#symbols!", + name: "field with spaces", + node: NewStringKey(nil, "field with spaces"), + String: "['field with spaces']", + StringKey: "field with spaces", }, { - name: "field with spaces", - node: NewStructField(nil, reflect.StructTag(""), "field with spaces"), - String: "field with spaces", - Field: "field with spaces", + name: "field starting with digit", + node: NewStringKey(nil, "123field"), + String: "123field", + StringKey: "123field", }, { - name: "field starting with digit", - node: NewStructField(nil, reflect.StructTag(""), "123field"), - String: "123field", - Field: "123field", + name: "field with unicode", + node: NewStringKey(nil, "名前🙂"), + String: "名前🙂", + StringKey: "名前🙂", }, { - name: "field with unicode", - node: NewStructField(nil, reflect.StructTag(""), "名前🙂"), - String: "名前🙂", - Field: "名前🙂", + name: "map key with reserved characters", + node: NewStringKey(nil, "key\x00[],`"), + String: "['key\x00[],`']", + StringKey: "key\x00[],`", + }, + + { + name: "field dot star bracket index", + node: NewIndex(NewDotStar(NewStringKey(nil, "bla")), 0), + String: "bla.*[0]", + Index: 0, }, { - name: "map key with reserved characters", - node: NewMapKey(nil, "key\x00[],`"), - String: "['key\x00[],`']", - DynPath: "key\x00[],`", - MapKey: "key\x00[],`", + name: "field dot star bracket star", + node: NewBracketStar(NewDotStar(NewStringKey(nil, "bla"))), + String: "bla.*[*]", + BracketStar: true, }, } @@ -223,23 +185,12 @@ func TestPathNode(t *testing.T) { // Test roundtrip conversion: String() -> Parse() -> String() parsed, err := Parse(tt.String) - assert.NoError(t, err, "Parse() should not error") - if parsed != nil { + if assert.NoError(t, err, "Parse() should not error") { + assert.Equal(t, tt.node, parsed) roundtripResult := parsed.String() assert.Equal(t, tt.String, roundtripResult, "Roundtrip conversion should be identical") } - if !tt.IgnoreDynPath { - dynResult := tt.node.DynPath() - expectedDyn := tt.String - if tt.DynPath != "" { - expectedDyn = tt.DynPath - // Enforce rule: DynPath should only be set when different from String - assert.NotEqual(t, expectedDyn, tt.String, "Test case %q: DynPath should only be set when different from String", tt.name) - } - assert.Equal(t, expectedDyn, dynResult, "DynPath() method") - } - // Index gotIndex, isIndex := tt.node.Index() if tt.Index == nil { @@ -251,26 +202,14 @@ func TestPathNode(t *testing.T) { assert.True(t, isIndex) } - // Field - gotField, isField := tt.node.Field() - if tt.Field == nil { - assert.Equal(t, "", gotField) - assert.False(t, isField) - } else { - expected := tt.Field.(string) - assert.Equal(t, expected, gotField) - assert.True(t, isField) - } - - // MapKey - gotMapKey, isMapKey := tt.node.MapKey() - if tt.MapKey == nil { - assert.Equal(t, "", gotMapKey) - assert.False(t, isMapKey) + gotStringKey, isStringKey := tt.node.StringKey() + if tt.StringKey == nil { + assert.Equal(t, "", gotStringKey) + assert.False(t, isStringKey) } else { - expected := tt.MapKey.(string) - assert.Equal(t, expected, gotMapKey) - assert.True(t, isMapKey) + expected := tt.StringKey.(string) + assert.Equal(t, expected, gotStringKey) + assert.True(t, isStringKey) } // IsRoot @@ -281,9 +220,9 @@ func TestPathNode(t *testing.T) { assert.True(t, isRoot) } - // AnyKey, AnyIndex - assert.Equal(t, tt.AnyKey, tt.node.AnyKey()) - assert.Equal(t, tt.AnyIndex, tt.node.AnyIndex()) + // DotStar and BracketStar + assert.Equal(t, tt.DotStar, tt.node.DotStar()) + assert.Equal(t, tt.BracketStar, tt.node.BracketStar()) }) } } @@ -434,6 +373,13 @@ func TestParseErrors(t *testing.T) { input: "field['key'", error: "unexpected end of input after quote in map key", }, + + // Invalid dot-star patterns + { + name: "dot star followed by field name", + input: "bla.*foo", + error: "unexpected character 'f' after '.*' at position 5", + }, } for _, tt := range tests { @@ -449,9 +395,165 @@ func TestParseErrors(t *testing.T) { func TestNewIndexPanic(t *testing.T) { defer func() { if r := recover(); r != nil { - assert.Contains(t, r.(string), "index msut be non-negative") + assert.Contains(t, r.(string), "index must be non-negative") } }() NewIndex(nil, -1) // Should panic t.Error("Expected panic did not occur") } + +func TestPrefixAndSkipPrefix(t *testing.T) { + tests := []struct { + input string + n int + prefix string + skipPrefix string + }{ + { + input: "resources.jobs.my_job.tasks[0].name", + n: 0, + prefix: "", + skipPrefix: "resources.jobs.my_job.tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 1, + prefix: "resources", + skipPrefix: "jobs.my_job.tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 3, + prefix: "resources.jobs.my_job", + skipPrefix: "tasks[0].name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 5, + prefix: "resources.jobs.my_job.tasks[0]", + skipPrefix: "name", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 6, + prefix: "resources.jobs.my_job.tasks[0].name", + skipPrefix: "", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: 10, + prefix: "resources.jobs.my_job.tasks[0].name", + skipPrefix: "", + }, + { + input: "resources.jobs.my_job.tasks[0].name", + n: -1, + prefix: "", + skipPrefix: "resources.jobs.my_job.tasks[0].name", + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + path, err := Parse(tt.input) + assert.NoError(t, err) + + // Test Prefix + prefixResult := path.Prefix(tt.n) + if tt.prefix == "" { + assert.Nil(t, prefixResult) + } else { + assert.NotNil(t, prefixResult) + assert.Equal(t, tt.prefix, prefixResult.String()) + } + + // Test SkipPrefix + skipResult := path.SkipPrefix(tt.n) + if tt.skipPrefix == "" { + assert.Nil(t, skipResult) + } else { + assert.NotNil(t, skipResult) + assert.Equal(t, tt.skipPrefix, skipResult.String()) + } + }) + } +} + +func TestLen(t *testing.T) { + tests := []struct { + input string + expected int + }{ + { + input: "", + expected: 0, + }, + { + input: "field", + expected: 1, + }, + { + input: "resources.jobs['my_job'].tasks[0]", + expected: 5, + }, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + var path *PathNode + var err error + path, err = Parse(tt.input) + assert.NoError(t, err) + assert.Equal(t, tt.expected, path.Len()) + }) + } +} + +func TestPureReferenceToPath(t *testing.T) { + tests := []struct { + name string + input string + expected string + ok bool + }{ + { + name: "simple reference", + input: "${resources.jobs.foo.id}", + expected: "resources.jobs.foo.id", + ok: true, + }, + { + name: "simple reference", + input: "${resources.jobs.foo.tasks[1].env.key}", + expected: "resources.jobs.foo.tasks[1].env.key", + ok: true, + }, + { + name: "complex nested reference", + input: "${var.resources.jobs['my_job'].tasks[0]}", + // we use regex from dyn module which only support integers inside brackets: + // expected: "resources.jobs['my_job'].tasks[0]", + }, + { + name: "not a pure reference", + input: "prefix_${var.field}", + }, + { + name: "not a variable reference", + input: "plain_string", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + pathNode, ok := PureReferenceToPath(tt.input) + assert.Equal(t, tt.ok, ok) + if tt.ok { + assert.NotNil(t, pathNode) + assert.Equal(t, tt.expected, pathNode.String()) + } else { + assert.Nil(t, pathNode) + } + }) + } +} diff --git a/libs/structs/structwalk/walk.go b/libs/structs/structwalk/walk.go index 4eeaed348f..0adb1cf6df 100644 --- a/libs/structs/structwalk/walk.go +++ b/libs/structs/structwalk/walk.go @@ -92,7 +92,7 @@ func walkValue(path *structpath.PathNode, val reflect.Value, field *reflect.Stru sort.Strings(keys) for _, ks := range keys { v := val.MapIndex(reflect.ValueOf(ks)) - node := structpath.NewMapKey(path, ks) + node := structpath.NewStringKey(path, ks) walkValue(node, v, nil, visit) } @@ -115,12 +115,18 @@ func walkStruct(path *structpath.PathNode, s reflect.Value, visit VisitFunc) { continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) jsonTag := structtag.JSONTag(sf.Tag.Get("json")) if jsonTag.Name() == "-" { continue // skip fields without json name } + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTag.Name() + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStringKey(path, fieldName) + fieldVal := s.Field(i) // Skip zero values with omitempty unless field is explicitly forced. if jsonTag.OmitEmpty() && fieldVal.IsZero() && !slices.Contains(forced, sf.Name) { diff --git a/libs/structs/structwalk/walk_test.go b/libs/structs/structwalk/walk_test.go index b36960e5fe..f65f9ab5ae 100644 --- a/libs/structs/structwalk/walk_test.go +++ b/libs/structs/structwalk/walk_test.go @@ -87,8 +87,8 @@ func TestValueJobSettings(t *testing.T) { } assert.Equal(t, map[string]any{ - `tags['env']`: "test", - `tags['team']`: "data", + `tags.env`: "test", + `tags.team`: "data", "name": "test-job", "max_concurrent_runs": 5, "timeout_seconds": 3600, diff --git a/libs/structs/structwalk/walktype.go b/libs/structs/structwalk/walktype.go index d1f5a361af..4d572e792f 100644 --- a/libs/structs/structwalk/walktype.go +++ b/libs/structs/structwalk/walktype.go @@ -84,14 +84,14 @@ func walkTypeValue(path *structpath.PathNode, typ reflect.Type, field *reflect.S walkTypeStruct(path, typ, visit, visitedCount) case reflect.Slice, reflect.Array: - walkTypeValue(structpath.NewAnyIndex(path), typ.Elem(), nil, visit, visitedCount) + walkTypeValue(structpath.NewBracketStar(path), typ.Elem(), nil, visit, visitedCount) case reflect.Map: if typ.Key().Kind() != reflect.String { return // unsupported map key type } // For maps, we walk the value type directly at the current path - walkTypeValue(structpath.NewAnyKey(path), typ.Elem(), nil, visit, visitedCount) + walkTypeValue(structpath.NewDotStar(path), typ.Elem(), nil, visit, visitedCount) default: // func, chan, interface, invalid, etc. -> ignore @@ -122,7 +122,12 @@ func walkTypeStruct(path *structpath.PathNode, st reflect.Type, visit VisitTypeF continue } - node := structpath.NewStructField(path, sf.Tag, sf.Name) + // Resolve field name from JSON tag or fall back to Go field name + fieldName := jsonTagName + if fieldName == "" { + fieldName = sf.Name + } + node := structpath.NewStringKey(path, fieldName) walkTypeValue(node, sf.Type, &sf, visit, visitedCount) } } diff --git a/libs/structs/structwalk/walktype_test.go b/libs/structs/structwalk/walktype_test.go index 4b1ce085b0..fded9f8055 100644 --- a/libs/structs/structwalk/walktype_test.go +++ b/libs/structs/structwalk/walktype_test.go @@ -29,8 +29,7 @@ func getScalarFields(t *testing.T, typ reflect.Type) map[string]any { pathNew, err := structpath.Parse(s) if assert.NoError(t, err, "Parse(path.String()) failed for %q: %s", s, err) { newS := pathNew.String() - // This still does not work because of AnyKey / AnyIndex ambiguity - // assert.Equal(t, path, pathNew, "Parse(path.String()) returned different path;\npath=%#v %q\npathNew=%#v %q", path, s, pathNew, newS) + assert.Equal(t, path, pathNew, "Parse(path.String()) returned different path;\npath=%#v %q\npathNew=%#v %q", path, s, pathNew, newS) assert.Equal(t, s, newS, "Parse(path.String()).String() is different from path.String()\npath.String()=%q\npathNew.String()=%q", path, pathNew) } @@ -62,8 +61,8 @@ func TestTypes(t *testing.T) { "EmptyTagField": "", "EmptyTagFieldPtr": "", "IntField": 0, - `Map[*].X`: 0, - `MapPtr[*].X`: 0, + `Map.*.X`: 0, + `MapPtr.*.X`: 0, "Nested.X": 0, "NestedPtr.X": 0, "SliceString[*]": "", @@ -84,8 +83,8 @@ func TestTypeSelf(t *testing.T) { "SelfArrayPtr[*].valid_field": "", "SelfIndirect.X.valid_field": "", "SelfIndirectPtr.X.valid_field": "", - `SelfMapPtr[*].valid_field`: "", - `SelfMap[*].valid_field`: "", + `SelfMapPtr.*.valid_field`: "", + `SelfMap.*.valid_field`: "", "SelfReference.valid_field": "", "SelfSlicePtr[*].valid_field": "", "SelfSlice[*].valid_field": "", @@ -139,42 +138,42 @@ func TestTypeRoot(t *testing.T) { reflect.TypeOf(config.Root{}), 4000, 4300, // 4003 at the time of the update map[string]any{ - "bundle.target": "", - `variables[*].lookup.dashboard`: "", + "bundle.target": "", + `variables.*.lookup.dashboard`: "", - `resources.jobs[*].name`: "", - `resources.jobs[*].timeout_seconds`: 0, - `resources.jobs[*].max_concurrent_runs`: 0, - `resources.jobs[*].format`: jobs.Format(""), - `resources.jobs[*].description`: "", + `resources.jobs.*.name`: "", + `resources.jobs.*.timeout_seconds`: 0, + `resources.jobs.*.max_concurrent_runs`: 0, + `resources.jobs.*.format`: jobs.Format(""), + `resources.jobs.*.description`: "", // Verify nested task fields are accessible - `resources.jobs[*].tasks[*].task_key`: "", - `resources.jobs[*].tasks[*].notebook_task.notebook_path`: "", - `resources.jobs[*].tasks[*].spark_jar_task.main_class_name`: "", - `resources.jobs[*].tasks[*].for_each_task.inputs`: "", - `resources.jobs[*].tasks[*].for_each_task.task.task_key`: "", - `resources.jobs[*].tasks[*].for_each_task.task.notebook_task.notebook_path`: "", - `resources.jobs[*].tasks[*].new_cluster.node_type_id`: "", - `resources.jobs[*].tasks[*].new_cluster.num_workers`: 0, + `resources.jobs.*.tasks[*].task_key`: "", + `resources.jobs.*.tasks[*].notebook_task.notebook_path`: "", + `resources.jobs.*.tasks[*].spark_jar_task.main_class_name`: "", + `resources.jobs.*.tasks[*].for_each_task.inputs`: "", + `resources.jobs.*.tasks[*].for_each_task.task.task_key`: "", + `resources.jobs.*.tasks[*].for_each_task.task.notebook_task.notebook_path`: "", + `resources.jobs.*.tasks[*].new_cluster.node_type_id`: "", + `resources.jobs.*.tasks[*].new_cluster.num_workers`: 0, // Verify job cluster fields are accessible - `resources.jobs[*].job_clusters[*].job_cluster_key`: "", - `resources.jobs[*].job_clusters[*].new_cluster.num_workers`: 0, + `resources.jobs.*.job_clusters[*].job_cluster_key`: "", + `resources.jobs.*.job_clusters[*].new_cluster.num_workers`: 0, }, nil, ) } -func getReadonlyFields(t *testing.T, typ reflect.Type) []string { +func getReadonlyFields(t *testing.T, rootType reflect.Type) []string { var results []string - err := WalkType(typ, func(path *structpath.PathNode, typ reflect.Type, field *reflect.StructField) (continueWalk bool) { + err := WalkType(rootType, func(path *structpath.PathNode, typ reflect.Type, field *reflect.StructField) (continueWalk bool) { if path == nil || field == nil { return true } bundleTag := field.Tag.Get("bundle") if strings.Contains(bundleTag, "readonly") { - results = append(results, path.DynPath()) + results = append(results, path.String()) } return true }) @@ -256,9 +255,9 @@ func TestWalkTypeVisited(t *testing.T) { "Inner.A", "Inner.B", "MapInner", - "MapInner[*]", - "MapInner[*].A", - "MapInner[*].B", + "MapInner.*", + "MapInner.*.A", + "MapInner.*.B", "SliceInner", "SliceInner[*]", "SliceInner[*].A",