diff --git a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala index 548545b969d5a..2d6781dd69c8b 100644 --- a/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/connector/connect/client/jvm/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -34,6 +34,7 @@ import org.apache.spark.sql.catalyst.encoders.{AgnosticEncoder, RowEncoder} import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{BoxedLongEncoder, UnboundRowEncoder} import org.apache.spark.sql.connect.client.{SparkConnectClient, SparkResult} import org.apache.spark.sql.connect.client.util.{Cleaner, ConvertToArrow} +import org.apache.spark.sql.connect.common.LiteralValueProtoConverter.toLiteralProto import org.apache.spark.sql.types.StructType /** @@ -213,15 +214,16 @@ class SparkSession private[sql] ( * @param sqlText * A SQL statement with named parameters to execute. * @param args - * A map of parameter names to string values that are parsed as SQL literal expressions. For - * example, map keys: "rank", "name", "birthdate"; map values: "1", "'Steven'", - * "DATE'2023-03-21'". The fragments of string values belonged to SQL comments are skipped - * while parsing. + * A map of parameter names to Java/Scala objects that can be converted to SQL literal + * expressions. See + * Supported Data Types for supported value types in Scala/Java. For example, map keys: + * "rank", "name", "birthdate"; map values: 1, "Steven", LocalDate.of(2023, 4, 2). Map value + * can be also a `Column` of literal expression, in that case it is taken as is. * * @since 3.4.0 */ @Experimental - def sql(sqlText: String, args: Map[String, String]): DataFrame = { + def sql(sqlText: String, args: Map[String, Any]): DataFrame = { sql(sqlText, args.asJava) } @@ -232,19 +234,24 @@ class SparkSession private[sql] ( * @param sqlText * A SQL statement with named parameters to execute. * @param args - * A map of parameter names to string values that are parsed as SQL literal expressions. For - * example, map keys: "rank", "name", "birthdate"; map values: "1", "'Steven'", - * "DATE'2023-03-21'". The fragments of string values belonged to SQL comments are skipped - * while parsing. + * A map of parameter names to Java/Scala objects that can be converted to SQL literal + * expressions. See + * Supported Data Types for supported value types in Scala/Java. For example, map keys: + * "rank", "name", "birthdate"; map values: 1, "Steven", LocalDate.of(2023, 4, 2). Map value + * can be also a `Column` of literal expression, in that case it is taken as is. * * @since 3.4.0 */ @Experimental - def sql(sqlText: String, args: java.util.Map[String, String]): DataFrame = newDataFrame { + def sql(sqlText: String, args: java.util.Map[String, Any]): DataFrame = newDataFrame { builder => // Send the SQL once to the server and then check the output. val cmd = newCommand(b => - b.setSqlCommand(proto.SqlCommand.newBuilder().setSql(sqlText).putAllArgs(args))) + b.setSqlCommand( + proto.SqlCommand + .newBuilder() + .setSql(sqlText) + .putAllArgs(args.asScala.mapValues(toLiteralProto).toMap.asJava))) val plan = proto.Plan.newBuilder().setCommand(cmd) val responseIter = client.execute(plan.build()) diff --git a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto index 604421fdd4f21..5af6ef5bbad04 100644 --- a/connector/connect/common/src/main/protobuf/spark/connect/commands.proto +++ b/connector/connect/common/src/main/protobuf/spark/connect/commands.proto @@ -53,11 +53,8 @@ message SqlCommand { // (Required) SQL Query. string sql = 1; - // (Optional) A map of parameter names to string values that are parsed as - // SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - // map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - // belonged to SQL comments are skipped while parsing. - map args = 2; + // (Optional) A map of parameter names to literal expressions. + map args = 2; } // A command that can create DataFrame global temp view or local temp view. diff --git a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto index 976bd68e7fe6a..68ce84f2cbed4 100644 --- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto +++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto @@ -108,11 +108,8 @@ message SQL { // (Required) The SQL query. string query = 1; - // (Optional) A map of parameter names to string values that are parsed as - // SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - // map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - // belonged to SQL comments are skipped while parsing. - map args = 2; + // (Optional) A map of parameter names to literal expressions. + map args = 2; } // Relation that reads from a file / table or other data source. Does not have additional diff --git a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala index 45cfbb1be4989..7650532fcf9af 100644 --- a/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala +++ b/connector/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala @@ -212,11 +212,11 @@ class SparkConnectPlanner(val session: SparkSession) { } private def transformSql(sql: proto.SQL): LogicalPlan = { - val args = sql.getArgsMap.asScala.toMap + val args = sql.getArgsMap val parser = session.sessionState.sqlParser val parsedPlan = parser.parsePlan(sql.getQuery) - if (args.nonEmpty) { - ParameterizedQuery(parsedPlan, args.mapValues(parser.parseExpression).toMap) + if (!args.isEmpty) { + ParameterizedQuery(parsedPlan, args.asScala.mapValues(transformLiteral).toMap) } else { parsedPlan } @@ -1604,7 +1604,9 @@ class SparkConnectPlanner(val session: SparkSession) { sessionId: String, responseObserver: StreamObserver[ExecutePlanResponse]): Unit = { // Eagerly execute commands of the provided SQL string. - val df = session.sql(getSqlCommand.getSql, getSqlCommand.getArgsMap) + val df = session.sql( + getSqlCommand.getSql, + getSqlCommand.getArgsMap.asScala.mapValues(transformLiteral).toMap) // Check if commands have been executed. val isCommand = df.queryExecution.commandExecuted.isInstanceOf[CommandResult] val rows = df.logicalPlan match { diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py index 7988cc33009b5..97360a969b2b9 100644 --- a/python/pyspark/sql/connect/plan.py +++ b/python/pyspark/sql/connect/plan.py @@ -945,13 +945,12 @@ def plan(self, session: "SparkConnectClient") -> proto.Relation: class SQL(LogicalPlan): - def __init__(self, query: str, args: Optional[Dict[str, str]] = None) -> None: + def __init__(self, query: str, args: Optional[Dict[str, Any]] = None) -> None: super().__init__(None) if args is not None: for k, v in args.items(): assert isinstance(k, str) - assert isinstance(v, str) self._query = query self._args = args @@ -962,7 +961,7 @@ def plan(self, session: "SparkConnectClient") -> proto.Relation: if self._args is not None and len(self._args) > 0: for k, v in self._args.items(): - plan.sql.args[k] = v + plan.sql.args[k].CopyFrom(LiteralExpression._from_value(v).to_plan(session).literal) return plan @@ -971,7 +970,9 @@ def command(self, session: "SparkConnectClient") -> proto.Command: cmd.sql_command.sql = self._query if self._args is not None and len(self._args) > 0: for k, v in self._args.items(): - cmd.sql_command.args[k] = v + cmd.sql_command.args[k].CopyFrom( + LiteralExpression._from_value(v).to_plan(session).literal + ) return cmd diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py index b2b8934068c6b..69e083a008710 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.py +++ b/python/pyspark/sql/connect/proto/commands_pb2.py @@ -35,7 +35,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\xe9\x03\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\x90\x01\n\nSqlCommand\x12\x10\n\x03sql\x18\x01 \x01(\tR\x03sql\x12\x37\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryR\x04\x61rgs\x1a\x37\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\x9b\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\xad\x06\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_providerB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto"\xe9\x03\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOperationH\x00R\x0ewriteOperation\x12_\n\x15\x63reate_dataframe_view\x18\x03 \x01(\x0b\x32).spark.connect.CreateDataFrameViewCommandH\x00R\x13\x63reateDataframeView\x12O\n\x12write_operation_v2\x18\x04 \x01(\x0b\x32\x1f.spark.connect.WriteOperationV2H\x00R\x10writeOperationV2\x12<\n\x0bsql_command\x18\x05 \x01(\x0b\x32\x19.spark.connect.SqlCommandH\x00R\nsqlCommand\x12\x35\n\textension\x18\xe7\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textensionB\x0e\n\x0c\x63ommand_type"\xb3\x01\n\nSqlCommand\x12\x10\n\x03sql\x18\x01 \x01(\tR\x03sql\x12\x37\n\x04\x61rgs\x18\x02 \x03(\x0b\x32#.spark.connect.SqlCommand.ArgsEntryR\x04\x61rgs\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01"\x96\x01\n\x1a\x43reateDataFrameViewCommand\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x1b\n\tis_global\x18\x03 \x01(\x08R\x08isGlobal\x12\x18\n\x07replace\x18\x04 \x01(\x08R\x07replace"\x9b\x08\n\x0eWriteOperation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1b\n\x06source\x18\x02 \x01(\tH\x01R\x06source\x88\x01\x01\x12\x14\n\x04path\x18\x03 \x01(\tH\x00R\x04path\x12?\n\x05table\x18\x04 \x01(\x0b\x32\'.spark.connect.WriteOperation.SaveTableH\x00R\x05table\x12:\n\x04mode\x18\x05 \x01(\x0e\x32&.spark.connect.WriteOperation.SaveModeR\x04mode\x12*\n\x11sort_column_names\x18\x06 \x03(\tR\x0fsortColumnNames\x12\x31\n\x14partitioning_columns\x18\x07 \x03(\tR\x13partitioningColumns\x12\x43\n\tbucket_by\x18\x08 \x01(\x0b\x32&.spark.connect.WriteOperation.BucketByR\x08\x62ucketBy\x12\x44\n\x07options\x18\t \x03(\x0b\x32*.spark.connect.WriteOperation.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x82\x02\n\tSaveTable\x12\x1d\n\ntable_name\x18\x01 \x01(\tR\ttableName\x12X\n\x0bsave_method\x18\x02 \x01(\x0e\x32\x37.spark.connect.WriteOperation.SaveTable.TableSaveMethodR\nsaveMethod"|\n\x0fTableSaveMethod\x12!\n\x1dTABLE_SAVE_METHOD_UNSPECIFIED\x10\x00\x12#\n\x1fTABLE_SAVE_METHOD_SAVE_AS_TABLE\x10\x01\x12!\n\x1dTABLE_SAVE_METHOD_INSERT_INTO\x10\x02\x1a[\n\x08\x42ucketBy\x12.\n\x13\x62ucket_column_names\x18\x01 \x03(\tR\x11\x62ucketColumnNames\x12\x1f\n\x0bnum_buckets\x18\x02 \x01(\x05R\nnumBuckets"\x89\x01\n\x08SaveMode\x12\x19\n\x15SAVE_MODE_UNSPECIFIED\x10\x00\x12\x14\n\x10SAVE_MODE_APPEND\x10\x01\x12\x17\n\x13SAVE_MODE_OVERWRITE\x10\x02\x12\x1d\n\x19SAVE_MODE_ERROR_IF_EXISTS\x10\x03\x12\x14\n\x10SAVE_MODE_IGNORE\x10\x04\x42\x0b\n\tsave_typeB\t\n\x07_source"\xad\x06\n\x10WriteOperationV2\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1d\n\ntable_name\x18\x02 \x01(\tR\ttableName\x12\x1f\n\x08provider\x18\x03 \x01(\tH\x00R\x08provider\x88\x01\x01\x12L\n\x14partitioning_columns\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13partitioningColumns\x12\x46\n\x07options\x18\x05 \x03(\x0b\x32,.spark.connect.WriteOperationV2.OptionsEntryR\x07options\x12_\n\x10table_properties\x18\x06 \x03(\x0b\x32\x34.spark.connect.WriteOperationV2.TablePropertiesEntryR\x0ftableProperties\x12\x38\n\x04mode\x18\x07 \x01(\x0e\x32$.spark.connect.WriteOperationV2.ModeR\x04mode\x12J\n\x13overwrite_condition\x18\x08 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x12overwriteCondition\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x42\n\x14TablePropertiesEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\x9f\x01\n\x04Mode\x12\x14\n\x10MODE_UNSPECIFIED\x10\x00\x12\x0f\n\x0bMODE_CREATE\x10\x01\x12\x12\n\x0eMODE_OVERWRITE\x10\x02\x12\x1d\n\x19MODE_OVERWRITE_PARTITIONS\x10\x03\x12\x0f\n\x0bMODE_APPEND\x10\x04\x12\x10\n\x0cMODE_REPLACE\x10\x05\x12\x1a\n\x16MODE_CREATE_OR_REPLACE\x10\x06\x42\x0b\n\t_providerB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) @@ -187,29 +187,29 @@ _COMMAND._serialized_start = 139 _COMMAND._serialized_end = 628 _SQLCOMMAND._serialized_start = 631 - _SQLCOMMAND._serialized_end = 775 + _SQLCOMMAND._serialized_end = 810 _SQLCOMMAND_ARGSENTRY._serialized_start = 720 - _SQLCOMMAND_ARGSENTRY._serialized_end = 775 - _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 778 - _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 928 - _WRITEOPERATION._serialized_start = 931 - _WRITEOPERATION._serialized_end = 1982 - _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1406 - _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1464 - _WRITEOPERATION_SAVETABLE._serialized_start = 1467 - _WRITEOPERATION_SAVETABLE._serialized_end = 1725 - _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 1601 - _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 1725 - _WRITEOPERATION_BUCKETBY._serialized_start = 1727 - _WRITEOPERATION_BUCKETBY._serialized_end = 1818 - _WRITEOPERATION_SAVEMODE._serialized_start = 1821 - _WRITEOPERATION_SAVEMODE._serialized_end = 1958 - _WRITEOPERATIONV2._serialized_start = 1985 - _WRITEOPERATIONV2._serialized_end = 2798 - _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1406 - _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1464 - _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 2557 - _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2623 - _WRITEOPERATIONV2_MODE._serialized_start = 2626 - _WRITEOPERATIONV2_MODE._serialized_end = 2785 + _SQLCOMMAND_ARGSENTRY._serialized_end = 810 + _CREATEDATAFRAMEVIEWCOMMAND._serialized_start = 813 + _CREATEDATAFRAMEVIEWCOMMAND._serialized_end = 963 + _WRITEOPERATION._serialized_start = 966 + _WRITEOPERATION._serialized_end = 2017 + _WRITEOPERATION_OPTIONSENTRY._serialized_start = 1441 + _WRITEOPERATION_OPTIONSENTRY._serialized_end = 1499 + _WRITEOPERATION_SAVETABLE._serialized_start = 1502 + _WRITEOPERATION_SAVETABLE._serialized_end = 1760 + _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_start = 1636 + _WRITEOPERATION_SAVETABLE_TABLESAVEMETHOD._serialized_end = 1760 + _WRITEOPERATION_BUCKETBY._serialized_start = 1762 + _WRITEOPERATION_BUCKETBY._serialized_end = 1853 + _WRITEOPERATION_SAVEMODE._serialized_start = 1856 + _WRITEOPERATION_SAVEMODE._serialized_end = 1993 + _WRITEOPERATIONV2._serialized_start = 2020 + _WRITEOPERATIONV2._serialized_end = 2833 + _WRITEOPERATIONV2_OPTIONSENTRY._serialized_start = 1441 + _WRITEOPERATIONV2_OPTIONSENTRY._serialized_end = 1499 + _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_start = 2592 + _WRITEOPERATIONV2_TABLEPROPERTIESENTRY._serialized_end = 2658 + _WRITEOPERATIONV2_MODE._serialized_start = 2661 + _WRITEOPERATIONV2_MODE._serialized_end = 2820 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index b01745e888fdf..7619b76434aa9 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -161,13 +161,17 @@ class SqlCommand(google.protobuf.message.Message): KEY_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int key: builtins.str - value: builtins.str + @property + def value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal: ... def __init__( self, *, key: builtins.str = ..., - value: builtins.str = ..., + value: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ..., ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] ) -> None: ... @@ -177,17 +181,20 @@ class SqlCommand(google.protobuf.message.Message): sql: builtins.str """(Required) SQL Query.""" @property - def args(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: - """(Optional) A map of parameter names to string values that are parsed as - SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - belonged to SQL comments are skipped while parsing. - """ + def args( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal + ]: + """(Optional) A map of parameter names to literal expressions.""" def __init__( self, *, sql: builtins.str = ..., - args: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + args: collections.abc.Mapping[ + builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal + ] + | None = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["args", b"args", "sql", b"sql"] diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py index 81a0499666b28..a7d46534a573d 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.py +++ b/python/pyspark/sql/connect/proto/relations_pb2.py @@ -36,7 +36,7 @@ DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile( - b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xaf\x14\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12\x34\n\x06set_op\x18\x06 \x01(\x0b\x32\x1b.spark.connect.SetOperationH\x00R\x05setOp\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12,\n\x05range\x18\x0f \x01(\x0b\x32\x14.spark.connect.RangeH\x00R\x05range\x12\x45\n\x0esubquery_alias\x18\x10 \x01(\x0b\x32\x1c.spark.connect.SubqueryAliasH\x00R\rsubqueryAlias\x12>\n\x0brepartition\x18\x11 \x01(\x0b\x32\x1a.spark.connect.RepartitionH\x00R\x0brepartition\x12*\n\x05to_df\x18\x12 \x01(\x0b\x32\x13.spark.connect.ToDFH\x00R\x04toDf\x12U\n\x14with_columns_renamed\x18\x13 \x01(\x0b\x32!.spark.connect.WithColumnsRenamedH\x00R\x12withColumnsRenamed\x12<\n\x0bshow_string\x18\x14 \x01(\x0b\x32\x19.spark.connect.ShowStringH\x00R\nshowString\x12)\n\x04\x64rop\x18\x15 \x01(\x0b\x32\x13.spark.connect.DropH\x00R\x04\x64rop\x12)\n\x04tail\x18\x16 \x01(\x0b\x32\x13.spark.connect.TailH\x00R\x04tail\x12?\n\x0cwith_columns\x18\x17 \x01(\x0b\x32\x1a.spark.connect.WithColumnsH\x00R\x0bwithColumns\x12)\n\x04hint\x18\x18 \x01(\x0b\x32\x13.spark.connect.HintH\x00R\x04hint\x12\x32\n\x07unpivot\x18\x19 \x01(\x0b\x32\x16.spark.connect.UnpivotH\x00R\x07unpivot\x12\x36\n\tto_schema\x18\x1a \x01(\x0b\x32\x17.spark.connect.ToSchemaH\x00R\x08toSchema\x12\x64\n\x19repartition_by_expression\x18\x1b \x01(\x0b\x32&.spark.connect.RepartitionByExpressionH\x00R\x17repartitionByExpression\x12\x45\n\x0emap_partitions\x18\x1c \x01(\x0b\x32\x1c.spark.connect.MapPartitionsH\x00R\rmapPartitions\x12H\n\x0f\x63ollect_metrics\x18\x1d \x01(\x0b\x32\x1d.spark.connect.CollectMetricsH\x00R\x0e\x63ollectMetrics\x12,\n\x05parse\x18\x1e \x01(\x0b\x32\x14.spark.connect.ParseH\x00R\x05parse\x12\x36\n\tgroup_map\x18\x1f \x01(\x0b\x32\x17.spark.connect.GroupMapH\x00R\x08groupMap\x12=\n\x0c\x63o_group_map\x18 \x01(\x0b\x32\x19.spark.connect.CoGroupMapH\x00R\ncoGroupMap\x12\x30\n\x07\x66ill_na\x18Z \x01(\x0b\x32\x15.spark.connect.NAFillH\x00R\x06\x66illNa\x12\x30\n\x07\x64rop_na\x18[ \x01(\x0b\x32\x15.spark.connect.NADropH\x00R\x06\x64ropNa\x12\x34\n\x07replace\x18\\ \x01(\x0b\x32\x18.spark.connect.NAReplaceH\x00R\x07replace\x12\x36\n\x07summary\x18\x64 \x01(\x0b\x32\x1a.spark.connect.StatSummaryH\x00R\x07summary\x12\x39\n\x08\x63rosstab\x18\x65 \x01(\x0b\x32\x1b.spark.connect.StatCrosstabH\x00R\x08\x63rosstab\x12\x39\n\x08\x64\x65scribe\x18\x66 \x01(\x0b\x32\x1b.spark.connect.StatDescribeH\x00R\x08\x64\x65scribe\x12*\n\x03\x63ov\x18g \x01(\x0b\x32\x16.spark.connect.StatCovH\x00R\x03\x63ov\x12-\n\x04\x63orr\x18h \x01(\x0b\x32\x17.spark.connect.StatCorrH\x00R\x04\x63orr\x12L\n\x0f\x61pprox_quantile\x18i \x01(\x0b\x32!.spark.connect.StatApproxQuantileH\x00R\x0e\x61pproxQuantile\x12=\n\nfreq_items\x18j \x01(\x0b\x32\x1c.spark.connect.StatFreqItemsH\x00R\tfreqItems\x12:\n\tsample_by\x18k \x01(\x0b\x32\x1b.spark.connect.StatSampleByH\x00R\x08sampleBy\x12\x33\n\x07\x63\x61talog\x18\xc8\x01 \x01(\x0b\x32\x16.spark.connect.CatalogH\x00R\x07\x63\x61talog\x12\x35\n\textension\x18\xe6\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"[\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id"\x86\x01\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query\x12\x30\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.spark.connect.SQL.ArgsEntryR\x04\x61rgs\x1a\x37\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"\xf4\x04\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x1a\xc0\x01\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x45\n\x07options\x18\x02 \x03(\x0b\x32+.spark.connect.Read.NamedTable.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x95\x02\n\nDataSource\x12\x1b\n\x06\x66ormat\x18\x01 \x01(\tH\x00R\x06\x66ormat\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x12\x14\n\x05paths\x18\x04 \x03(\tR\x05paths\x12\x1e\n\npredicates\x18\x05 \x03(\tR\npredicates\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07_formatB\t\n\x07_schemaB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\xd7\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType\x12#\n\rusing_columns\x18\x05 \x03(\tR\x0cusingColumns"\xd0\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06\x12\x13\n\x0fJOIN_TYPE_CROSS\x10\x07"\xdf\x03\n\x0cSetOperation\x12\x36\n\nleft_input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\tleftInput\x12\x38\n\x0bright_input\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\nrightInput\x12\x45\n\x0bset_op_type\x18\x03 \x01(\x0e\x32%.spark.connect.SetOperation.SetOpTypeR\tsetOpType\x12\x1a\n\x06is_all\x18\x04 \x01(\x08H\x00R\x05isAll\x88\x01\x01\x12\x1c\n\x07\x62y_name\x18\x05 \x01(\x08H\x01R\x06\x62yName\x88\x01\x01\x12\x37\n\x15\x61llow_missing_columns\x18\x06 \x01(\x08H\x02R\x13\x61llowMissingColumns\x88\x01\x01"r\n\tSetOpType\x12\x1b\n\x17SET_OP_TYPE_UNSPECIFIED\x10\x00\x12\x19\n\x15SET_OP_TYPE_INTERSECT\x10\x01\x12\x15\n\x11SET_OP_TYPE_UNION\x10\x02\x12\x16\n\x12SET_OP_TYPE_EXCEPT\x10\x03\x42\t\n\x07_is_allB\n\n\x08_by_nameB\x18\n\x16_allow_missing_columns"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"K\n\x04Tail\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"\xc6\x04\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x41\n\ngroup_type\x18\x02 \x01(\x0e\x32".spark.connect.Aggregate.GroupTypeR\tgroupType\x12L\n\x14grouping_expressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12N\n\x15\x61ggregate_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x14\x61ggregateExpressions\x12\x34\n\x05pivot\x18\x05 \x01(\x0b\x32\x1e.spark.connect.Aggregate.PivotR\x05pivot\x1ao\n\x05Pivot\x12+\n\x03\x63ol\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x81\x01\n\tGroupType\x12\x1a\n\x16GROUP_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12GROUP_TYPE_GROUPBY\x10\x01\x12\x15\n\x11GROUP_TYPE_ROLLUP\x10\x02\x12\x13\n\x0fGROUP_TYPE_CUBE\x10\x03\x12\x14\n\x10GROUP_TYPE_PIVOT\x10\x04"\xa0\x01\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x05order\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\x05order\x12 \n\tis_global\x18\x03 \x01(\x08H\x00R\x08isGlobal\x88\x01\x01\x42\x0c\n\n_is_global"\x8d\x01\n\x04\x44rop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x33\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07\x63olumns\x12!\n\x0c\x63olumn_names\x18\x03 \x03(\tR\x0b\x63olumnNames"\xab\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12\x32\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08H\x00R\x10\x61llColumnsAsKeys\x88\x01\x01\x42\x16\n\x14_all_columns_as_keys"Y\n\rLocalRelation\x12\x17\n\x04\x64\x61ta\x18\x01 \x01(\x0cH\x00R\x04\x64\x61ta\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x42\x07\n\x05_dataB\t\n\x07_schema"\x91\x02\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12.\n\x10with_replacement\x18\x04 \x01(\x08H\x00R\x0fwithReplacement\x88\x01\x01\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x01R\x04seed\x88\x01\x01\x12/\n\x13\x64\x65terministic_order\x18\x06 \x01(\x08R\x12\x64\x65terministicOrderB\x13\n\x11_with_replacementB\x07\n\x05_seed"\x91\x01\n\x05Range\x12\x19\n\x05start\x18\x01 \x01(\x03H\x00R\x05start\x88\x01\x01\x12\x10\n\x03\x65nd\x18\x02 \x01(\x03R\x03\x65nd\x12\x12\n\x04step\x18\x03 \x01(\x03R\x04step\x12*\n\x0enum_partitions\x18\x04 \x01(\x05H\x01R\rnumPartitions\x88\x01\x01\x42\x08\n\x06_startB\x11\n\x0f_num_partitions"r\n\rSubqueryAlias\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\x12\x1c\n\tqualifier\x18\x03 \x03(\tR\tqualifier"\x8e\x01\n\x0bRepartition\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12%\n\x0enum_partitions\x18\x02 \x01(\x05R\rnumPartitions\x12\x1d\n\x07shuffle\x18\x03 \x01(\x08H\x00R\x07shuffle\x88\x01\x01\x42\n\n\x08_shuffle"\x8e\x01\n\nShowString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate\x12\x1a\n\x08vertical\x18\x04 \x01(\x08R\x08vertical"\\\n\x0bStatSummary\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1e\n\nstatistics\x18\x02 \x03(\tR\nstatistics"Q\n\x0cStatDescribe\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols"e\n\x0cStatCrosstab\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"`\n\x07StatCov\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"\x89\x01\n\x08StatCorr\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2\x12\x1b\n\x06method\x18\x04 \x01(\tH\x00R\x06method\x88\x01\x01\x42\t\n\x07_method"\xa4\x01\n\x12StatApproxQuantile\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12$\n\rprobabilities\x18\x03 \x03(\x01R\rprobabilities\x12%\n\x0erelative_error\x18\x04 \x01(\x01R\rrelativeError"}\n\rStatFreqItems\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x1d\n\x07support\x18\x03 \x01(\x01H\x00R\x07support\x88\x01\x01\x42\n\n\x08_support"\xb5\x02\n\x0cStatSampleBy\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03\x63ol\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x42\n\tfractions\x18\x03 \x03(\x0b\x32$.spark.connect.StatSampleBy.FractionR\tfractions\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x00R\x04seed\x88\x01\x01\x1a\x63\n\x08\x46raction\x12;\n\x07stratum\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x07stratum\x12\x1a\n\x08\x66raction\x18\x02 \x01(\x01R\x08\x66ractionB\x07\n\x05_seed"\x86\x01\n\x06NAFill\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x39\n\x06values\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x86\x01\n\x06NADrop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\'\n\rmin_non_nulls\x18\x03 \x01(\x05H\x00R\x0bminNonNulls\x88\x01\x01\x42\x10\n\x0e_min_non_nulls"\xa8\x02\n\tNAReplace\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12H\n\x0creplacements\x18\x03 \x03(\x0b\x32$.spark.connect.NAReplace.ReplacementR\x0creplacements\x1a\x8d\x01\n\x0bReplacement\x12>\n\told_value\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08oldValue\x12>\n\tnew_value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08newValue"X\n\x04ToDF\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames"\xef\x01\n\x12WithColumnsRenamed\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x65\n\x12rename_columns_map\x18\x02 \x03(\x0b\x32\x37.spark.connect.WithColumnsRenamed.RenameColumnsMapEntryR\x10renameColumnsMap\x1a\x43\n\x15RenameColumnsMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"w\n\x0bWithColumns\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x07\x61liases\x18\x02 \x03(\x0b\x32\x1f.spark.connect.Expression.AliasR\x07\x61liases"\x84\x01\n\x04Hint\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x39\n\nparameters\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\nparameters"\xc7\x02\n\x07Unpivot\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03ids\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x03ids\x12:\n\x06values\x18\x03 \x01(\x0b\x32\x1d.spark.connect.Unpivot.ValuesH\x00R\x06values\x88\x01\x01\x12\x30\n\x14variable_column_name\x18\x04 \x01(\tR\x12variableColumnName\x12*\n\x11value_column_name\x18\x05 \x01(\tR\x0fvalueColumnName\x1a;\n\x06Values\x12\x31\n\x06values\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x06valuesB\t\n\x07_values"j\n\x08ToSchema\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12/\n\x06schema\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema"\xcb\x01\n\x17RepartitionByExpression\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x0fpartition_exprs\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0epartitionExprs\x12*\n\x0enum_partitions\x18\x03 \x01(\x05H\x00R\rnumPartitions\x88\x01\x01\x42\x11\n\x0f_num_partitions"\x82\x01\n\rMapPartitions\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x04\x66unc\x18\x02 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\xcb\x01\n\x08GroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\xe0\x02\n\nCoGroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12W\n\x1ainput_grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18inputGroupingExpressions\x12-\n\x05other\x18\x03 \x01(\x0b\x32\x17.spark.connect.RelationR\x05other\x12W\n\x1aother_grouping_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18otherGroupingExpressions\x12\x42\n\x04\x66unc\x18\x05 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\x88\x01\n\x0e\x43ollectMetrics\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x33\n\x07metrics\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07metrics"\x84\x03\n\x05Parse\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x38\n\x06\x66ormat\x18\x02 \x01(\x0e\x32 .spark.connect.Parse.ParseFormatR\x06\x66ormat\x12\x34\n\x06schema\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x06schema\x88\x01\x01\x12;\n\x07options\x18\x04 \x03(\x0b\x32!.spark.connect.Parse.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"X\n\x0bParseFormat\x12\x1c\n\x18PARSE_FORMAT_UNSPECIFIED\x10\x00\x12\x14\n\x10PARSE_FORMAT_CSV\x10\x01\x12\x15\n\x11PARSE_FORMAT_JSON\x10\x02\x42\t\n\x07_schemaB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' + b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xaf\x14\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66ilter\x18\x04 \x01(\x0b\x32\x15.spark.connect.FilterH\x00R\x06\x66ilter\x12)\n\x04join\x18\x05 \x01(\x0b\x32\x13.spark.connect.JoinH\x00R\x04join\x12\x34\n\x06set_op\x18\x06 \x01(\x0b\x32\x1b.spark.connect.SetOperationH\x00R\x05setOp\x12)\n\x04sort\x18\x07 \x01(\x0b\x32\x13.spark.connect.SortH\x00R\x04sort\x12,\n\x05limit\x18\x08 \x01(\x0b\x32\x14.spark.connect.LimitH\x00R\x05limit\x12\x38\n\taggregate\x18\t \x01(\x0b\x32\x18.spark.connect.AggregateH\x00R\taggregate\x12&\n\x03sql\x18\n \x01(\x0b\x32\x12.spark.connect.SQLH\x00R\x03sql\x12\x45\n\x0elocal_relation\x18\x0b \x01(\x0b\x32\x1c.spark.connect.LocalRelationH\x00R\rlocalRelation\x12/\n\x06sample\x18\x0c \x01(\x0b\x32\x15.spark.connect.SampleH\x00R\x06sample\x12/\n\x06offset\x18\r \x01(\x0b\x32\x15.spark.connect.OffsetH\x00R\x06offset\x12>\n\x0b\x64\x65\x64uplicate\x18\x0e \x01(\x0b\x32\x1a.spark.connect.DeduplicateH\x00R\x0b\x64\x65\x64uplicate\x12,\n\x05range\x18\x0f \x01(\x0b\x32\x14.spark.connect.RangeH\x00R\x05range\x12\x45\n\x0esubquery_alias\x18\x10 \x01(\x0b\x32\x1c.spark.connect.SubqueryAliasH\x00R\rsubqueryAlias\x12>\n\x0brepartition\x18\x11 \x01(\x0b\x32\x1a.spark.connect.RepartitionH\x00R\x0brepartition\x12*\n\x05to_df\x18\x12 \x01(\x0b\x32\x13.spark.connect.ToDFH\x00R\x04toDf\x12U\n\x14with_columns_renamed\x18\x13 \x01(\x0b\x32!.spark.connect.WithColumnsRenamedH\x00R\x12withColumnsRenamed\x12<\n\x0bshow_string\x18\x14 \x01(\x0b\x32\x19.spark.connect.ShowStringH\x00R\nshowString\x12)\n\x04\x64rop\x18\x15 \x01(\x0b\x32\x13.spark.connect.DropH\x00R\x04\x64rop\x12)\n\x04tail\x18\x16 \x01(\x0b\x32\x13.spark.connect.TailH\x00R\x04tail\x12?\n\x0cwith_columns\x18\x17 \x01(\x0b\x32\x1a.spark.connect.WithColumnsH\x00R\x0bwithColumns\x12)\n\x04hint\x18\x18 \x01(\x0b\x32\x13.spark.connect.HintH\x00R\x04hint\x12\x32\n\x07unpivot\x18\x19 \x01(\x0b\x32\x16.spark.connect.UnpivotH\x00R\x07unpivot\x12\x36\n\tto_schema\x18\x1a \x01(\x0b\x32\x17.spark.connect.ToSchemaH\x00R\x08toSchema\x12\x64\n\x19repartition_by_expression\x18\x1b \x01(\x0b\x32&.spark.connect.RepartitionByExpressionH\x00R\x17repartitionByExpression\x12\x45\n\x0emap_partitions\x18\x1c \x01(\x0b\x32\x1c.spark.connect.MapPartitionsH\x00R\rmapPartitions\x12H\n\x0f\x63ollect_metrics\x18\x1d \x01(\x0b\x32\x1d.spark.connect.CollectMetricsH\x00R\x0e\x63ollectMetrics\x12,\n\x05parse\x18\x1e \x01(\x0b\x32\x14.spark.connect.ParseH\x00R\x05parse\x12\x36\n\tgroup_map\x18\x1f \x01(\x0b\x32\x17.spark.connect.GroupMapH\x00R\x08groupMap\x12=\n\x0c\x63o_group_map\x18 \x01(\x0b\x32\x19.spark.connect.CoGroupMapH\x00R\ncoGroupMap\x12\x30\n\x07\x66ill_na\x18Z \x01(\x0b\x32\x15.spark.connect.NAFillH\x00R\x06\x66illNa\x12\x30\n\x07\x64rop_na\x18[ \x01(\x0b\x32\x15.spark.connect.NADropH\x00R\x06\x64ropNa\x12\x34\n\x07replace\x18\\ \x01(\x0b\x32\x18.spark.connect.NAReplaceH\x00R\x07replace\x12\x36\n\x07summary\x18\x64 \x01(\x0b\x32\x1a.spark.connect.StatSummaryH\x00R\x07summary\x12\x39\n\x08\x63rosstab\x18\x65 \x01(\x0b\x32\x1b.spark.connect.StatCrosstabH\x00R\x08\x63rosstab\x12\x39\n\x08\x64\x65scribe\x18\x66 \x01(\x0b\x32\x1b.spark.connect.StatDescribeH\x00R\x08\x64\x65scribe\x12*\n\x03\x63ov\x18g \x01(\x0b\x32\x16.spark.connect.StatCovH\x00R\x03\x63ov\x12-\n\x04\x63orr\x18h \x01(\x0b\x32\x17.spark.connect.StatCorrH\x00R\x04\x63orr\x12L\n\x0f\x61pprox_quantile\x18i \x01(\x0b\x32!.spark.connect.StatApproxQuantileH\x00R\x0e\x61pproxQuantile\x12=\n\nfreq_items\x18j \x01(\x0b\x32\x1c.spark.connect.StatFreqItemsH\x00R\tfreqItems\x12:\n\tsample_by\x18k \x01(\x0b\x32\x1b.spark.connect.StatSampleByH\x00R\x08sampleBy\x12\x33\n\x07\x63\x61talog\x18\xc8\x01 \x01(\x0b\x32\x16.spark.connect.CatalogH\x00R\x07\x63\x61talog\x12\x35\n\textension\x18\xe6\x07 \x01(\x0b\x32\x14.google.protobuf.AnyH\x00R\textension\x12\x33\n\x07unknown\x18\xe7\x07 \x01(\x0b\x32\x16.spark.connect.UnknownH\x00R\x07unknownB\n\n\x08rel_type"\t\n\x07Unknown"[\n\x0eRelationCommon\x12\x1f\n\x0bsource_info\x18\x01 \x01(\tR\nsourceInfo\x12\x1c\n\x07plan_id\x18\x02 \x01(\x03H\x00R\x06planId\x88\x01\x01\x42\n\n\x08_plan_id"\xa9\x01\n\x03SQL\x12\x14\n\x05query\x18\x01 \x01(\tR\x05query\x12\x30\n\x04\x61rgs\x18\x02 \x03(\x0b\x32\x1c.spark.connect.SQL.ArgsEntryR\x04\x61rgs\x1aZ\n\tArgsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01"\xf4\x04\n\x04Read\x12\x41\n\x0bnamed_table\x18\x01 \x01(\x0b\x32\x1e.spark.connect.Read.NamedTableH\x00R\nnamedTable\x12\x41\n\x0b\x64\x61ta_source\x18\x02 \x01(\x0b\x32\x1e.spark.connect.Read.DataSourceH\x00R\ndataSource\x1a\xc0\x01\n\nNamedTable\x12/\n\x13unparsed_identifier\x18\x01 \x01(\tR\x12unparsedIdentifier\x12\x45\n\x07options\x18\x02 \x03(\x0b\x32+.spark.connect.Read.NamedTable.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x1a\x95\x02\n\nDataSource\x12\x1b\n\x06\x66ormat\x18\x01 \x01(\tH\x00R\x06\x66ormat\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x12\x45\n\x07options\x18\x03 \x03(\x0b\x32+.spark.connect.Read.DataSource.OptionsEntryR\x07options\x12\x14\n\x05paths\x18\x04 \x03(\tR\x05paths\x12\x1e\n\npredicates\x18\x05 \x03(\tR\npredicates\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01\x42\t\n\x07_formatB\t\n\x07_schemaB\x0b\n\tread_type"u\n\x07Project\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12;\n\x0b\x65xpressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0b\x65xpressions"p\n\x06\x46ilter\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x37\n\tcondition\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\tcondition"\xd7\x03\n\x04Join\x12+\n\x04left\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x04left\x12-\n\x05right\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\x05right\x12@\n\x0ejoin_condition\x18\x03 \x01(\x0b\x32\x19.spark.connect.ExpressionR\rjoinCondition\x12\x39\n\tjoin_type\x18\x04 \x01(\x0e\x32\x1c.spark.connect.Join.JoinTypeR\x08joinType\x12#\n\rusing_columns\x18\x05 \x03(\tR\x0cusingColumns"\xd0\x01\n\x08JoinType\x12\x19\n\x15JOIN_TYPE_UNSPECIFIED\x10\x00\x12\x13\n\x0fJOIN_TYPE_INNER\x10\x01\x12\x18\n\x14JOIN_TYPE_FULL_OUTER\x10\x02\x12\x18\n\x14JOIN_TYPE_LEFT_OUTER\x10\x03\x12\x19\n\x15JOIN_TYPE_RIGHT_OUTER\x10\x04\x12\x17\n\x13JOIN_TYPE_LEFT_ANTI\x10\x05\x12\x17\n\x13JOIN_TYPE_LEFT_SEMI\x10\x06\x12\x13\n\x0fJOIN_TYPE_CROSS\x10\x07"\xdf\x03\n\x0cSetOperation\x12\x36\n\nleft_input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\tleftInput\x12\x38\n\x0bright_input\x18\x02 \x01(\x0b\x32\x17.spark.connect.RelationR\nrightInput\x12\x45\n\x0bset_op_type\x18\x03 \x01(\x0e\x32%.spark.connect.SetOperation.SetOpTypeR\tsetOpType\x12\x1a\n\x06is_all\x18\x04 \x01(\x08H\x00R\x05isAll\x88\x01\x01\x12\x1c\n\x07\x62y_name\x18\x05 \x01(\x08H\x01R\x06\x62yName\x88\x01\x01\x12\x37\n\x15\x61llow_missing_columns\x18\x06 \x01(\x08H\x02R\x13\x61llowMissingColumns\x88\x01\x01"r\n\tSetOpType\x12\x1b\n\x17SET_OP_TYPE_UNSPECIFIED\x10\x00\x12\x19\n\x15SET_OP_TYPE_INTERSECT\x10\x01\x12\x15\n\x11SET_OP_TYPE_UNION\x10\x02\x12\x16\n\x12SET_OP_TYPE_EXCEPT\x10\x03\x42\t\n\x07_is_allB\n\n\x08_by_nameB\x18\n\x16_allow_missing_columns"L\n\x05Limit\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"O\n\x06Offset\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x16\n\x06offset\x18\x02 \x01(\x05R\x06offset"K\n\x04Tail\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05limit\x18\x02 \x01(\x05R\x05limit"\xc6\x04\n\tAggregate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x41\n\ngroup_type\x18\x02 \x01(\x0e\x32".spark.connect.Aggregate.GroupTypeR\tgroupType\x12L\n\x14grouping_expressions\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12N\n\x15\x61ggregate_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x14\x61ggregateExpressions\x12\x34\n\x05pivot\x18\x05 \x01(\x0b\x32\x1e.spark.connect.Aggregate.PivotR\x05pivot\x1ao\n\x05Pivot\x12+\n\x03\x63ol\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x39\n\x06values\x18\x02 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x81\x01\n\tGroupType\x12\x1a\n\x16GROUP_TYPE_UNSPECIFIED\x10\x00\x12\x16\n\x12GROUP_TYPE_GROUPBY\x10\x01\x12\x15\n\x11GROUP_TYPE_ROLLUP\x10\x02\x12\x13\n\x0fGROUP_TYPE_CUBE\x10\x03\x12\x14\n\x10GROUP_TYPE_PIVOT\x10\x04"\xa0\x01\n\x04Sort\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x05order\x18\x02 \x03(\x0b\x32#.spark.connect.Expression.SortOrderR\x05order\x12 \n\tis_global\x18\x03 \x01(\x08H\x00R\x08isGlobal\x88\x01\x01\x42\x0c\n\n_is_global"\x8d\x01\n\x04\x44rop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x33\n\x07\x63olumns\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07\x63olumns\x12!\n\x0c\x63olumn_names\x18\x03 \x03(\tR\x0b\x63olumnNames"\xab\x01\n\x0b\x44\x65\x64uplicate\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames\x12\x32\n\x13\x61ll_columns_as_keys\x18\x03 \x01(\x08H\x00R\x10\x61llColumnsAsKeys\x88\x01\x01\x42\x16\n\x14_all_columns_as_keys"Y\n\rLocalRelation\x12\x17\n\x04\x64\x61ta\x18\x01 \x01(\x0cH\x00R\x04\x64\x61ta\x88\x01\x01\x12\x1b\n\x06schema\x18\x02 \x01(\tH\x01R\x06schema\x88\x01\x01\x42\x07\n\x05_dataB\t\n\x07_schema"\x91\x02\n\x06Sample\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1f\n\x0blower_bound\x18\x02 \x01(\x01R\nlowerBound\x12\x1f\n\x0bupper_bound\x18\x03 \x01(\x01R\nupperBound\x12.\n\x10with_replacement\x18\x04 \x01(\x08H\x00R\x0fwithReplacement\x88\x01\x01\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x01R\x04seed\x88\x01\x01\x12/\n\x13\x64\x65terministic_order\x18\x06 \x01(\x08R\x12\x64\x65terministicOrderB\x13\n\x11_with_replacementB\x07\n\x05_seed"\x91\x01\n\x05Range\x12\x19\n\x05start\x18\x01 \x01(\x03H\x00R\x05start\x88\x01\x01\x12\x10\n\x03\x65nd\x18\x02 \x01(\x03R\x03\x65nd\x12\x12\n\x04step\x18\x03 \x01(\x03R\x04step\x12*\n\x0enum_partitions\x18\x04 \x01(\x05H\x01R\rnumPartitions\x88\x01\x01\x42\x08\n\x06_startB\x11\n\x0f_num_partitions"r\n\rSubqueryAlias\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x14\n\x05\x61lias\x18\x02 \x01(\tR\x05\x61lias\x12\x1c\n\tqualifier\x18\x03 \x03(\tR\tqualifier"\x8e\x01\n\x0bRepartition\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12%\n\x0enum_partitions\x18\x02 \x01(\x05R\rnumPartitions\x12\x1d\n\x07shuffle\x18\x03 \x01(\x08H\x00R\x07shuffle\x88\x01\x01\x42\n\n\x08_shuffle"\x8e\x01\n\nShowString\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x19\n\x08num_rows\x18\x02 \x01(\x05R\x07numRows\x12\x1a\n\x08truncate\x18\x03 \x01(\x05R\x08truncate\x12\x1a\n\x08vertical\x18\x04 \x01(\x08R\x08vertical"\\\n\x0bStatSummary\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x1e\n\nstatistics\x18\x02 \x03(\tR\nstatistics"Q\n\x0cStatDescribe\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols"e\n\x0cStatCrosstab\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"`\n\x07StatCov\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2"\x89\x01\n\x08StatCorr\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ol1\x18\x02 \x01(\tR\x04\x63ol1\x12\x12\n\x04\x63ol2\x18\x03 \x01(\tR\x04\x63ol2\x12\x1b\n\x06method\x18\x04 \x01(\tH\x00R\x06method\x88\x01\x01\x42\t\n\x07_method"\xa4\x01\n\x12StatApproxQuantile\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12$\n\rprobabilities\x18\x03 \x03(\x01R\rprobabilities\x12%\n\x0erelative_error\x18\x04 \x01(\x01R\rrelativeError"}\n\rStatFreqItems\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x1d\n\x07support\x18\x03 \x01(\x01H\x00R\x07support\x88\x01\x01\x42\n\n\x08_support"\xb5\x02\n\x0cStatSampleBy\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03\x63ol\x18\x02 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x03\x63ol\x12\x42\n\tfractions\x18\x03 \x03(\x0b\x32$.spark.connect.StatSampleBy.FractionR\tfractions\x12\x17\n\x04seed\x18\x05 \x01(\x03H\x00R\x04seed\x88\x01\x01\x1a\x63\n\x08\x46raction\x12;\n\x07stratum\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x07stratum\x12\x1a\n\x08\x66raction\x18\x02 \x01(\x01R\x08\x66ractionB\x07\n\x05_seed"\x86\x01\n\x06NAFill\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\x39\n\x06values\x18\x03 \x03(\x0b\x32!.spark.connect.Expression.LiteralR\x06values"\x86\x01\n\x06NADrop\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12\'\n\rmin_non_nulls\x18\x03 \x01(\x05H\x00R\x0bminNonNulls\x88\x01\x01\x42\x10\n\x0e_min_non_nulls"\xa8\x02\n\tNAReplace\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04\x63ols\x18\x02 \x03(\tR\x04\x63ols\x12H\n\x0creplacements\x18\x03 \x03(\x0b\x32$.spark.connect.NAReplace.ReplacementR\x0creplacements\x1a\x8d\x01\n\x0bReplacement\x12>\n\told_value\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08oldValue\x12>\n\tnew_value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x08newValue"X\n\x04ToDF\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63olumn_names\x18\x02 \x03(\tR\x0b\x63olumnNames"\xef\x01\n\x12WithColumnsRenamed\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x65\n\x12rename_columns_map\x18\x02 \x03(\x0b\x32\x37.spark.connect.WithColumnsRenamed.RenameColumnsMapEntryR\x10renameColumnsMap\x1a\x43\n\x15RenameColumnsMapEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"w\n\x0bWithColumns\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x39\n\x07\x61liases\x18\x02 \x03(\x0b\x32\x1f.spark.connect.Expression.AliasR\x07\x61liases"\x84\x01\n\x04Hint\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x39\n\nparameters\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\nparameters"\xc7\x02\n\x07Unpivot\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12+\n\x03ids\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x03ids\x12:\n\x06values\x18\x03 \x01(\x0b\x32\x1d.spark.connect.Unpivot.ValuesH\x00R\x06values\x88\x01\x01\x12\x30\n\x14variable_column_name\x18\x04 \x01(\tR\x12variableColumnName\x12*\n\x11value_column_name\x18\x05 \x01(\tR\x0fvalueColumnName\x1a;\n\x06Values\x12\x31\n\x06values\x18\x01 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x06valuesB\t\n\x07_values"j\n\x08ToSchema\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12/\n\x06schema\x18\x02 \x01(\x0b\x32\x17.spark.connect.DataTypeR\x06schema"\xcb\x01\n\x17RepartitionByExpression\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x0fpartition_exprs\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x0epartitionExprs\x12*\n\x0enum_partitions\x18\x03 \x01(\x05H\x00R\rnumPartitions\x88\x01\x01\x42\x11\n\x0f_num_partitions"\x82\x01\n\rMapPartitions\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x42\n\x04\x66unc\x18\x02 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\xcb\x01\n\x08GroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12L\n\x14grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x13groupingExpressions\x12\x42\n\x04\x66unc\x18\x03 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\xe0\x02\n\nCoGroupMap\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12W\n\x1ainput_grouping_expressions\x18\x02 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18inputGroupingExpressions\x12-\n\x05other\x18\x03 \x01(\x0b\x32\x17.spark.connect.RelationR\x05other\x12W\n\x1aother_grouping_expressions\x18\x04 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x18otherGroupingExpressions\x12\x42\n\x04\x66unc\x18\x05 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionR\x04\x66unc"\x88\x01\n\x0e\x43ollectMetrics\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x12\n\x04name\x18\x02 \x01(\tR\x04name\x12\x33\n\x07metrics\x18\x03 \x03(\x0b\x32\x19.spark.connect.ExpressionR\x07metrics"\x84\x03\n\x05Parse\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12\x38\n\x06\x66ormat\x18\x02 \x01(\x0e\x32 .spark.connect.Parse.ParseFormatR\x06\x66ormat\x12\x34\n\x06schema\x18\x03 \x01(\x0b\x32\x17.spark.connect.DataTypeH\x00R\x06schema\x88\x01\x01\x12;\n\x07options\x18\x04 \x03(\x0b\x32!.spark.connect.Parse.OptionsEntryR\x07options\x1a:\n\x0cOptionsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value:\x02\x38\x01"X\n\x0bParseFormat\x12\x1c\n\x18PARSE_FORMAT_UNSPECIFIED\x10\x00\x12\x14\n\x10PARSE_FORMAT_CSV\x10\x01\x12\x15\n\x11PARSE_FORMAT_JSON\x10\x02\x42\t\n\x07_schemaB"\n\x1eorg.apache.spark.connect.protoP\x01\x62\x06proto3' ) @@ -728,117 +728,117 @@ _RELATIONCOMMON._serialized_start = 2785 _RELATIONCOMMON._serialized_end = 2876 _SQL._serialized_start = 2879 - _SQL._serialized_end = 3013 + _SQL._serialized_end = 3048 _SQL_ARGSENTRY._serialized_start = 2958 - _SQL_ARGSENTRY._serialized_end = 3013 - _READ._serialized_start = 3016 - _READ._serialized_end = 3644 - _READ_NAMEDTABLE._serialized_start = 3159 - _READ_NAMEDTABLE._serialized_end = 3351 - _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3293 - _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 3351 - _READ_DATASOURCE._serialized_start = 3354 - _READ_DATASOURCE._serialized_end = 3631 - _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3293 - _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3351 - _PROJECT._serialized_start = 3646 - _PROJECT._serialized_end = 3763 - _FILTER._serialized_start = 3765 - _FILTER._serialized_end = 3877 - _JOIN._serialized_start = 3880 - _JOIN._serialized_end = 4351 - _JOIN_JOINTYPE._serialized_start = 4143 - _JOIN_JOINTYPE._serialized_end = 4351 - _SETOPERATION._serialized_start = 4354 - _SETOPERATION._serialized_end = 4833 - _SETOPERATION_SETOPTYPE._serialized_start = 4670 - _SETOPERATION_SETOPTYPE._serialized_end = 4784 - _LIMIT._serialized_start = 4835 - _LIMIT._serialized_end = 4911 - _OFFSET._serialized_start = 4913 - _OFFSET._serialized_end = 4992 - _TAIL._serialized_start = 4994 - _TAIL._serialized_end = 5069 - _AGGREGATE._serialized_start = 5072 - _AGGREGATE._serialized_end = 5654 - _AGGREGATE_PIVOT._serialized_start = 5411 - _AGGREGATE_PIVOT._serialized_end = 5522 - _AGGREGATE_GROUPTYPE._serialized_start = 5525 - _AGGREGATE_GROUPTYPE._serialized_end = 5654 - _SORT._serialized_start = 5657 - _SORT._serialized_end = 5817 - _DROP._serialized_start = 5820 - _DROP._serialized_end = 5961 - _DEDUPLICATE._serialized_start = 5964 - _DEDUPLICATE._serialized_end = 6135 - _LOCALRELATION._serialized_start = 6137 - _LOCALRELATION._serialized_end = 6226 - _SAMPLE._serialized_start = 6229 - _SAMPLE._serialized_end = 6502 - _RANGE._serialized_start = 6505 - _RANGE._serialized_end = 6650 - _SUBQUERYALIAS._serialized_start = 6652 - _SUBQUERYALIAS._serialized_end = 6766 - _REPARTITION._serialized_start = 6769 - _REPARTITION._serialized_end = 6911 - _SHOWSTRING._serialized_start = 6914 - _SHOWSTRING._serialized_end = 7056 - _STATSUMMARY._serialized_start = 7058 - _STATSUMMARY._serialized_end = 7150 - _STATDESCRIBE._serialized_start = 7152 - _STATDESCRIBE._serialized_end = 7233 - _STATCROSSTAB._serialized_start = 7235 - _STATCROSSTAB._serialized_end = 7336 - _STATCOV._serialized_start = 7338 - _STATCOV._serialized_end = 7434 - _STATCORR._serialized_start = 7437 - _STATCORR._serialized_end = 7574 - _STATAPPROXQUANTILE._serialized_start = 7577 - _STATAPPROXQUANTILE._serialized_end = 7741 - _STATFREQITEMS._serialized_start = 7743 - _STATFREQITEMS._serialized_end = 7868 - _STATSAMPLEBY._serialized_start = 7871 - _STATSAMPLEBY._serialized_end = 8180 - _STATSAMPLEBY_FRACTION._serialized_start = 8072 - _STATSAMPLEBY_FRACTION._serialized_end = 8171 - _NAFILL._serialized_start = 8183 - _NAFILL._serialized_end = 8317 - _NADROP._serialized_start = 8320 - _NADROP._serialized_end = 8454 - _NAREPLACE._serialized_start = 8457 - _NAREPLACE._serialized_end = 8753 - _NAREPLACE_REPLACEMENT._serialized_start = 8612 - _NAREPLACE_REPLACEMENT._serialized_end = 8753 - _TODF._serialized_start = 8755 - _TODF._serialized_end = 8843 - _WITHCOLUMNSRENAMED._serialized_start = 8846 - _WITHCOLUMNSRENAMED._serialized_end = 9085 - _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 9018 - _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 9085 - _WITHCOLUMNS._serialized_start = 9087 - _WITHCOLUMNS._serialized_end = 9206 - _HINT._serialized_start = 9209 - _HINT._serialized_end = 9341 - _UNPIVOT._serialized_start = 9344 - _UNPIVOT._serialized_end = 9671 - _UNPIVOT_VALUES._serialized_start = 9601 - _UNPIVOT_VALUES._serialized_end = 9660 - _TOSCHEMA._serialized_start = 9673 - _TOSCHEMA._serialized_end = 9779 - _REPARTITIONBYEXPRESSION._serialized_start = 9782 - _REPARTITIONBYEXPRESSION._serialized_end = 9985 - _MAPPARTITIONS._serialized_start = 9988 - _MAPPARTITIONS._serialized_end = 10118 - _GROUPMAP._serialized_start = 10121 - _GROUPMAP._serialized_end = 10324 - _COGROUPMAP._serialized_start = 10327 - _COGROUPMAP._serialized_end = 10679 - _COLLECTMETRICS._serialized_start = 10682 - _COLLECTMETRICS._serialized_end = 10818 - _PARSE._serialized_start = 10821 - _PARSE._serialized_end = 11209 - _PARSE_OPTIONSENTRY._serialized_start = 3293 - _PARSE_OPTIONSENTRY._serialized_end = 3351 - _PARSE_PARSEFORMAT._serialized_start = 11110 - _PARSE_PARSEFORMAT._serialized_end = 11198 + _SQL_ARGSENTRY._serialized_end = 3048 + _READ._serialized_start = 3051 + _READ._serialized_end = 3679 + _READ_NAMEDTABLE._serialized_start = 3194 + _READ_NAMEDTABLE._serialized_end = 3386 + _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3328 + _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 3386 + _READ_DATASOURCE._serialized_start = 3389 + _READ_DATASOURCE._serialized_end = 3666 + _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3328 + _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 3386 + _PROJECT._serialized_start = 3681 + _PROJECT._serialized_end = 3798 + _FILTER._serialized_start = 3800 + _FILTER._serialized_end = 3912 + _JOIN._serialized_start = 3915 + _JOIN._serialized_end = 4386 + _JOIN_JOINTYPE._serialized_start = 4178 + _JOIN_JOINTYPE._serialized_end = 4386 + _SETOPERATION._serialized_start = 4389 + _SETOPERATION._serialized_end = 4868 + _SETOPERATION_SETOPTYPE._serialized_start = 4705 + _SETOPERATION_SETOPTYPE._serialized_end = 4819 + _LIMIT._serialized_start = 4870 + _LIMIT._serialized_end = 4946 + _OFFSET._serialized_start = 4948 + _OFFSET._serialized_end = 5027 + _TAIL._serialized_start = 5029 + _TAIL._serialized_end = 5104 + _AGGREGATE._serialized_start = 5107 + _AGGREGATE._serialized_end = 5689 + _AGGREGATE_PIVOT._serialized_start = 5446 + _AGGREGATE_PIVOT._serialized_end = 5557 + _AGGREGATE_GROUPTYPE._serialized_start = 5560 + _AGGREGATE_GROUPTYPE._serialized_end = 5689 + _SORT._serialized_start = 5692 + _SORT._serialized_end = 5852 + _DROP._serialized_start = 5855 + _DROP._serialized_end = 5996 + _DEDUPLICATE._serialized_start = 5999 + _DEDUPLICATE._serialized_end = 6170 + _LOCALRELATION._serialized_start = 6172 + _LOCALRELATION._serialized_end = 6261 + _SAMPLE._serialized_start = 6264 + _SAMPLE._serialized_end = 6537 + _RANGE._serialized_start = 6540 + _RANGE._serialized_end = 6685 + _SUBQUERYALIAS._serialized_start = 6687 + _SUBQUERYALIAS._serialized_end = 6801 + _REPARTITION._serialized_start = 6804 + _REPARTITION._serialized_end = 6946 + _SHOWSTRING._serialized_start = 6949 + _SHOWSTRING._serialized_end = 7091 + _STATSUMMARY._serialized_start = 7093 + _STATSUMMARY._serialized_end = 7185 + _STATDESCRIBE._serialized_start = 7187 + _STATDESCRIBE._serialized_end = 7268 + _STATCROSSTAB._serialized_start = 7270 + _STATCROSSTAB._serialized_end = 7371 + _STATCOV._serialized_start = 7373 + _STATCOV._serialized_end = 7469 + _STATCORR._serialized_start = 7472 + _STATCORR._serialized_end = 7609 + _STATAPPROXQUANTILE._serialized_start = 7612 + _STATAPPROXQUANTILE._serialized_end = 7776 + _STATFREQITEMS._serialized_start = 7778 + _STATFREQITEMS._serialized_end = 7903 + _STATSAMPLEBY._serialized_start = 7906 + _STATSAMPLEBY._serialized_end = 8215 + _STATSAMPLEBY_FRACTION._serialized_start = 8107 + _STATSAMPLEBY_FRACTION._serialized_end = 8206 + _NAFILL._serialized_start = 8218 + _NAFILL._serialized_end = 8352 + _NADROP._serialized_start = 8355 + _NADROP._serialized_end = 8489 + _NAREPLACE._serialized_start = 8492 + _NAREPLACE._serialized_end = 8788 + _NAREPLACE_REPLACEMENT._serialized_start = 8647 + _NAREPLACE_REPLACEMENT._serialized_end = 8788 + _TODF._serialized_start = 8790 + _TODF._serialized_end = 8878 + _WITHCOLUMNSRENAMED._serialized_start = 8881 + _WITHCOLUMNSRENAMED._serialized_end = 9120 + _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 9053 + _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 9120 + _WITHCOLUMNS._serialized_start = 9122 + _WITHCOLUMNS._serialized_end = 9241 + _HINT._serialized_start = 9244 + _HINT._serialized_end = 9376 + _UNPIVOT._serialized_start = 9379 + _UNPIVOT._serialized_end = 9706 + _UNPIVOT_VALUES._serialized_start = 9636 + _UNPIVOT_VALUES._serialized_end = 9695 + _TOSCHEMA._serialized_start = 9708 + _TOSCHEMA._serialized_end = 9814 + _REPARTITIONBYEXPRESSION._serialized_start = 9817 + _REPARTITIONBYEXPRESSION._serialized_end = 10020 + _MAPPARTITIONS._serialized_start = 10023 + _MAPPARTITIONS._serialized_end = 10153 + _GROUPMAP._serialized_start = 10156 + _GROUPMAP._serialized_end = 10359 + _COGROUPMAP._serialized_start = 10362 + _COGROUPMAP._serialized_end = 10714 + _COLLECTMETRICS._serialized_start = 10717 + _COLLECTMETRICS._serialized_end = 10853 + _PARSE._serialized_start = 10856 + _PARSE._serialized_end = 11244 + _PARSE_OPTIONSENTRY._serialized_start = 3328 + _PARSE_OPTIONSENTRY._serialized_end = 3386 + _PARSE_PARSEFORMAT._serialized_start = 11145 + _PARSE_PARSEFORMAT._serialized_end = 11233 # @@protoc_insertion_point(module_scope) diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi index f287a74034676..4dd1954f00f82 100644 --- a/python/pyspark/sql/connect/proto/relations_pb2.pyi +++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi @@ -560,13 +560,17 @@ class SQL(google.protobuf.message.Message): KEY_FIELD_NUMBER: builtins.int VALUE_FIELD_NUMBER: builtins.int key: builtins.str - value: builtins.str + @property + def value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal: ... def __init__( self, *, key: builtins.str = ..., - value: builtins.str = ..., + value: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ..., ) -> None: ... + def HasField( + self, field_name: typing_extensions.Literal["value", b"value"] + ) -> builtins.bool: ... def ClearField( self, field_name: typing_extensions.Literal["key", b"key", "value", b"value"] ) -> None: ... @@ -576,17 +580,20 @@ class SQL(google.protobuf.message.Message): query: builtins.str """(Required) The SQL query.""" @property - def args(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: - """(Optional) A map of parameter names to string values that are parsed as - SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - belonged to SQL comments are skipped while parsing. - """ + def args( + self, + ) -> google.protobuf.internal.containers.MessageMap[ + builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal + ]: + """(Optional) A map of parameter names to literal expressions.""" def __init__( self, *, query: builtins.str = ..., - args: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., + args: collections.abc.Mapping[ + builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal + ] + | None = ..., ) -> None: ... def ClearField( self, field_name: typing_extensions.Literal["args", b"args", "query", b"query"] diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py index b49f6df969c2a..2fa32a0096eae 100644 --- a/python/pyspark/sql/connect/session.py +++ b/python/pyspark/sql/connect/session.py @@ -382,7 +382,7 @@ def createDataFrame( createDataFrame.__doc__ = PySparkSession.createDataFrame.__doc__ - def sql(self, sqlQuery: str, args: Optional[Dict[str, str]] = None) -> "DataFrame": + def sql(self, sqlQuery: str, args: Optional[Dict[str, Any]] = None) -> "DataFrame": cmd = SQL(sqlQuery, args) data, properties = self.client.execute_command(cmd.command(self._client)) if "sql_command_result" in properties: diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 8818092e5b507..17a85876e288c 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -41,8 +41,10 @@ from pyspark import SparkConf, SparkContext from pyspark.rdd import RDD +from pyspark.sql.column import _to_java_column from pyspark.sql.conf import RuntimeConfig from pyspark.sql.dataframe import DataFrame +from pyspark.sql.functions import lit from pyspark.sql.pandas.conversion import SparkConversionMixin from pyspark.sql.readwriter import DataFrameReader from pyspark.sql.sql_formatter import SQLStringFormatter @@ -1321,7 +1323,7 @@ def prepare(obj: Any) -> Any: df._schema = struct return df - def sql(self, sqlQuery: str, args: Optional[Dict[str, str]] = None, **kwargs: Any) -> DataFrame: + def sql(self, sqlQuery: str, args: Optional[Dict[str, Any]] = None, **kwargs: Any) -> DataFrame: """Returns a :class:`DataFrame` representing the result of the given query. When ``kwargs`` is specified, this method formats the given string by using the Python standard formatter. The method binds named parameters to SQL literals from `args`. @@ -1336,10 +1338,13 @@ def sql(self, sqlQuery: str, args: Optional[Dict[str, str]] = None, **kwargs: An sqlQuery : str SQL query string. args : dict - A dictionary of parameter names to string values that are parsed as SQL literal - expressions. For example, dict keys: "rank", "name", "birthdate"; dict values: - "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values belonged to - SQL comments are skipped while parsing. + A dictionary of parameter names to Python objects that can be converted to + SQL literal expressions. See + + Supported Data Types for supported value types in Python. + For example, dictionary keys: "rank", "name", "birthdate"; + dictionary values: 1, "Steven", datetime.date(2023, 4, 2). + Map value can be also a `Column` of literal expression, in that case it is taken as is. .. versionadded:: 3.4.0 @@ -1419,7 +1424,7 @@ def sql(self, sqlQuery: str, args: Optional[Dict[str, str]] = None, **kwargs: An And substitude named parameters with the `:` prefix by SQL literals. - >>> spark.sql("SELECT * FROM {df} WHERE {df[B]} > :minB", {"minB" : "5"}, df=mydf).show() + >>> spark.sql("SELECT * FROM {df} WHERE {df[B]} > :minB", {"minB" : 5}, df=mydf).show() +---+---+ | A| B| +---+---+ @@ -1431,7 +1436,8 @@ def sql(self, sqlQuery: str, args: Optional[Dict[str, str]] = None, **kwargs: An if len(kwargs) > 0: sqlQuery = formatter.format(sqlQuery, **kwargs) try: - return DataFrame(self._jsparkSession.sql(sqlQuery, args or {}), self) + litArgs = {k: _to_java_column(lit(v)) for k, v in (args or {}).items()} + return DataFrame(self._jsparkSession.sql(sqlQuery, litArgs), self) finally: if len(kwargs) > 0: formatter.clear() diff --git a/python/pyspark/sql/tests/connect/test_connect_basic.py b/python/pyspark/sql/tests/connect/test_connect_basic.py index 8e3a12dc678fd..008b95d6f1437 100644 --- a/python/pyspark/sql/tests/connect/test_connect_basic.py +++ b/python/pyspark/sql/tests/connect/test_connect_basic.py @@ -1177,8 +1177,8 @@ def test_sql(self): self.assertEqual(1, len(pdf.index)) def test_sql_with_args(self): - df = self.connect.sql("SELECT * FROM range(10) WHERE id > :minId", args={"minId": "7"}) - df2 = self.spark.sql("SELECT * FROM range(10) WHERE id > :minId", args={"minId": "7"}) + df = self.connect.sql("SELECT * FROM range(10) WHERE id > :minId", args={"minId": 7}) + df2 = self.spark.sql("SELECT * FROM range(10) WHERE id > :minId", args={"minId": 7}) self.assert_eq(df.toPandas(), df2.toPandas()) def test_head(self): diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala index f9411a10fa6f9..c595b50950bcf 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SparkSession.scala @@ -45,6 +45,7 @@ import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution._ import org.apache.spark.sql.execution.command.ExternalCommandExecutor import org.apache.spark.sql.execution.datasources.{DataSource, LogicalRelation} +import org.apache.spark.sql.functions.lit import org.apache.spark.sql.internal._ import org.apache.spark.sql.internal.StaticSQLConf.CATALOG_IMPLEMENTATION import org.apache.spark.sql.sources.BaseRelation @@ -614,21 +615,24 @@ class SparkSession private( * This API eagerly runs DDL/DML commands, but not for SELECT queries. * * @param sqlText A SQL statement with named parameters to execute. - * @param args A map of parameter names to string values that are parsed as - * SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - * map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - * belonged to SQL comments are skipped while parsing. + * @param args A map of parameter names to Java/Scala objects that can be converted to + * SQL literal expressions. See + * + * Supported Data Types for supported value types in Scala/Java. + * For example, map keys: "rank", "name", "birthdate"; + * map values: 1, "Steven", LocalDate.of(2023, 4, 2). + * Map value can be also a `Column` of literal expression, in that case + * it is taken as is. * * @since 3.4.0 */ @Experimental - def sql(sqlText: String, args: Map[String, String]): DataFrame = withActive { + def sql(sqlText: String, args: Map[String, Any]): DataFrame = withActive { val tracker = new QueryPlanningTracker val plan = tracker.measurePhase(QueryPlanningTracker.PARSING) { - val parser = sessionState.sqlParser - val parsedPlan = parser.parsePlan(sqlText) + val parsedPlan = sessionState.sqlParser.parsePlan(sqlText) if (args.nonEmpty) { - ParameterizedQuery(parsedPlan, args.mapValues(parser.parseExpression).toMap) + ParameterizedQuery(parsedPlan, args.mapValues(lit(_).expr).toMap) } else { parsedPlan } @@ -642,15 +646,19 @@ class SparkSession private( * This API eagerly runs DDL/DML commands, but not for SELECT queries. * * @param sqlText A SQL statement with named parameters to execute. - * @param args A map of parameter names to string values that are parsed as - * SQL literal expressions. For example, map keys: "rank", "name", "birthdate"; - * map values: "1", "'Steven'", "DATE'2023-03-21'". The fragments of string values - * belonged to SQL comments are skipped while parsing. + * @param args A map of parameter names to Java/Scala objects that can be converted to + * SQL literal expressions. See + * + * Supported Data Types for supported value types in Scala/Java. + * For example, map keys: "rank", "name", "birthdate"; + * map values: 1, "Steven", LocalDate.of(2023, 4, 2). + * Map value can be also a `Column` of literal expression, in that case + * it is taken as is. * * @since 3.4.0 */ @Experimental - def sql(sqlText: String, args: java.util.Map[String, String]): DataFrame = { + def sql(sqlText: String, args: java.util.Map[String, Any]): DataFrame = { sql(sqlText, args.asScala.toMap) } @@ -660,7 +668,7 @@ class SparkSession private( * * @since 2.0.0 */ - def sql(sqlText: String): DataFrame = sql(sqlText, Map.empty[String, String]) + def sql(sqlText: String): DataFrame = sql(sqlText, Map.empty[String, Any]) /** * Execute an arbitrary string command inside an external execution engine rather than Spark. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ParametersSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ParametersSuite.scala index e6e5eb9fac4fd..074630492395f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ParametersSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ParametersSuite.scala @@ -17,6 +17,10 @@ package org.apache.spark.sql +import java.time.{Instant, LocalDate, LocalDateTime, ZoneId} + +import org.apache.spark.sql.functions.lit +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSparkSession class ParametersSuite extends QueryTest with SharedSparkSession { @@ -28,25 +32,25 @@ class ParametersSuite extends QueryTest with SharedSparkSession { |FROM VALUES (0), (1), (2), (3), (4), (5), (6), (7), (8), (9) AS t(id) |WHERE id < :constA |""".stripMargin - val args = Map("div" -> "3", "constA" -> "4L") + val args = Map("div" -> 3, "constA" -> 4L) checkAnswer( spark.sql(sqlText, args), Row(0, 0) :: Row(1, 1) :: Row(2, 2) :: Row(3, 0) :: Nil) checkAnswer( - spark.sql("""SELECT contains('Spark \'SQL\'', :subStr)""", Map("subStr" -> "'SQL'")), + spark.sql("""SELECT contains('Spark \'SQL\'', :subStr)""", Map("subStr" -> "SQL")), Row(true)) } test("parameter binding is case sensitive") { checkAnswer( - spark.sql("SELECT :p, :P", Map("p" -> "1", "P" -> "2")), + spark.sql("SELECT :p, :P", Map("p" -> 1, "P" -> 2)), Row(1, 2) ) checkError( exception = intercept[AnalysisException] { - spark.sql("select :P", Map("p" -> "1")) + spark.sql("select :P", Map("p" -> 1)) }, errorClass = "UNBOUND_SQL_PARAMETER", parameters = Map("name" -> "P"), @@ -62,7 +66,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { |WITH w1 AS (SELECT :p1 AS p) |SELECT p + :p2 FROM w1 |""".stripMargin - val args = Map("p1" -> "1", "p2" -> "2") + val args = Map("p1" -> 1, "p2" -> 2) checkAnswer( spark.sql(sqlText, args), Row(3)) @@ -75,7 +79,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { | (WITH w2 AS (SELECT :p1 AS p) SELECT p + :p2 AS p2 FROM w2) |SELECT p2 + :p3 FROM w1 |""".stripMargin - val args = Map("p1" -> "1", "p2" -> "2", "p3" -> "3") + val args = Map("p1" -> 1, "p2" -> 2, "p3" -> 3) checkAnswer( spark.sql(sqlText, args), Row(6)) @@ -83,7 +87,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { test("parameters in subquery expression") { val sqlText = "SELECT (SELECT max(id) + :p1 FROM range(10)) + :p2" - val args = Map("p1" -> "1", "p2" -> "2") + val args = Map("p1" -> 1, "p2" -> 2) checkAnswer( spark.sql(sqlText, args), Row(12)) @@ -91,7 +95,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { test("parameters in nested subquery expression") { val sqlText = "SELECT (SELECT (SELECT max(id) + :p1 FROM range(10)) + :p2) + :p3" - val args = Map("p1" -> "1", "p2" -> "2", "p3" -> "3") + val args = Map("p1" -> 1, "p2" -> 2, "p3" -> 3) checkAnswer( spark.sql(sqlText, args), Row(15)) @@ -103,7 +107,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { |WITH w1 AS (SELECT (SELECT max(id) + :p1 FROM range(10)) + :p2 AS p) |SELECT p + :p3 FROM w1 |""".stripMargin - val args = Map("p1" -> "1", "p2" -> "2", "p3" -> "3") + val args = Map("p1" -> 1, "p2" -> 2, "p3" -> 3) checkAnswer( spark.sql(sqlText, args), Row(15)) @@ -112,14 +116,14 @@ class ParametersSuite extends QueryTest with SharedSparkSession { test("parameters in INSERT") { withTable("t") { sql("CREATE TABLE t (col INT) USING json") - spark.sql("INSERT INTO t SELECT :p", Map("p" -> "1")) + spark.sql("INSERT INTO t SELECT :p", Map("p" -> 1)) checkAnswer(spark.table("t"), Row(1)) } } test("parameters not allowed in DDL commands") { val sqlText = "CREATE VIEW v AS SELECT :p AS p" - val args = Map("p" -> "1") + val args = Map("p" -> 1) checkError( exception = intercept[AnalysisException] { spark.sql(sqlText, args) @@ -135,7 +139,7 @@ class ParametersSuite extends QueryTest with SharedSparkSession { test("non-substituted parameters") { checkError( exception = intercept[AnalysisException] { - spark.sql("select :abc, :def", Map("abc" -> "1")) + spark.sql("select :abc, :def", Map("abc" -> 1)) }, errorClass = "UNBOUND_SQL_PARAMETER", parameters = Map("name" -> "def"), @@ -155,18 +159,42 @@ class ParametersSuite extends QueryTest with SharedSparkSession { stop = 10)) } - test("non-literal argument of `sql()`") { - Seq("col1 + 1", "CAST('100' AS INT)", "map('a', 1, 'b', 2)", "array(1)").foreach { arg => - checkError( - exception = intercept[AnalysisException] { - spark.sql("SELECT :param1 FROM VALUES (1) AS t(col1)", Map("param1" -> arg)) - }, - errorClass = "INVALID_SQL_ARG", - parameters = Map("name" -> "param1"), - context = ExpectedContext( - fragment = arg, - start = 0, - stop = arg.length - 1)) + test("literal argument of `sql()`") { + val sqlText = + """SELECT s FROM VALUES ('Jeff /*__*/ Green'), ('E\'Twaun Moore'), ('Vander Blue') AS t(s) + |WHERE s = :player_name""".stripMargin + checkAnswer( + spark.sql(sqlText, args = Map("player_name" -> lit("E'Twaun Moore"))), + Row("E'Twaun Moore") :: Nil) + checkAnswer( + spark.sql(sqlText, args = Map("player_name" -> lit("Vander Blue--comment"))), + Nil) + checkAnswer( + spark.sql(sqlText, args = Map("player_name" -> lit("Jeff /*__*/ Green"))), + Row("Jeff /*__*/ Green") :: Nil) + + withSQLConf(SQLConf.DATETIME_JAVA8API_ENABLED.key -> "true") { + checkAnswer( + spark.sql( + sqlText = """ + |SELECT d + |FROM VALUES (DATE'1970-01-01'), (DATE'2023-12-31') AS t(d) + |WHERE d < :currDate + |""".stripMargin, + args = Map("currDate" -> lit(LocalDate.of(2023, 4, 1)))), + Row(LocalDate.of(1970, 1, 1)) :: Nil) + checkAnswer( + spark.sql( + sqlText = """ + |SELECT d + |FROM VALUES (TIMESTAMP_LTZ'1970-01-01 01:02:03 Europe/Amsterdam'), + | (TIMESTAMP_LTZ'2023-12-31 04:05:06 America/Los_Angeles') AS t(d) + |WHERE d < :currDate + |""".stripMargin, + args = Map("currDate" -> lit(Instant.parse("2023-04-01T00:00:00Z")))), + Row(LocalDateTime.of(1970, 1, 1, 1, 2, 3) + .atZone(ZoneId.of("Europe/Amsterdam")) + .toInstant) :: Nil) } } }