diff --git a/core/src/main/java/org/apache/iceberg/ContentFileParser.java b/core/src/main/java/org/apache/iceberg/ContentFileParser.java index 0033fa97725a..f024a24b18ce 100644 --- a/core/src/main/java/org/apache/iceberg/ContentFileParser.java +++ b/core/src/main/java/org/apache/iceberg/ContentFileParser.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.util.List; +import java.util.Locale; import java.util.Map; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.types.Types; @@ -50,6 +51,9 @@ public class ContentFileParser { private static final String REFERENCED_DATA_FILE = "referenced-data-file"; private static final String CONTENT_OFFSET = "content-offset"; private static final String CONTENT_SIZE = "content-size-in-bytes"; + private static final String CONTENT_DATA = "data"; + private static final String CONTENT_POSITION_DELETES = "position-deletes"; + private static final String CONTENT_EQUALITY_DELETES = "equality-deletes"; private ContentFileParser() {} @@ -84,9 +88,13 @@ public static void toJson(ContentFile contentFile, PartitionSpec spec, JsonGe // as it isn't used and BaseFile constructor doesn't support it. generator.writeNumberField(SPEC_ID, contentFile.specId()); - generator.writeStringField(CONTENT, contentFile.content().name()); + // Since 1.11, we serialize content as lowercase kebab-case values like "equality-deletes" + generator.writeStringField( + CONTENT, contentFile.content().name().toLowerCase(Locale.ENGLISH).replace('_', '-')); generator.writeStringField(FILE_PATH, contentFile.location()); - generator.writeStringField(FILE_FORMAT, contentFile.format().name()); + // Since 1.11, we serialize format as lower-case strings like "parquet" + generator.writeStringField( + FILE_FORMAT, contentFile.format().name().toLowerCase(Locale.ENGLISH)); if (contentFile.partition() != null) { generator.writeFieldName(PARTITION); @@ -147,7 +155,7 @@ public static ContentFile fromJson(JsonNode jsonNode, Map ContentFileParser.fromJson(node, Map.of(0, PartitionSpec.unpartitioned()))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Invalid file content value: 'invalid-content'"); + } + + @Test + public void testUppercaseFileFormat() throws Exception { + String jsonStr = + "{\"spec-id\":0," + + "\"content\":\"data\"," + + "\"file-path\":\"/path/to/file.parquet\"," + + "\"file-format\":\"PARQUET\"," + + "\"partition\":{}," + + "\"file-size-in-bytes\":1," + + "\"record-count\":1}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserializedContentFile = + ContentFileParser.fromJson(jsonNode, Map.of(0, PartitionSpec.unpartitioned())); + assertThat(deserializedContentFile.format()).isEqualTo(FileFormat.PARQUET); + } + + @ParameterizedTest + @MethodSource("enumContentTypeCases") + public void testEnumContentTypeSerialization(FileContent content, String expectedJsonContent) + throws Exception { + String jsonStr = + "{\"spec-id\":0," + + "\"content\":\"" + + content.name() + + "\"," + + "\"file-path\":\"/path/to/data.parquet\"," + + "\"file-format\":\"parquet\"," + + "\"partition\":{}," + + "\"file-size-in-bytes\":1," + + "\"record-count\":1}"; + + JsonNode jsonNode = JsonUtil.mapper().readTree(jsonStr); + ContentFile deserializedContentFile = + ContentFileParser.fromJson(jsonNode, Map.of(0, PartitionSpec.unpartitioned())); + assertThat(deserializedContentFile.content()).isEqualTo(content); + + String serializedStr = + ContentFileParser.toJson(deserializedContentFile, PartitionSpec.unpartitioned()); + assertThat(serializedStr).contains("\"content\":\"" + expectedJsonContent + "\""); + } + + private static Stream enumContentTypeCases() { + return Stream.of( + Arguments.of(FileContent.DATA, "data"), + Arguments.of(FileContent.POSITION_DELETES, "position-deletes"), + Arguments.of(FileContent.EQUALITY_DELETES, "equality-deletes")); + } + private static Stream provideSpecAndDataFile() { return Stream.of( Arguments.of( @@ -271,18 +339,18 @@ private static DataFile dataFileWithOnlyNanCounts(PartitionSpec spec) { private static String dataFileJsonWithRequiredOnly(PartitionSpec spec) { if (spec.isUnpartitioned()) { - return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\"," + return "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}"; } else { - return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"PARQUET\"," + return "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[1],\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}"; } } private static String dataFileJsonWithAllOptional(PartitionSpec spec) { if (spec.isUnpartitioned()) { - return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":350,\"record-count\":10," + return "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-with-stats.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":350,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," @@ -292,8 +360,8 @@ private static String dataFileJsonWithAllOptional(PartitionSpec spec) { + "\"key-metadata\":\"00000000000000000000000000000000\"," + "\"split-offsets\":[128,256],\"sort-order-id\":1}"; } else { - return "{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[1],\"file-size-in-bytes\":350,\"record-count\":10," + return "{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-with-stats.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[1],\"file-size-in-bytes\":350,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," @@ -388,8 +456,8 @@ private static DeleteFile deleteFileWithDataRef(PartitionSpec spec) { } private static String deleteFileWithDataRefJson() { - return "{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[4],\"file-size-in-bytes\":1234," + return "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[4],\"file-size-in-bytes\":1234," + "\"record-count\":10,\"referenced-data-file\":\"/path/to/data/file.parquet\"}"; } @@ -414,8 +482,8 @@ private static DeleteFile dv(PartitionSpec spec) { } private static String dvJson() { - return "{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete.puffin\"," - + "\"file-format\":\"PUFFIN\",\"partition\":[4],\"file-size-in-bytes\":1234,\"record-count\":10," + return "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete.puffin\"," + + "\"file-format\":\"puffin\",\"partition\":[4],\"file-size-in-bytes\":1234,\"record-count\":10," + "\"referenced-data-file\":\"/path/to/data/file.parquet\",\"content-offset\":4,\"content-size-in-bytes\":40}"; } @@ -487,18 +555,18 @@ private static DeleteFile deleteFileWithAllOptional(PartitionSpec spec) { private static String deleteFileJsonWithRequiredOnly(PartitionSpec spec) { if (spec.isUnpartitioned()) { - return "{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":9}"; + return "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":9}"; } else { - return "{\"spec-id\":0,\"content\":\"POSITION_DELETES\",\"file-path\":\"/path/to/delete-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":9}"; + return "{\"spec-id\":0,\"content\":\"position-deletes\",\"file-path\":\"/path/to/delete-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":9}"; } } private static String deleteFileJsonWithAllOptional(PartitionSpec spec) { if (spec.isUnpartitioned()) { - return "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":10," + return "{\"spec-id\":0,\"content\":\"equality-deletes\",\"file-path\":\"/path/to/delete-with-stats.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[],\"file-size-in-bytes\":1234,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," @@ -508,8 +576,8 @@ private static String deleteFileJsonWithAllOptional(PartitionSpec spec) { + "\"key-metadata\":\"00000000000000000000000000000000\"," + "\"split-offsets\":[128],\"equality-ids\":[3],\"sort-order-id\":1}"; } else { - return "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/delete-with-stats.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":10," + return "{\"spec-id\":0,\"content\":\"equality-deletes\",\"file-path\":\"/path/to/delete-with-stats.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[9],\"file-size-in-bytes\":1234,\"record-count\":10," + "\"column-sizes\":{\"keys\":[3,4],\"values\":[100,200]}," + "\"value-counts\":{\"keys\":[3,4],\"values\":[90,180]}," + "\"null-value-counts\":{\"keys\":[3,4],\"values\":[10,20]}," diff --git a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java index 758c11db788a..03065abe8744 100644 --- a/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java +++ b/core/src/test/java/org/apache/iceberg/TestDataTaskParser.java @@ -144,9 +144,9 @@ public void missingFields() throws Exception { + "{\"id\":6,\"name\":\"summary\",\"required\":false,\"type\":{\"type\":\"map\"," + "\"key-id\":7,\"key\":\"string\",\"value-id\":8," + "\"value\":\"string\",\"value-required\":true}}]}," - + "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\"," + + "\"metadata-file\":{\"spec-id\":0,\"content\":\"data\"," + "\"file-path\":\"/tmp/metadata2.json\"," - + "\"file-format\":\"METADATA\",\"partition\":[]," + + "\"file-format\":\"metadata\",\"partition\":[]," + "\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0}" + "}"; JsonNode missingTableRowsNode = mapper.reader().readTree(missingTableRowsStr); @@ -172,8 +172,8 @@ public void testDataTaskParsesFieldIdPartitionMap() { + "\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]}," + "\"projection\":{\"type\":\"struct\",\"schema-id\":0," + "\"fields\":[{\"id\":1,\"name\":\"committed_at\",\"required\":true,\"type\":\"timestamptz\"}]}," - + "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/tmp/metadata.json\"," - + "\"file-format\":\"METADATA\",\"partition\":{},\"file-size-in-bytes\":0,\"record-count\":1,\"sort-order-id\":0}," + + "\"metadata-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/tmp/metadata.json\"," + + "\"file-format\":\"metadata\",\"partition\":{},\"file-size-in-bytes\":0,\"record-count\":1,\"sort-order-id\":0}," + "\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\"}]}"; StaticDataTask deserializedTask = (StaticDataTask) ScanTaskParser.fromJson(jsonStr, true); @@ -263,9 +263,9 @@ private String snapshotsDataTaskJson() { + "{\"id\":6,\"name\":\"summary\",\"required\":false,\"type\":{\"type\":\"map\"," + "\"key-id\":7,\"key\":\"string\",\"value-id\":8," + "\"value\":\"string\",\"value-required\":true}}]}," - + "\"metadata-file\":{\"spec-id\":0,\"content\":\"DATA\"," + + "\"metadata-file\":{\"spec-id\":0,\"content\":\"data\"," + "\"file-path\":\"/tmp/metadata2.json\"," - + "\"file-format\":\"METADATA\",\"partition\":[]," + + "\"file-format\":\"metadata\",\"partition\":[]," + "\"file-size-in-bytes\":0,\"record-count\":2,\"sort-order-id\":0}," + "\"rows\":[{\"1\":\"2009-02-13T23:31:30+00:00\",\"2\":1,\"4\":\"append\"," + "\"5\":\"file:/tmp/manifest1.avro\"," diff --git a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java index 882c2b33496d..1eff487d86cf 100644 --- a/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java +++ b/core/src/test/java/org/apache/iceberg/TestFileScanTaskParser.java @@ -96,15 +96,15 @@ private String fileScanTaskJsonWithoutTaskType() { + "{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]}," + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\"," + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]}," - + "\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"start\":0,\"length\":10," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}," - + "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10," + + "{\"spec-id\":0,\"content\":\"equality-deletes\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0],\"file-size-in-bytes\":10," + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}"; } @@ -116,15 +116,15 @@ private String fileScanTaskJson() { + "{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]}," + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\"," + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]}," - + "\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"start\":0,\"length\":10," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}," - + "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0],\"file-size-in-bytes\":10," + + "{\"spec-id\":0,\"content\":\"equality-deletes\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0],\"file-size-in-bytes\":10," + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}"; } @@ -136,15 +136,15 @@ private String fileScanTaskFieldIdPartitionMapJson() { + "{\"id\":4,\"name\":\"data\",\"required\":true,\"type\":\"string\"}]}," + "\"spec\":{\"spec-id\":0,\"fields\":[{\"name\":\"data_bucket\"," + "\"transform\":\"bucket[16]\",\"source-id\":4,\"field-id\":1000}]}," - + "\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0}," + + "\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":{\"1000\":0}," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"start\":0,\"length\":10," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":{\"1000\":0},\"file-size-in-bytes\":10,\"record-count\":1}," - + "{\"spec-id\":0,\"content\":\"EQUALITY_DELETES\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":{\"1000\":0},\"file-size-in-bytes\":10," + + "{\"spec-id\":0,\"content\":\"equality-deletes\",\"file-path\":\"/path/to/data-a2-deletes.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":{\"1000\":0},\"file-size-in-bytes\":10," + "\"record-count\":1,\"equality-ids\":[1],\"sort-order-id\":0}]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}"; } diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java index 8b82e9794d5d..7c1021d4afab 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchPlanningResultResponseParser.java @@ -155,8 +155,8 @@ public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileSca String invalidJson = "{\"status\":\"submitted\"," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]" + "}"; @@ -193,12 +193,12 @@ public void roundTripSerdeWithValidStatusAndFileScanTasks() throws JsonProcessin String expectedToJson = "{\"status\":\"completed\"," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]," + "\"file-scan-tasks\":[" - + "{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "{\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"delete-file-references\":[0]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]" diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java index 62de75e3818b..d7824bc6a673 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestFetchScanTasksResponseParser.java @@ -95,8 +95,8 @@ public void roundTripSerdeWithDeleteFilesNoFileScanTasksPresent() { String invalidJson = "{\"plan-tasks\":[\"task1\",\"task2\"]," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]" + "}"; @@ -129,12 +129,12 @@ public void roundTripSerdeWithFileScanTasks() { String expectedToJson = "{" - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]," + "\"file-scan-tasks\":[" - + "{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "{\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"delete-file-references\":[0]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]" diff --git a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java index a59bbbd16b45..e2c9f21dabba 100644 --- a/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java +++ b/core/src/test/java/org/apache/iceberg/rest/responses/TestPlanTableScanResponseParser.java @@ -210,8 +210,8 @@ public void roundTripSerdeWithInvalidPlanStatusSubmittedWithDeleteFilesNoFileSca String invalidJson = "{\"status\":\"submitted\"," + "\"plan-id\":\"somePlanId\"," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]" + "}"; @@ -244,12 +244,12 @@ public void roundTripSerdeWithValidStatusAndFileScanTasks() { String expectedToJson = "{\"status\":\"completed\"," - + "\"delete-files\":[{\"spec-id\":0,\"content\":\"POSITION_DELETES\"," - + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"PARQUET\"," + + "\"delete-files\":[{\"spec-id\":0,\"content\":\"position-deletes\"," + + "\"file-path\":\"/path/to/data-a-deletes.parquet\",\"file-format\":\"parquet\"," + "\"partition\":[0],\"file-size-in-bytes\":10,\"record-count\":1}]," + "\"file-scan-tasks\":[" - + "{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "{\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"delete-file-references\":[0]," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]" @@ -316,25 +316,25 @@ public void multipleTasksWithDifferentDeleteFilesDontAccumulateReferences() { + " \"status\" : \"completed\",\n" + " \"delete-files\" : [ {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"POSITION_DELETES\",\n" + + " \"content\" : \"position-deletes\",\n" + " \"file-path\" : \"/path/to/data-a-deletes.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 0 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1\n" + " }, {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"POSITION_DELETES\",\n" + + " \"content\" : \"position-deletes\",\n" + " \"file-path\" : \"/path/to/data-b-deletes.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 1 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1\n" + " }, {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"EQUALITY_DELETES\",\n" + + " \"content\" : \"equality-deletes\",\n" + " \"file-path\" : \"/path/to/data-c-deletes.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 2 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1,\n" @@ -344,9 +344,9 @@ public void multipleTasksWithDifferentDeleteFilesDontAccumulateReferences() { + " \"file-scan-tasks\" : [ {\n" + " \"data-file\" : {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"DATA\",\n" + + " \"content\" : \"data\",\n" + " \"file-path\" : \"/path/to/data-a.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 0 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1,\n" @@ -357,9 +357,9 @@ public void multipleTasksWithDifferentDeleteFilesDontAccumulateReferences() { + " }, {\n" + " \"data-file\" : {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"DATA\",\n" + + " \"content\" : \"data\",\n" + " \"file-path\" : \"/path/to/data-b.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 1 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1,\n" @@ -371,9 +371,9 @@ public void multipleTasksWithDifferentDeleteFilesDontAccumulateReferences() { + " }, {\n" + " \"data-file\" : {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"DATA\",\n" + + " \"content\" : \"data\",\n" + " \"file-path\" : \"/path/to/data-c.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 2 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1,\n" @@ -409,8 +409,8 @@ public void roundTripSerdeWithoutDeleteFiles() { String expectedJson = "{\"status\":\"completed\"," + "\"file-scan-tasks\":[" - + "{\"data-file\":{\"spec-id\":0,\"content\":\"DATA\",\"file-path\":\"/path/to/data-a.parquet\"," - + "\"file-format\":\"PARQUET\",\"partition\":[0]," + + "{\"data-file\":{\"spec-id\":0,\"content\":\"data\",\"file-path\":\"/path/to/data-a.parquet\"," + + "\"file-format\":\"parquet\",\"partition\":[0]," + "\"file-size-in-bytes\":10,\"record-count\":1,\"sort-order-id\":0}," + "\"residual-filter\":{\"type\":\"eq\",\"term\":\"id\",\"value\":1}}]" + "}"; @@ -608,9 +608,9 @@ public void roundTripSerdeWithValidStatusAndFileScanTasksAndCredentials() { + " } ],\n" + " \"delete-files\" : [ {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"POSITION_DELETES\",\n" + + " \"content\" : \"position-deletes\",\n" + " \"file-path\" : \"/path/to/data-a-deletes.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 0 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1\n" @@ -618,9 +618,9 @@ public void roundTripSerdeWithValidStatusAndFileScanTasksAndCredentials() { + " \"file-scan-tasks\" : [ {\n" + " \"data-file\" : {\n" + " \"spec-id\" : 0,\n" - + " \"content\" : \"DATA\",\n" + + " \"content\" : \"data\",\n" + " \"file-path\" : \"/path/to/data-a.parquet\",\n" - + " \"file-format\" : \"PARQUET\",\n" + + " \"file-format\" : \"parquet\",\n" + " \"partition\" : [ 0 ],\n" + " \"file-size-in-bytes\" : 10,\n" + " \"record-count\" : 1,\n"