From e2f3e01f14812a0cac876e470f87d64ad83a910a Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 20:44:24 -0700 Subject: [PATCH 1/8] rename spark to spark-common --- build.gradle | 19 +++++++++++-------- settings.gradle | 4 ++-- .../iceberg/spark/SparkBenchmarkUtil.java | 0 .../SparkParquetReadersFlatDataBenchmark.java | 0 ...parkParquetReadersNestedDataBenchmark.java | 0 .../SparkParquetWritersFlatDataBenchmark.java | 0 ...parkParquetWritersNestedDataBenchmark.java | 0 .../apache/iceberg/spark/source/Action.java | 0 .../spark/source/IcebergSourceBenchmark.java | 0 .../IcebergSourceFlatDataBenchmark.java | 0 .../IcebergSourceNestedDataBenchmark.java | 0 .../IcebergSourceNestedListDataBenchmark.java | 0 .../spark/source/WritersBenchmark.java | 0 .../source/avro/AvroWritersBenchmark.java | 0 ...cebergSourceFlatAvroDataReadBenchmark.java | 0 ...bergSourceNestedAvroDataReadBenchmark.java | 0 .../IcebergSourceFlatORCDataBenchmark.java | 0 ...IcebergSourceFlatORCDataReadBenchmark.java | 0 ...SourceNestedListORCDataWriteBenchmark.java | 0 ...ebergSourceNestedORCDataReadBenchmark.java | 0 ...gSourceFlatParquetDataFilterBenchmark.java | 0 ...ergSourceFlatParquetDataReadBenchmark.java | 0 ...rgSourceFlatParquetDataWriteBenchmark.java | 0 ...ceNestedListParquetDataWriteBenchmark.java | 0 ...ourceNestedParquetDataFilterBenchmark.java | 0 ...gSourceNestedParquetDataReadBenchmark.java | 0 ...SourceNestedParquetDataWriteBenchmark.java | 0 .../parquet/ParquetWritersBenchmark.java | 0 ...ionaryEncodedFlatParquetDataBenchmark.java | 0 ...ectorizedReadFlatParquetDataBenchmark.java | 0 .../org/apache/iceberg/actions/Actions.java | 0 .../apache/iceberg/actions/CreateAction.java | 0 .../actions/ExpireSnapshotsAction.java | 0 .../actions/ExpireSnapshotsActionResult.java | 0 .../iceberg/actions/ManifestFileBean.java | 0 .../actions/RemoveOrphanFilesAction.java | 0 .../actions/RewriteDataFilesAction.java | 0 .../actions/RewriteManifestsAction.java | 0 .../actions/RewriteManifestsActionResult.java | 0 .../iceberg/actions/SnapshotAction.java | 0 .../apache/iceberg/spark/IcebergSpark.java | 0 .../apache/iceberg/spark/JobGroupInfo.java | 0 .../apache/iceberg/spark/JobGroupUtils.java | 0 .../spark/PruneColumnsWithReordering.java | 0 .../spark/PruneColumnsWithoutReordering.java | 0 .../apache/iceberg/spark/SparkDataFile.java | 0 .../iceberg/spark/SparkExceptionUtil.java | 0 .../spark/SparkFixupTimestampType.java | 0 .../apache/iceberg/spark/SparkFixupTypes.java | 0 .../iceberg/spark/SparkReadOptions.java | 0 .../apache/iceberg/spark/SparkSchemaUtil.java | 0 .../apache/iceberg/spark/SparkStructLike.java | 0 .../apache/iceberg/spark/SparkTableUtil.java | 0 .../apache/iceberg/spark/SparkTypeToType.java | 0 .../iceberg/spark/SparkTypeVisitor.java | 0 .../org/apache/iceberg/spark/SparkUtil.java | 0 .../iceberg/spark/SparkValueConverter.java | 0 .../iceberg/spark/SparkWriteOptions.java | 0 .../apache/iceberg/spark/TypeToSparkType.java | 0 .../BaseDeleteOrphanFilesSparkAction.java | 0 .../BaseDeleteReachableFilesSparkAction.java | 0 .../BaseExpireSnapshotsSparkAction.java | 0 .../BaseRewriteDataFilesSparkAction.java | 0 .../BaseRewriteManifestsSparkAction.java | 0 .../BaseSnapshotUpdateSparkAction.java | 0 .../spark/actions/BaseSparkAction.java | 0 .../spark/actions/BaseSparkActions.java | 0 .../data/AvroWithSparkSchemaVisitor.java | 0 .../data/ParquetWithSparkSchemaVisitor.java | 0 .../iceberg/spark/data/SparkAvroReader.java | 0 .../iceberg/spark/data/SparkAvroWriter.java | 0 .../iceberg/spark/data/SparkOrcReader.java | 0 .../spark/data/SparkOrcValueReaders.java | 0 .../spark/data/SparkOrcValueWriter.java | 0 .../spark/data/SparkOrcValueWriters.java | 0 .../iceberg/spark/data/SparkOrcWriter.java | 0 .../spark/data/SparkParquetReaders.java | 0 .../spark/data/SparkParquetWriters.java | 0 .../iceberg/spark/data/SparkValueReaders.java | 0 .../iceberg/spark/data/SparkValueWriters.java | 0 .../ArrowVectorAccessorFactory.java | 0 .../data/vectorized/ArrowVectorAccessors.java | 0 .../data/vectorized/ColumnarBatchReader.java | 0 .../data/vectorized/ConstantColumnVector.java | 0 .../vectorized/IcebergArrowColumnVector.java | 0 .../vectorized/RowPositionColumnVector.java | 0 .../vectorized/VectorizedSparkOrcReaders.java | 0 .../VectorizedSparkParquetReaders.java | 0 .../iceberg/spark/source/BaseDataReader.java | 0 .../iceberg/spark/source/BatchDataReader.java | 0 .../spark/source/EqualityDeleteRowReader.java | 0 .../spark/source/InternalRowWrapper.java | 0 .../iceberg/spark/source/RowDataReader.java | 0 .../iceberg/spark/source/RowDataRewriter.java | 0 .../spark/source/SparkAppenderFactory.java | 0 .../spark/source/SparkFileWriterFactory.java | 0 .../source/SparkPartitionedFanoutWriter.java | 0 .../spark/source/SparkPartitionedWriter.java | 0 .../spark/source/StructInternalRow.java | 0 .../java/org/apache/iceberg/KryoHelpers.java | 0 .../org/apache/iceberg/TaskCheckHelper.java | 0 .../iceberg/TestDataFileSerialization.java | 0 .../iceberg/TestFileIOSerialization.java | 0 .../TestManifestFileSerialization.java | 0 .../iceberg/TestScanTaskSerialization.java | 0 .../iceberg/TestTableSerialization.java | 0 .../TestDeleteReachableFilesAction.java | 0 .../actions/TestExpireSnapshotsAction.java | 0 .../actions/TestRemoveOrphanFilesAction.java | 0 .../actions/TestRewriteDataFilesAction.java | 0 .../actions/TestRewriteManifestsAction.java | 0 .../apache/iceberg/spark/SparkTestBase.java | 0 .../iceberg/spark/TestSparkSchemaUtil.java | 0 .../TestNewRewriteDataFilesAction.java | 0 .../iceberg/spark/data/AvroDataTest.java | 0 .../iceberg/spark/data/GenericsHelpers.java | 0 .../apache/iceberg/spark/data/RandomData.java | 0 .../iceberg/spark/data/TestHelpers.java | 0 .../iceberg/spark/data/TestOrcWrite.java | 0 .../spark/data/TestParquetAvroReader.java | 0 .../spark/data/TestParquetAvroWriter.java | 0 .../spark/data/TestSparkAvroEnums.java | 0 .../spark/data/TestSparkAvroReader.java | 0 .../spark/data/TestSparkDateTimes.java | 0 .../data/TestSparkOrcReadMetadataColumns.java | 0 .../spark/data/TestSparkOrcReader.java | 0 .../TestSparkParquetReadMetadataColumns.java | 0 .../spark/data/TestSparkParquetReader.java | 0 .../spark/data/TestSparkParquetWriter.java | 0 .../data/TestSparkRecordOrcReaderWriter.java | 0 ...rquetDictionaryEncodedVectorizedReads.java | 0 ...allbackToPlainEncodingVectorizedReads.java | 0 .../TestParquetVectorizedReads.java | 0 .../iceberg/spark/source/LogMessage.java | 0 .../iceberg/spark/source/SimpleRecord.java | 0 .../iceberg/spark/source/TestAvroScan.java | 0 .../spark/source/TestDataFrameWrites.java | 0 .../spark/source/TestDataSourceOptions.java | 0 .../source/TestForwardCompatibility.java | 0 .../source/TestIcebergSourceHadoopTables.java | 0 .../source/TestIcebergSourceHiveTables.java | 0 .../source/TestIcebergSourceTablesBase.java | 0 .../spark/source/TestIcebergSpark.java | 0 .../source/TestIdentityPartitionData.java | 0 .../spark/source/TestInternalRowWrapper.java | 0 .../iceberg/spark/source/TestParquetScan.java | 0 .../spark/source/TestPartitionPruning.java | 0 .../spark/source/TestPartitionValues.java | 0 .../spark/source/TestReadProjection.java | 0 .../spark/source/TestSnapshotSelection.java | 0 .../source/TestSparkAppenderFactory.java | 0 .../spark/source/TestSparkBaseDataReader.java | 0 .../spark/source/TestSparkDataFile.java | 0 .../spark/source/TestSparkDataWrite.java | 0 .../source/TestSparkFileWriterFactory.java | 0 .../spark/source/TestSparkMergingMetrics.java | 0 .../source/TestSparkPartitioningWriters.java | 0 .../source/TestSparkPositionDeltaWriters.java | 0 .../spark/source/TestSparkReadProjection.java | 0 .../spark/source/TestSparkReaderDeletes.java | 0 .../source/TestSparkRollingFileWriters.java | 0 .../iceberg/spark/source/TestSparkSchema.java | 0 .../spark/source/TestSparkWriterMetrics.java | 0 .../spark/source/TestStructuredStreaming.java | 0 .../iceberg/spark/source/TestTables.java | 0 .../source/TestTimestampWithoutZone.java | 0 .../spark/source/TestWriteMetricsConfig.java | 0 .../spark/source/ThreeColumnRecord.java | 0 168 files changed, 13 insertions(+), 10 deletions(-) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/SparkBenchmarkUtil.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersFlatDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersNestedDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/Action.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceFlatDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedListDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/avro/AvroWritersBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceFlatAvroDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceNestedAvroDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedListORCDataWriteBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataFilterBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataWriteBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedListParquetDataWriteBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataFilterBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataReadBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataWriteBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/ParquetWritersBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadDictionaryEncodedFlatParquetDataBenchmark.java (100%) rename {spark => spark-common}/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadFlatParquetDataBenchmark.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/Actions.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/CreateAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsActionResult.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/ManifestFileBean.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/RewriteDataFilesAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/actions/SnapshotAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/IcebergSpark.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/JobGroupInfo.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/JobGroupUtils.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/PruneColumnsWithReordering.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkDataFile.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkExceptionUtil.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkFixupTypes.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkStructLike.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkTypeVisitor.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkUtil.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseExpireSnapshotsSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteManifestsSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotUpdateSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseSparkAction.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/AvroWithSparkSchemaVisitor.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkAvroReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkAvroWriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriters.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkParquetReaders.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkParquetWriters.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkValueReaders.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/SparkValueWriters.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessorFactory.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessors.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/ColumnarBatchReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/RowPositionColumnVector.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/BaseDataReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/EqualityDeleteRowReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/InternalRowWrapper.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/RowDataRewriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java (100%) rename {spark => spark-common}/src/main/java/org/apache/iceberg/spark/source/StructInternalRow.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/KryoHelpers.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TaskCheckHelper.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TestDataFileSerialization.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TestFileIOSerialization.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TestManifestFileSerialization.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/TestTableSerialization.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/SparkTestBase.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/RandomData.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroReader.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroWriter.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroEnums.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroReader.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkDateTimes.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReadMetadataColumns.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReader.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryFallbackToPlainEncodingVectorizedReads.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetVectorizedReads.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/LogMessage.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestInternalRowWrapper.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkAppenderFactory.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkFileWriterFactory.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkMergingMetrics.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkPositionDeltaWriters.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestTables.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java (100%) rename {spark => spark-common}/src/test/java/org/apache/iceberg/spark/source/ThreeColumnRecord.java (100%) diff --git a/build.gradle b/build.gradle index f73ab2508906..eac7599d725b 100644 --- a/build.gradle +++ b/build.gradle @@ -67,6 +67,9 @@ allprojects { project.ext { Spark30Version = '3.0.3' Spark31Version = '3.1.1' + SparkBuildVersion = project.hasProperty("sparkBuild") ? project.getProperty("sparkBuild") : '3.1.1' + SparkSrcVersion = project.hasProperty("sparkSrc") ? project.getProperty("sparkSrc") : '3_0' + ScalaVersion = project.hasProperty("scala") ? project.getProperty("scala") : '2.12' } } @@ -932,7 +935,7 @@ project(':iceberg-arrow') { } } -project(':iceberg-spark') { +project(':iceberg-spark-common') { configurations.all { resolutionStrategy { // Spark 2.4.4 can only use the below datanucleus version, the versions introduced @@ -1021,7 +1024,7 @@ if (jdkVersion == '8') { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark') + implementation project(':iceberg-spark-common') implementation "com.github.ben-manes.caffeine:caffeine" compileOnly "org.apache.avro:avro" @@ -1029,7 +1032,7 @@ if (jdkVersion == '8') { exclude group: 'org.apache.avro', module: 'avro' } - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' @@ -1146,7 +1149,7 @@ project(':iceberg-spark3') { implementation project(':iceberg-parquet') implementation project(':iceberg-arrow') implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark') + implementation project(':iceberg-spark-common') compileOnly "org.apache.avro:avro" compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { @@ -1160,7 +1163,7 @@ project(':iceberg-spark3') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') testImplementation("org.apache.hadoop:hadoop-minicluster") { exclude group: 'org.apache.avro', module: 'avro' @@ -1237,7 +1240,7 @@ project(":iceberg-spark3-extensions") { compileOnly project(':iceberg-api') compileOnly project(':iceberg-core') compileOnly project(':iceberg-common') - compileOnly project(':iceberg-spark') + compileOnly project(':iceberg-spark-common') compileOnly project(':iceberg-spark3') compileOnly project(':iceberg-hive-metastore') compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { @@ -1249,7 +1252,7 @@ project(":iceberg-spark3-extensions") { testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { @@ -1332,7 +1335,7 @@ project(':iceberg-spark3-runtime') { integrationImplementation 'org.slf4j:slf4j-simple' integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-spark3-extensions', configuration: 'testArtifacts') // Not allowed on our classpath, only the runtime jar is allowed diff --git a/settings.gradle b/settings.gradle index 7f052475d008..29fab60fde5b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,7 +32,7 @@ include 'orc' include 'arrow' include 'parquet' include 'bundled-guava' -include 'spark' +include 'spark-common' include 'spark3' include 'spark3-extensions' include 'spark3-runtime' @@ -54,7 +54,7 @@ project(':orc').name = 'iceberg-orc' project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' project(':bundled-guava').name = 'iceberg-bundled-guava' -project(':spark').name = 'iceberg-spark' +project(':spark-common').name = 'iceberg-spark-common' project(':spark3').name = 'iceberg-spark3' project(':spark3-extensions').name = 'iceberg-spark3-extensions' project(':spark3-runtime').name = 'iceberg-spark3-runtime' diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/SparkBenchmarkUtil.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/SparkBenchmarkUtil.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/SparkBenchmarkUtil.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/SparkBenchmarkUtil.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersFlatDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersFlatDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersFlatDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersFlatDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersNestedDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersNestedDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersNestedDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetWritersNestedDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/Action.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/Action.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/Action.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceFlatDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceFlatDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceFlatDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceFlatDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedListDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedListDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedListDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/IcebergSourceNestedListDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/avro/AvroWritersBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/AvroWritersBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/avro/AvroWritersBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/AvroWritersBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceFlatAvroDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceFlatAvroDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceFlatAvroDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceFlatAvroDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceNestedAvroDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceNestedAvroDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceNestedAvroDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/avro/IcebergSourceNestedAvroDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceFlatORCDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedListORCDataWriteBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedListORCDataWriteBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedListORCDataWriteBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedListORCDataWriteBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/orc/IcebergSourceNestedORCDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataFilterBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataFilterBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataFilterBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataFilterBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataWriteBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataWriteBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataWriteBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceFlatParquetDataWriteBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedListParquetDataWriteBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedListParquetDataWriteBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedListParquetDataWriteBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedListParquetDataWriteBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataFilterBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataFilterBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataFilterBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataFilterBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataReadBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataReadBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataReadBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataReadBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataWriteBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataWriteBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataWriteBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/IcebergSourceNestedParquetDataWriteBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/ParquetWritersBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/ParquetWritersBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/ParquetWritersBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/ParquetWritersBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadDictionaryEncodedFlatParquetDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadDictionaryEncodedFlatParquetDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadDictionaryEncodedFlatParquetDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadDictionaryEncodedFlatParquetDataBenchmark.java diff --git a/spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadFlatParquetDataBenchmark.java b/spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadFlatParquetDataBenchmark.java similarity index 100% rename from spark/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadFlatParquetDataBenchmark.java rename to spark-common/src/jmh/java/org/apache/iceberg/spark/source/parquet/vectorized/VectorizedReadFlatParquetDataBenchmark.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java b/spark-common/src/main/java/org/apache/iceberg/actions/Actions.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/Actions.java rename to spark-common/src/main/java/org/apache/iceberg/actions/Actions.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/CreateAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/CreateAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/CreateAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/CreateAction.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsAction.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsActionResult.java b/spark-common/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsActionResult.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsActionResult.java rename to spark-common/src/main/java/org/apache/iceberg/actions/ExpireSnapshotsActionResult.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/ManifestFileBean.java b/spark-common/src/main/java/org/apache/iceberg/actions/ManifestFileBean.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/ManifestFileBean.java rename to spark-common/src/main/java/org/apache/iceberg/actions/ManifestFileBean.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/RewriteDataFilesAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/RewriteDataFilesAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/RewriteDataFilesAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/RewriteDataFilesAction.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/RewriteManifestsAction.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java b/spark-common/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java rename to spark-common/src/main/java/org/apache/iceberg/actions/RewriteManifestsActionResult.java diff --git a/spark/src/main/java/org/apache/iceberg/actions/SnapshotAction.java b/spark-common/src/main/java/org/apache/iceberg/actions/SnapshotAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/actions/SnapshotAction.java rename to spark-common/src/main/java/org/apache/iceberg/actions/SnapshotAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java b/spark-common/src/main/java/org/apache/iceberg/spark/IcebergSpark.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java rename to spark-common/src/main/java/org/apache/iceberg/spark/IcebergSpark.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/JobGroupInfo.java b/spark-common/src/main/java/org/apache/iceberg/spark/JobGroupInfo.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/JobGroupInfo.java rename to spark-common/src/main/java/org/apache/iceberg/spark/JobGroupInfo.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/JobGroupUtils.java b/spark-common/src/main/java/org/apache/iceberg/spark/JobGroupUtils.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/JobGroupUtils.java rename to spark-common/src/main/java/org/apache/iceberg/spark/JobGroupUtils.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithReordering.java b/spark-common/src/main/java/org/apache/iceberg/spark/PruneColumnsWithReordering.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithReordering.java rename to spark-common/src/main/java/org/apache/iceberg/spark/PruneColumnsWithReordering.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java b/spark-common/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java rename to spark-common/src/main/java/org/apache/iceberg/spark/PruneColumnsWithoutReordering.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkDataFile.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkDataFile.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkDataFile.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkExceptionUtil.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkExceptionUtil.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkExceptionUtil.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkExceptionUtil.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkFixupTimestampType.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkFixupTypes.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkFixupTypes.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkFixupTypes.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkFixupTypes.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkReadOptions.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkSchemaUtil.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkStructLike.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkStructLike.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkStructLike.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkStructLike.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkTableUtil.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkTypeToType.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkTypeVisitor.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkTypeVisitor.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkTypeVisitor.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkTypeVisitor.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkUtil.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkUtil.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkUtil.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkValueConverter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark-common/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java rename to spark-common/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java b/spark-common/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java rename to spark-common/src/main/java/org/apache/iceberg/spark/TypeToSparkType.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseExpireSnapshotsSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseExpireSnapshotsSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseExpireSnapshotsSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseExpireSnapshotsSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteManifestsSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteManifestsSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteManifestsSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteManifestsSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotUpdateSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotUpdateSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotUpdateSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotUpdateSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkAction.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSparkAction.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkAction.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSparkAction.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java b/spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java rename to spark-common/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/AvroWithSparkSchemaVisitor.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/AvroWithSparkSchemaVisitor.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/AvroWithSparkSchemaVisitor.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/AvroWithSparkSchemaVisitor.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/ParquetWithSparkSchemaVisitor.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkAvroReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkAvroReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkAvroReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkAvroReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkAvroWriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkAvroWriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkAvroWriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkAvroWriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueReaders.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriters.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriters.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriters.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcValueWriters.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkOrcWriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkParquetReaders.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkParquetReaders.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkParquetReaders.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkParquetReaders.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkParquetWriters.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkParquetWriters.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkParquetWriters.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkParquetWriters.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkValueReaders.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkValueReaders.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkValueReaders.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkValueReaders.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/SparkValueWriters.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/SparkValueWriters.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/SparkValueWriters.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/SparkValueWriters.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessorFactory.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessorFactory.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessorFactory.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessorFactory.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessors.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessors.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessors.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ArrowVectorAccessors.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ColumnarBatchReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ColumnarBatchReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ColumnarBatchReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ColumnarBatchReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/ConstantColumnVector.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/IcebergArrowColumnVector.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/RowPositionColumnVector.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/RowPositionColumnVector.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/RowPositionColumnVector.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/RowPositionColumnVector.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkOrcReaders.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java b/spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java rename to spark-common/src/main/java/org/apache/iceberg/spark/data/vectorized/VectorizedSparkParquetReaders.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/BaseDataReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/BaseDataReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/BaseDataReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/BaseDataReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/BatchDataReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/EqualityDeleteRowReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/EqualityDeleteRowReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/EqualityDeleteRowReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/EqualityDeleteRowReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/InternalRowWrapper.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/InternalRowWrapper.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/InternalRowWrapper.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/InternalRowWrapper.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/RowDataReader.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/RowDataRewriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/RowDataRewriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/RowDataRewriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/RowDataRewriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/SparkAppenderFactory.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/SparkFileWriterFactory.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedFanoutWriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/SparkPartitionedWriter.java diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/StructInternalRow.java b/spark-common/src/main/java/org/apache/iceberg/spark/source/StructInternalRow.java similarity index 100% rename from spark/src/main/java/org/apache/iceberg/spark/source/StructInternalRow.java rename to spark-common/src/main/java/org/apache/iceberg/spark/source/StructInternalRow.java diff --git a/spark/src/test/java/org/apache/iceberg/KryoHelpers.java b/spark-common/src/test/java/org/apache/iceberg/KryoHelpers.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/KryoHelpers.java rename to spark-common/src/test/java/org/apache/iceberg/KryoHelpers.java diff --git a/spark/src/test/java/org/apache/iceberg/TaskCheckHelper.java b/spark-common/src/test/java/org/apache/iceberg/TaskCheckHelper.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TaskCheckHelper.java rename to spark-common/src/test/java/org/apache/iceberg/TaskCheckHelper.java diff --git a/spark/src/test/java/org/apache/iceberg/TestDataFileSerialization.java b/spark-common/src/test/java/org/apache/iceberg/TestDataFileSerialization.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TestDataFileSerialization.java rename to spark-common/src/test/java/org/apache/iceberg/TestDataFileSerialization.java diff --git a/spark/src/test/java/org/apache/iceberg/TestFileIOSerialization.java b/spark-common/src/test/java/org/apache/iceberg/TestFileIOSerialization.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TestFileIOSerialization.java rename to spark-common/src/test/java/org/apache/iceberg/TestFileIOSerialization.java diff --git a/spark/src/test/java/org/apache/iceberg/TestManifestFileSerialization.java b/spark-common/src/test/java/org/apache/iceberg/TestManifestFileSerialization.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TestManifestFileSerialization.java rename to spark-common/src/test/java/org/apache/iceberg/TestManifestFileSerialization.java diff --git a/spark/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java b/spark-common/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java rename to spark-common/src/test/java/org/apache/iceberg/TestScanTaskSerialization.java diff --git a/spark/src/test/java/org/apache/iceberg/TestTableSerialization.java b/spark-common/src/test/java/org/apache/iceberg/TestTableSerialization.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/TestTableSerialization.java rename to spark-common/src/test/java/org/apache/iceberg/TestTableSerialization.java diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java b/spark-common/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java rename to spark-common/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java b/spark-common/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java rename to spark-common/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction.java diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java b/spark-common/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java rename to spark-common/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction.java diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction.java b/spark-common/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction.java rename to spark-common/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction.java diff --git a/spark/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction.java b/spark-common/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction.java rename to spark-common/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java b/spark-common/src/test/java/org/apache/iceberg/spark/SparkTestBase.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/SparkTestBase.java rename to spark-common/src/test/java/org/apache/iceberg/spark/SparkTestBase.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java b/spark-common/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java rename to spark-common/src/test/java/org/apache/iceberg/spark/TestSparkSchemaUtil.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java b/spark-common/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java rename to spark-common/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/AvroDataTest.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/GenericsHelpers.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/RandomData.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/RandomData.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/RandomData.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/RandomData.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestHelpers.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroReader.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroReader.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroReader.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroReader.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroWriter.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroWriter.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroWriter.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestParquetAvroWriter.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroEnums.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroEnums.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroEnums.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroEnums.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroReader.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroReader.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroReader.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkAvroReader.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkDateTimes.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkDateTimes.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkDateTimes.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkDateTimes.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReadMetadataColumns.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReadMetadataColumns.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReadMetadataColumns.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReadMetadataColumns.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReader.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReader.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReader.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetReader.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkParquetWriter.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryEncodedVectorizedReads.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryFallbackToPlainEncodingVectorizedReads.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryFallbackToPlainEncodingVectorizedReads.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryFallbackToPlainEncodingVectorizedReads.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetDictionaryFallbackToPlainEncodingVectorizedReads.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetVectorizedReads.java b/spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetVectorizedReads.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetVectorizedReads.java rename to spark-common/src/test/java/org/apache/iceberg/spark/data/parquet/vectorized/TestParquetVectorizedReads.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/LogMessage.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/LogMessage.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/LogMessage.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/LogMessage.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/SimpleRecord.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestAvroScan.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceTablesBase.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestInternalRowWrapper.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestInternalRowWrapper.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestInternalRowWrapper.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestInternalRowWrapper.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestParquetScan.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestReadProjection.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkAppenderFactory.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkAppenderFactory.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkAppenderFactory.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkAppenderFactory.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkFileWriterFactory.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkFileWriterFactory.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkFileWriterFactory.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkFileWriterFactory.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMergingMetrics.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkMergingMetrics.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMergingMetrics.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkMergingMetrics.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkPartitioningWriters.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPositionDeltaWriters.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkPositionDeltaWriters.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkPositionDeltaWriters.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkPositionDeltaWriters.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkRollingFileWriters.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestSparkWriterMetrics.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestTables.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestTables.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestTables.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig.java diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/ThreeColumnRecord.java b/spark-common/src/test/java/org/apache/iceberg/spark/source/ThreeColumnRecord.java similarity index 100% rename from spark/src/test/java/org/apache/iceberg/spark/source/ThreeColumnRecord.java rename to spark-common/src/test/java/org/apache/iceberg/spark/source/ThreeColumnRecord.java From 5dd43896b8f305011b222c84cc88819c1a66119f Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 20:46:54 -0700 Subject: [PATCH 2/8] reorganize spark modules in setting --- settings.gradle | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/settings.gradle b/settings.gradle index 29fab60fde5b..399b68914087 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,10 +32,6 @@ include 'orc' include 'arrow' include 'parquet' include 'bundled-guava' -include 'spark-common' -include 'spark3' -include 'spark3-extensions' -include 'spark3-runtime' include 'pig' include 'hive-metastore' include 'nessie' @@ -54,13 +50,20 @@ project(':orc').name = 'iceberg-orc' project(':arrow').name = 'iceberg-arrow' project(':parquet').name = 'iceberg-parquet' project(':bundled-guava').name = 'iceberg-bundled-guava' +project(':pig').name = 'iceberg-pig' +project(':hive-metastore').name = 'iceberg-hive-metastore' +project(':nessie').name = 'iceberg-nessie' + +// spark +include 'spark-common' +include 'spark3' +include 'spark3-extensions' +include 'spark3-runtime' + project(':spark-common').name = 'iceberg-spark-common' project(':spark3').name = 'iceberg-spark3' project(':spark3-extensions').name = 'iceberg-spark3-extensions' project(':spark3-runtime').name = 'iceberg-spark3-runtime' -project(':pig').name = 'iceberg-pig' -project(':hive-metastore').name = 'iceberg-hive-metastore' -project(':nessie').name = 'iceberg-nessie' if (JavaVersion.current() == JavaVersion.VERSION_1_8) { include 'spark2' From 09bdf3fe06077ab59bdffdf82fc10fc2ac3149e5 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 20:59:30 -0700 Subject: [PATCH 3/8] use new modules in setting --- settings.gradle | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/settings.gradle b/settings.gradle index 399b68914087..5ddf2f137343 100644 --- a/settings.gradle +++ b/settings.gradle @@ -56,23 +56,30 @@ project(':nessie').name = 'iceberg-nessie' // spark include 'spark-common' -include 'spark3' -include 'spark3-extensions' -include 'spark3-runtime' +include 'spark' +include 'spark-extensions' +include 'spark-runtime' + +SparkBuildVersion = System.getProperty("sparkBuild") == null ? System.getProperty("sparkBuild") : '3.1.1' +SparkSrcVersion = System.getProperty("sparkSrc") == null ? System.getProperty("sparkSrc") : '3_0' +ScalaVersion = System.getProperty("scala") == null ? System.getProperty("scala") : '2.12' + +if ('2_4' == SparkSrcVersion && JavaVersion.current() == JavaVersion.VERSION_1_8) { + throw new Exception("Expected java8 to build Spark 2.4, but got ${SparkSrcVersion}") +} project(':spark-common').name = 'iceberg-spark-common' -project(':spark3').name = 'iceberg-spark3' -project(':spark3-extensions').name = 'iceberg-spark3-extensions' -project(':spark3-runtime').name = 'iceberg-spark3-runtime' +project(':spark').name = 'iceberg-spark' +project(':spark').projectDir = file("spark/${SparkSrcVersion}/core") +project(':spark-extensions').name = 'iceberg-spark-extensions' +project(':spark-extensions').projectDir = file("spark/${SparkSrcVersion}/extensions") +project(':spark-runtime').name = 'iceberg-spark-runtime' +project(':spark-runtime').projectDir = file("spark/${SparkSrcVersion}/runtime") if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - include 'spark2' - include 'spark-runtime' include 'hive3' include 'hive3-orc-bundle' - project(':spark2').name = 'iceberg-spark2' - project(':spark-runtime').name = 'iceberg-spark-runtime' project(':hive3').name = 'iceberg-hive3' project(':hive3-orc-bundle').name = 'iceberg-hive3-orc-bundle' } From 7039ff7b7affa131da17b942e0da2b0c0ed8316e Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 21:04:15 -0700 Subject: [PATCH 4/8] refactor spark directories --- .gitignore | 8 ++++---- {spark2 => spark/2_4/core}/benchmark/.gitkeep | 0 .../java/org/apache/iceberg/actions/SparkActions.java | 0 .../main/java/org/apache/iceberg/spark/SparkFilters.java | 0 .../org/apache/iceberg/spark/actions/SparkActions.java | 0 .../org/apache/iceberg/spark/source/CustomCatalogs.java | 0 .../org/apache/iceberg/spark/source/IcebergSource.java | 0 .../main/java/org/apache/iceberg/spark/source/Reader.java | 0 .../main/java/org/apache/iceberg/spark/source/Stats.java | 0 .../org/apache/iceberg/spark/source/StreamingOffset.java | 0 .../org/apache/iceberg/spark/source/StreamingWriter.java | 0 .../main/java/org/apache/iceberg/spark/source/Writer.java | 0 .../org.apache.spark.sql.sources.DataSourceRegister | 0 .../org/apache/iceberg/TestScanTaskSerialization24.java | 0 .../iceberg/actions/TestDeleteReachableFilesAction24.java | 0 .../iceberg/actions/TestExpireSnapshotsAction24.java | 0 .../iceberg/actions/TestRemoveOrphanFilesAction24.java | 0 .../iceberg/actions/TestRewriteDataFilesAction24.java | 0 .../iceberg/actions/TestRewriteManifestsAction24.java | 0 .../java/org/apache/iceberg/examples/ConcurrencyTest.java | 0 .../src/test/java/org/apache/iceberg/examples/README.md | 0 .../apache/iceberg/examples/ReadAndWriteTablesTest.java | 0 .../org/apache/iceberg/examples/SchemaEvolutionTest.java | 0 .../java/org/apache/iceberg/examples/SimpleRecord.java | 0 .../iceberg/examples/SnapshotFunctionalityTest.java | 0 .../org/apache/iceberg/spark/source/TestAvroScan24.java | 0 .../java/org/apache/iceberg/spark/source/TestCatalog.java | 0 .../apache/iceberg/spark/source/TestCustomCatalog.java | 0 .../iceberg/spark/source/TestDataFrameWrites24.java | 0 .../iceberg/spark/source/TestDataSourceOptions24.java | 0 .../org/apache/iceberg/spark/source/TestFilteredScan.java | 0 .../iceberg/spark/source/TestForwardCompatibility24.java | 0 .../apache/iceberg/spark/source/TestIcebergSource.java | 0 .../spark/source/TestIcebergSourceHadoopTables24.java | 0 .../spark/source/TestIcebergSourceHiveTables24.java | 0 .../apache/iceberg/spark/source/TestIcebergSpark24.java | 0 .../iceberg/spark/source/TestIdentityPartitionData24.java | 0 .../iceberg/spark/source/TestNameMappingProjection.java | 0 .../apache/iceberg/spark/source/TestParquetScan24.java | 0 .../iceberg/spark/source/TestPartitionPruning24.java | 0 .../iceberg/spark/source/TestPartitionValues24.java | 0 .../java/org/apache/iceberg/spark/source/TestSelect.java | 0 .../iceberg/spark/source/TestSnapshotSelection24.java | 0 .../iceberg/spark/source/TestSparkBaseDataReader24.java | 0 .../apache/iceberg/spark/source/TestSparkDataFile24.java | 0 .../apache/iceberg/spark/source/TestSparkDataWrite24.java | 0 .../iceberg/spark/source/TestSparkReadProjection24.java | 0 .../iceberg/spark/source/TestSparkReaderDeletes24.java | 0 .../apache/iceberg/spark/source/TestSparkSchema24.java | 0 .../apache/iceberg/spark/source/TestSparkTableUtil.java | 0 .../source/TestSparkTableUtilWithInMemoryCatalog.java | 0 .../apache/iceberg/spark/source/TestStreamingOffset.java | 0 .../iceberg/spark/source/TestStructuredStreaming24.java | 0 .../iceberg/spark/source/TestTimestampWithoutZone24.java | 0 .../iceberg/spark/source/TestWriteMetricsConfig24.java | 0 .../2_4/core}/src/test/resources/data/books.json | 0 .../2_4/core}/src/test/resources/data/new-books.json | 0 {spark-runtime => spark/2_4/runtime}/LICENSE | 0 {spark-runtime => spark/2_4/runtime}/NOTICE | 0 {spark3 => spark/3_0/core}/benchmark/.gitkeep | 0 .../org/apache/iceberg/actions/Spark3MigrateAction.java | 0 .../org/apache/iceberg/actions/Spark3SnapshotAction.java | 0 .../java/org/apache/iceberg/actions/SparkActions.java | 0 .../main/java/org/apache/iceberg/spark/BaseCatalog.java | 0 .../org/apache/iceberg/spark/FileRewriteCoordinator.java | 0 .../org/apache/iceberg/spark/FileScanTaskSetManager.java | 0 .../main/java/org/apache/iceberg/spark/OrderField.java | 0 .../java/org/apache/iceberg/spark/PathIdentifier.java | 0 .../org/apache/iceberg/spark/RollbackStagedTable.java | 0 .../java/org/apache/iceberg/spark/SortOrderToSpark.java | 0 .../main/java/org/apache/iceberg/spark/Spark3Util.java | 0 .../java/org/apache/iceberg/spark/Spark3VersionUtil.java | 0 .../main/java/org/apache/iceberg/spark/SparkCatalog.java | 0 .../main/java/org/apache/iceberg/spark/SparkFilters.java | 0 .../org/apache/iceberg/spark/SparkSessionCatalog.java | 0 .../spark/actions/BaseMigrateTableSparkAction.java | 0 .../spark/actions/BaseRewriteDataFilesSpark3Action.java | 0 .../spark/actions/BaseSnapshotTableSparkAction.java | 0 .../spark/actions/BaseTableCreationSparkAction.java | 0 .../iceberg/spark/actions/Spark3BinPackStrategy.java | 0 .../org/apache/iceberg/spark/actions/SparkActions.java | 0 .../iceberg/spark/procedures/AddFilesProcedure.java | 0 .../apache/iceberg/spark/procedures/BaseProcedure.java | 0 .../spark/procedures/CherrypickSnapshotProcedure.java | 0 .../spark/procedures/ExpireSnapshotsProcedure.java | 0 .../iceberg/spark/procedures/MigrateTableProcedure.java | 0 .../spark/procedures/RemoveOrphanFilesProcedure.java | 0 .../spark/procedures/RewriteManifestsProcedure.java | 0 .../spark/procedures/RollbackToSnapshotProcedure.java | 0 .../spark/procedures/RollbackToTimestampProcedure.java | 0 .../spark/procedures/SetCurrentSnapshotProcedure.java | 0 .../iceberg/spark/procedures/SnapshotTableProcedure.java | 0 .../apache/iceberg/spark/procedures/SparkProcedures.java | 0 .../org/apache/iceberg/spark/source/IcebergSource.java | 0 .../apache/iceberg/spark/source/SparkBatchQueryScan.java | 0 .../org/apache/iceberg/spark/source/SparkBatchScan.java | 0 .../org/apache/iceberg/spark/source/SparkFilesScan.java | 0 .../iceberg/spark/source/SparkFilesScanBuilder.java | 0 .../apache/iceberg/spark/source/SparkMergeBuilder.java | 0 .../org/apache/iceberg/spark/source/SparkMergeScan.java | 0 .../iceberg/spark/source/SparkMicroBatchStream.java | 0 .../apache/iceberg/spark/source/SparkRewriteBuilder.java | 0 .../org/apache/iceberg/spark/source/SparkScanBuilder.java | 0 .../java/org/apache/iceberg/spark/source/SparkTable.java | 0 .../java/org/apache/iceberg/spark/source/SparkWrite.java | 0 .../apache/iceberg/spark/source/SparkWriteBuilder.java | 0 .../org/apache/iceberg/spark/source/StagedSparkTable.java | 0 .../main/java/org/apache/iceberg/spark/source/Stats.java | 0 .../org/apache/iceberg/spark/source/StreamingOffset.java | 0 .../sql/catalyst/analysis/NoSuchProcedureException.java | 0 .../connector/iceberg/catalog/ExtendedSupportsDelete.java | 0 .../spark/sql/connector/iceberg/catalog/Procedure.java | 0 .../sql/connector/iceberg/catalog/ProcedureCatalog.java | 0 .../sql/connector/iceberg/catalog/ProcedureParameter.java | 0 .../connector/iceberg/catalog/ProcedureParameterImpl.java | 0 .../sql/connector/iceberg/catalog/SupportsMerge.java | 0 .../iceberg/distributions/ClusteredDistribution.java | 0 .../sql/connector/iceberg/distributions/Distribution.java | 0 .../connector/iceberg/distributions/Distributions.java | 0 .../iceberg/distributions/OrderedDistribution.java | 0 .../iceberg/distributions/UnspecifiedDistribution.java | 0 .../distributions/impl/ClusterDistributionImpl.java | 0 .../distributions/impl/OrderedDistributionImpl.java | 0 .../distributions/impl/UnspecifiedDistributionImpl.java | 0 .../sql/connector/iceberg/expressions/NullOrdering.java | 0 .../sql/connector/iceberg/expressions/SortDirection.java | 0 .../sql/connector/iceberg/expressions/SortOrder.java | 0 .../sql/connector/iceberg/read/SupportsFileFilter.java | 0 .../spark/sql/connector/iceberg/write/MergeBuilder.java | 0 .../org.apache.spark.sql.sources.DataSourceRegister | 0 .../sql/catalyst/expressions/TransformExpressions.scala | 0 .../sql/catalyst/utils/DistributionAndOrderingUtils.scala | 0 .../org/apache/spark/sql/catalyst/utils/PlanUtils.scala | 0 .../org/apache/iceberg/TestScanTaskSerialization3.java | 0 .../org/apache/iceberg/actions/TestCreateActions.java | 0 .../iceberg/actions/TestExpireSnapshotsAction3.java | 0 .../iceberg/actions/TestNewRewriteDataFilesAction3.java | 0 .../apache/iceberg/actions/TestRemoveFilesAction3.java | 0 .../iceberg/actions/TestRemoveOrphanFilesAction3.java | 0 .../iceberg/actions/TestRewriteDataFilesAction3.java | 0 .../iceberg/actions/TestRewriteManifestsAction3.java | 0 .../org/apache/iceberg/spark/SparkCatalogTestBase.java | 0 .../apache/iceberg/spark/TestFileRewriteCoordinator.java | 0 .../java/org/apache/iceberg/spark/TestSpark3Util.java | 0 .../java/org/apache/iceberg/spark/TestSparkFilters.java | 0 .../org/apache/iceberg/spark/source/SparkTestTable.java | 0 .../org/apache/iceberg/spark/source/TestAvroScan3.java | 0 .../apache/iceberg/spark/source/TestDataFrameWrites3.java | 0 .../iceberg/spark/source/TestDataSourceOptions3.java | 0 .../org/apache/iceberg/spark/source/TestFilteredScan.java | 0 .../iceberg/spark/source/TestForwardCompatibility3.java | 0 .../apache/iceberg/spark/source/TestIcebergSource.java | 0 .../spark/source/TestIcebergSourceHadoopTables3.java | 0 .../spark/source/TestIcebergSourceHiveTables3.java | 0 .../apache/iceberg/spark/source/TestIcebergSpark3.java | 0 .../iceberg/spark/source/TestIdentityPartitionData3.java | 0 .../source/TestMetadataTablesWithPartitionEvolution.java | 0 .../org/apache/iceberg/spark/source/TestParquetScan3.java | 0 .../iceberg/spark/source/TestPartitionPruning3.java | 0 .../apache/iceberg/spark/source/TestPartitionValues3.java | 0 .../apache/iceberg/spark/source/TestPathIdentifier.java | 0 .../iceberg/spark/source/TestSnapshotSelection3.java | 0 .../iceberg/spark/source/TestSparkBaseDataReader3.java | 0 .../org/apache/iceberg/spark/source/TestSparkCatalog.java | 0 .../spark/source/TestSparkCatalogHadoopOverrides.java | 0 .../apache/iceberg/spark/source/TestSparkDataFile3.java | 0 .../apache/iceberg/spark/source/TestSparkDataWrite3.java | 0 .../apache/iceberg/spark/source/TestSparkFilesScan.java | 0 .../iceberg/spark/source/TestSparkMetadataColumns.java | 0 .../iceberg/spark/source/TestSparkReadProjection3.java | 0 .../iceberg/spark/source/TestSparkReaderDeletes3.java | 0 .../org/apache/iceberg/spark/source/TestSparkTable.java | 0 .../apache/iceberg/spark/source/TestStreamingOffset.java | 0 .../iceberg/spark/source/TestStructuredStreaming3.java | 0 .../spark/source/TestStructuredStreamingRead3.java | 0 .../iceberg/spark/source/TestTimestampWithoutZone3.java | 0 .../iceberg/spark/source/TestWriteMetricsConfig3.java | 0 .../java/org/apache/iceberg/spark/sql/TestAlterTable.java | 0 .../org/apache/iceberg/spark/sql/TestCreateTable.java | 0 .../apache/iceberg/spark/sql/TestCreateTableAsSelect.java | 0 .../java/org/apache/iceberg/spark/sql/TestDeleteFrom.java | 0 .../org/apache/iceberg/spark/sql/TestNamespaceSQL.java | 0 .../apache/iceberg/spark/sql/TestPartitionedWrites.java | 0 .../org/apache/iceberg/spark/sql/TestRefreshTable.java | 0 .../java/org/apache/iceberg/spark/sql/TestSelect.java | 0 .../iceberg/spark/sql/TestTimestampWithoutZone.java | 0 .../apache/iceberg/spark/sql/TestUnpartitionedWrites.java | 0 .../IcebergSqlExtensions.g4 | 0 .../spark/extensions/IcebergSparkSessionExtensions.scala | 0 .../sql/catalyst/analysis/AlignRowLevelOperations.scala | 0 .../catalyst/analysis/AssignmentAlignmentSupport.scala | 0 .../sql/catalyst/analysis/ProcedureArgumentCoercion.scala | 0 .../spark/sql/catalyst/analysis/ResolveProcedures.scala | 0 .../analysis/RowLevelOperationsPredicateCheck.scala | 0 .../spark/sql/catalyst/expressions/AccumulateFiles.scala | 0 .../OptimizeConditionsInRowLevelOperations.scala | 0 .../PullupCorrelatedPredicatesInRowLevelOperations.scala | 0 .../spark/sql/catalyst/optimizer/RewriteDelete.scala | 0 .../spark/sql/catalyst/optimizer/RewriteMergeInto.scala | 0 .../spark/sql/catalyst/optimizer/RewriteUpdate.scala | 0 .../extensions/IcebergSparkSqlExtensionsParser.scala | 0 .../extensions/IcebergSqlExtensionsAstBuilder.scala | 0 .../sql/catalyst/plans/logical/AddPartitionField.scala | 0 .../apache/spark/sql/catalyst/plans/logical/Call.scala | 0 .../sql/catalyst/plans/logical/DropIdentifierFields.scala | 0 .../sql/catalyst/plans/logical/DropPartitionField.scala | 0 .../sql/catalyst/plans/logical/DynamicFileFilter.scala | 0 .../spark/sql/catalyst/plans/logical/MergeInto.scala | 0 .../spark/sql/catalyst/plans/logical/ReplaceData.scala | 0 .../catalyst/plans/logical/ReplacePartitionField.scala | 0 .../sql/catalyst/plans/logical/SetIdentifierFields.scala | 0 .../plans/logical/SetWriteDistributionAndOrdering.scala | 0 .../spark/sql/catalyst/plans/logical/statements.scala | 0 .../catalyst/utils/RewriteRowLevelOperationHelper.scala | 0 .../apache/spark/sql/catalyst/utils/SetAccumulator.scala | 0 .../execution/datasources/v2/AddPartitionFieldExec.scala | 0 .../spark/sql/execution/datasources/v2/CallExec.scala | 0 .../datasources/v2/DropIdentifierFieldsExec.scala | 0 .../execution/datasources/v2/DropPartitionFieldExec.scala | 0 .../execution/datasources/v2/DynamicFileFilterExec.scala | 0 .../execution/datasources/v2/ExtendedBatchScanExec.scala | 0 .../datasources/v2/ExtendedDataSourceV2Implicits.scala | 0 .../datasources/v2/ExtendedDataSourceV2Strategy.scala | 0 .../sql/execution/datasources/v2/MergeIntoExec.scala | 0 .../sql/execution/datasources/v2/ReplaceDataExec.scala | 0 .../datasources/v2/ReplacePartitionFieldExec.scala | 0 .../datasources/v2/SetIdentifierFieldsExec.scala | 0 .../v2/SetWriteDistributionAndOrderingExec.scala | 0 .../org/apache/iceberg/spark/extensions/Employee.java | 0 .../iceberg/spark/extensions/SparkExtensionsTestBase.java | 0 .../spark/extensions/SparkRowLevelOperationsTestBase.java | 0 .../iceberg/spark/extensions/TestAddFilesProcedure.java | 0 .../spark/extensions/TestAlterTablePartitionFields.java | 0 .../iceberg/spark/extensions/TestAlterTableSchema.java | 0 .../iceberg/spark/extensions/TestCallStatementParser.java | 0 .../spark/extensions/TestCherrypickSnapshotProcedure.java | 0 .../iceberg/spark/extensions/TestCopyOnWriteDelete.java | 0 .../iceberg/spark/extensions/TestCopyOnWriteMerge.java | 0 .../iceberg/spark/extensions/TestCopyOnWriteUpdate.java | 0 .../org/apache/iceberg/spark/extensions/TestDelete.java | 0 .../spark/extensions/TestExpireSnapshotsProcedure.java | 0 .../iceberg/spark/extensions/TestIcebergExpressions.java | 0 .../org/apache/iceberg/spark/extensions/TestMerge.java | 0 .../spark/extensions/TestMigrateTableProcedure.java | 0 .../spark/extensions/TestRemoveOrphanFilesProcedure.java | 0 .../spark/extensions/TestRewriteManifestsProcedure.java | 0 .../spark/extensions/TestRollbackToSnapshotProcedure.java | 0 .../extensions/TestRollbackToTimestampProcedure.java | 0 .../spark/extensions/TestSetCurrentSnapshotProcedure.java | 0 .../extensions/TestSetWriteDistributionAndOrdering.java | 0 .../spark/extensions/TestSnapshotTableProcedure.java | 0 .../org/apache/iceberg/spark/extensions/TestUpdate.java | 0 {spark3-runtime => spark/3_0/runtime}/LICENSE | 0 {spark3-runtime => spark/3_0/runtime}/NOTICE | 0 .../java/org/apache/iceberg/spark/SmokeTest.java | 0 255 files changed, 4 insertions(+), 4 deletions(-) rename {spark2 => spark/2_4/core}/benchmark/.gitkeep (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/actions/SparkActions.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/SparkFilters.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/Reader.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/Stats.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java (100%) rename {spark2 => spark/2_4/core}/src/main/java/org/apache/iceberg/spark/source/Writer.java (100%) rename {spark2 => spark/2_4/core}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/README.md (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/SimpleRecord.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSelect.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java (100%) rename {spark2 => spark/2_4/core}/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java (100%) rename {spark2 => spark/2_4/core}/src/test/resources/data/books.json (100%) rename {spark2 => spark/2_4/core}/src/test/resources/data/new-books.json (100%) rename {spark-runtime => spark/2_4/runtime}/LICENSE (100%) rename {spark-runtime => spark/2_4/runtime}/NOTICE (100%) rename {spark3 => spark/3_0/core}/benchmark/.gitkeep (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/actions/SparkActions.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/BaseCatalog.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/OrderField.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/PathIdentifier.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/Spark3Util.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/SparkCatalog.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/SparkFilters.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkTable.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/Stats.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java (100%) rename {spark3 => spark/3_0/core}/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java (100%) rename {spark3 => spark/3_0/core}/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister (100%) rename {spark3 => spark/3_0/core}/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala (100%) rename {spark3 => spark/3_0/core}/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala (100%) rename {spark3 => spark/3_0/core}/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestCreateActions.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java (100%) rename {spark3 => spark/3_0/core}/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/Employee.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java (100%) rename {spark3-extensions => spark/3_0/extensions}/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java (100%) rename {spark3-runtime => spark/3_0/runtime}/LICENSE (100%) rename {spark3-runtime => spark/3_0/runtime}/NOTICE (100%) rename {spark3-runtime => spark/3_0/runtime}/src/integration/java/org/apache/iceberg/spark/SmokeTest.java (100%) diff --git a/.gitignore b/.gitignore index 6c06d4ab6da0..1cdbc48ce5da 100644 --- a/.gitignore +++ b/.gitignore @@ -26,10 +26,10 @@ lib/ site/site # benchmark output -spark2/benchmark/* -!spark2/benchmark/.gitkeep -spark3/benchmark/* -!spark3/benchmark/.gitkeep +spark/2_4/core/benchmark/* +!spark/2_4/core/benchmark/.gitkeep +spark/3_0/core/benchmark/* +!spark/3_0/core/benchmark/.gitkeep __pycache__/ *.py[cod] diff --git a/spark2/benchmark/.gitkeep b/spark/2_4/core/benchmark/.gitkeep similarity index 100% rename from spark2/benchmark/.gitkeep rename to spark/2_4/core/benchmark/.gitkeep diff --git a/spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java b/spark/2_4/core/src/main/java/org/apache/iceberg/actions/SparkActions.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/actions/SparkActions.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/actions/SparkActions.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/SparkFilters.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/SparkFilters.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/SparkFilters.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/CustomCatalogs.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Reader.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Reader.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Reader.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Stats.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Stats.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Stats.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/StreamingWriter.java diff --git a/spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java b/spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Writer.java similarity index 100% rename from spark2/src/main/java/org/apache/iceberg/spark/source/Writer.java rename to spark/2_4/core/src/main/java/org/apache/iceberg/spark/source/Writer.java diff --git a/spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/2_4/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 100% rename from spark2/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark/2_4/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/TestScanTaskSerialization24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction24.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/ConcurrencyTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/README.md b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/README.md similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/README.md rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/README.md diff --git a/spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/ReadAndWriteTablesTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/SchemaEvolutionTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/SimpleRecord.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SimpleRecord.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/SimpleRecord.java diff --git a/spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java b/spark/2_4/core/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/examples/SnapshotFunctionalityTest.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestAvroScan24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestCustomCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestNameMappingProjection.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestParquetScan24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSelect.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSelect.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSelect.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkSchema24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtil.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTableUtilWithInMemoryCatalog.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone24.java diff --git a/spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java b/spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java similarity index 100% rename from spark2/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java rename to spark/2_4/core/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig24.java diff --git a/spark2/src/test/resources/data/books.json b/spark/2_4/core/src/test/resources/data/books.json similarity index 100% rename from spark2/src/test/resources/data/books.json rename to spark/2_4/core/src/test/resources/data/books.json diff --git a/spark2/src/test/resources/data/new-books.json b/spark/2_4/core/src/test/resources/data/new-books.json similarity index 100% rename from spark2/src/test/resources/data/new-books.json rename to spark/2_4/core/src/test/resources/data/new-books.json diff --git a/spark-runtime/LICENSE b/spark/2_4/runtime/LICENSE similarity index 100% rename from spark-runtime/LICENSE rename to spark/2_4/runtime/LICENSE diff --git a/spark-runtime/NOTICE b/spark/2_4/runtime/NOTICE similarity index 100% rename from spark-runtime/NOTICE rename to spark/2_4/runtime/NOTICE diff --git a/spark3/benchmark/.gitkeep b/spark/3_0/core/benchmark/.gitkeep similarity index 100% rename from spark3/benchmark/.gitkeep rename to spark/3_0/core/benchmark/.gitkeep diff --git a/spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java b/spark/3_0/core/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/actions/Spark3MigrateAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java b/spark/3_0/core/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/actions/Spark3SnapshotAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java b/spark/3_0/core/src/main/java/org/apache/iceberg/actions/SparkActions.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/actions/SparkActions.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/actions/SparkActions.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/BaseCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/BaseCatalog.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/BaseCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/FileRewriteCoordinator.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/FileScanTaskSetManager.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/OrderField.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/OrderField.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/OrderField.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/OrderField.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/PathIdentifier.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/PathIdentifier.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/PathIdentifier.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/RollbackStagedTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/SortOrderToSpark.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/Spark3Util.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/Spark3Util.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/Spark3Util.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/Spark3VersionUtil.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkCatalog.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkFilters.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkFilters.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkFilters.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/SparkSessionCatalog.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseMigrateTableSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseRewriteDataFilesSpark3Action.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseSnapshotTableSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/BaseTableCreationSparkAction.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/Spark3BinPackStrategy.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/actions/SparkActions.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/AddFilesProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/BaseProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/CherrypickSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/ExpireSnapshotsProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/MigrateTableProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RewriteManifestsProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RollbackToSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/RollbackToTimestampProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SetCurrentSnapshotProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SnapshotTableProcedure.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/procedures/SparkProcedures.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/IcebergSource.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkBatchQueryScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkBatchScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkFilesScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkFilesScanBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMergeBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMergeScan.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkMicroBatchStream.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkRewriteBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkScanBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkTable.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/SparkWriteBuilder.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/StagedSparkTable.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/Stats.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/Stats.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/Stats.java diff --git a/spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java b/spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java similarity index 100% rename from spark3/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java rename to spark/3_0/core/src/main/java/org/apache/iceberg/spark/source/StreamingOffset.java diff --git a/spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/catalyst/analysis/NoSuchProcedureException.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ExtendedSupportsDelete.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/Procedure.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureCatalog.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameter.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/ProcedureParameterImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/catalog/SupportsMerge.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/ClusteredDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/Distributions.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/OrderedDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/UnspecifiedDistribution.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/ClusterDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/OrderedDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/distributions/impl/UnspecifiedDistributionImpl.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/NullOrdering.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortDirection.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/expressions/SortOrder.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/read/SupportsFileFilter.java diff --git a/spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java b/spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java similarity index 100% rename from spark3/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java rename to spark/3_0/core/src/main/java/org/apache/spark/sql/connector/iceberg/write/MergeBuilder.java diff --git a/spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister b/spark/3_0/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister similarity index 100% rename from spark3/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister rename to spark/3_0/core/src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala b/spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala rename to spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala b/spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala rename to spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/utils/DistributionAndOrderingUtils.scala diff --git a/spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala b/spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala similarity index 100% rename from spark3/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala rename to spark/3_0/core/src/main/scala/org/apache/spark/sql/catalyst/utils/PlanUtils.scala diff --git a/spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/TestScanTaskSerialization3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestCreateActions.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestCreateActions.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestCreateActions.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestExpireSnapshotsAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestNewRewriteDataFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRemoveOrphanFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRewriteDataFilesAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/actions/TestRewriteManifestsAction3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestFileRewriteCoordinator.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestSpark3Util.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/TestSparkFilters.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/SparkTestTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestAvroScan3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWrites3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSource.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHadoopTables3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSourceHiveTables3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestParquetScan3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestPathIdentifier.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkBaseDataReader3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalog.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataFile3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkFilesScan.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestSparkTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStreamingOffset.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/source/TestWriteMetricsConfig3.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestDeleteFrom.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestPartitionedWrites.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestSelect.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestTimestampWithoutZone.java diff --git a/spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java b/spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java similarity index 100% rename from spark3/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java rename to spark/3_0/core/src/test/java/org/apache/iceberg/spark/sql/TestUnpartitionedWrites.java diff --git a/spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 b/spark/3_0/extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 similarity index 100% rename from spark3-extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 rename to spark/3_0/extensions/src/main/antlr/org.apache.spark.sql.catalyst.parser.extensions/IcebergSqlExtensions.g4 diff --git a/spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala b/spark/3_0/extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala rename to spark/3_0/extensions/src/main/scala/org/apache/iceberg/spark/extensions/IcebergSparkSessionExtensions.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AlignRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/AssignmentAlignmentSupport.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ProcedureArgumentCoercion.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveProcedures.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/analysis/RowLevelOperationsPredicateCheck.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/expressions/AccumulateFiles.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/OptimizeConditionsInRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/PullupCorrelatedPredicatesInRowLevelOperations.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteDelete.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteMergeInto.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/optimizer/RewriteUpdate.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSparkSqlExtensionsParser.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/parser/extensions/IcebergSqlExtensionsAstBuilder.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/AddPartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/Call.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropIdentifierFields.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DropPartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/DynamicFileFilter.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/MergeInto.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplaceData.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/ReplacePartitionField.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetIdentifierFields.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/SetWriteDistributionAndOrdering.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/RewriteRowLevelOperationHelper.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/catalyst/utils/SetAccumulator.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/AddPartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CallExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropIdentifierFieldsExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DropPartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DynamicFileFilterExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedBatchScanExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Implicits.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ExtendedDataSourceV2Strategy.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/MergeIntoExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplaceDataExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ReplacePartitionFieldExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetIdentifierFieldsExec.scala diff --git a/spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala b/spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala similarity index 100% rename from spark3-extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala rename to spark/3_0/extensions/src/main/scala/org/apache/spark/sql/execution/datasources/v2/SetWriteDistributionAndOrderingExec.scala diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/Employee.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkExtensionsTestBase.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAddFilesProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTablePartitionFields.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestAlterTableSchema.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCallStatementParser.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCherrypickSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteDelete.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteMerge.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestCopyOnWriteUpdate.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestExpireSnapshotsProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestIcebergExpressions.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMigrateTableProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRewriteManifestsProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRollbackToTimestampProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetCurrentSnapshotProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSetWriteDistributionAndOrdering.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestSnapshotTableProcedure.java diff --git a/spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java b/spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java similarity index 100% rename from spark3-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java rename to spark/3_0/extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java diff --git a/spark3-runtime/LICENSE b/spark/3_0/runtime/LICENSE similarity index 100% rename from spark3-runtime/LICENSE rename to spark/3_0/runtime/LICENSE diff --git a/spark3-runtime/NOTICE b/spark/3_0/runtime/NOTICE similarity index 100% rename from spark3-runtime/NOTICE rename to spark/3_0/runtime/NOTICE diff --git a/spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java b/spark/3_0/runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java similarity index 100% rename from spark3-runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java rename to spark/3_0/runtime/src/integration/java/org/apache/iceberg/spark/SmokeTest.java From b829a20950669848b5f8e0ab2c4fa89078c5dfef Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 21:20:23 -0700 Subject: [PATCH 5/8] make build continue to work for spark3 --- build.gradle | 266 ++++++++++++++++++++++++------------------------ jmh.gradle | 4 +- settings.gradle | 12 +-- 3 files changed, 141 insertions(+), 141 deletions(-) diff --git a/build.gradle b/build.gradle index eac7599d725b..59f3ab7e758b 100644 --- a/build.gradle +++ b/build.gradle @@ -999,129 +999,129 @@ project(':iceberg-spark-common') { maxHeapSize '2500m' } } - -if (jdkVersion == '8') { - - project(':iceberg-spark2') { - configurations.all { - resolutionStrategy { - // Spark 2.4.4 can only use the below datanucleus version, the versions introduced - // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions - // introduced by Hive 1.2.1. - force 'org.datanucleus:datanucleus-api-jdo:3.2.6' - force 'org.datanucleus:datanucleus-core:3.2.10' - force 'org.datanucleus:datanucleus-rdbms:3.2.9' - } - } - - dependencies { - implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - api project(':iceberg-api') - implementation project(':iceberg-common') - implementation project(':iceberg-core') - implementation project(':iceberg-data') - implementation project(':iceberg-orc') - implementation project(':iceberg-parquet') - implementation project(':iceberg-arrow') - implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark-common') - implementation "com.github.ben-manes.caffeine:caffeine" - - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.11") { - exclude group: 'org.apache.avro', module: 'avro' - } - - testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') - - testImplementation("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' - } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - } - - test { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") - - // Vectorized reads need more memory - maxHeapSize '2500m' - } - } - - // the runtime jar is a self-contained artifact for testing in a notebook - project(':iceberg-spark-runtime') { - apply plugin: 'com.github.johnrengelman.shadow' - - tasks.jar.dependsOn tasks.shadowJar - - configurations { - implementation { - exclude group: 'org.apache.spark' - // included in Spark - exclude group: 'org.slf4j' - exclude group: 'org.apache.commons' - exclude group: 'commons-pool' - exclude group: 'commons-codec' - exclude group: 'org.xerial.snappy' - exclude group: 'javax.xml.bind' - exclude group: 'javax.annotation' - } - } - - dependencies { - implementation project(':iceberg-spark2') - implementation project(':iceberg-aws') - implementation(project(':iceberg-nessie')) { - exclude group: 'com.google.code.findbugs', module: 'jsr305' - } - } - - shadowJar { - configurations = [project.configurations.runtimeClasspath] - - zip64 true - - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } - - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' - relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' - relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' - relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' - relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' - relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' - relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' - relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' - relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' - relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' - // relocate Arrow and related deps to shade Iceberg specific version - relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' - relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' - relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' - relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' - - classifier null - } - - jar { - enabled = false - } - } -} - -project(':iceberg-spark3') { +// +//if (jdkVersion == '8') { +// +// project(':iceberg-spark2') { +// configurations.all { +// resolutionStrategy { +// // Spark 2.4.4 can only use the below datanucleus version, the versions introduced +// // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions +// // introduced by Hive 1.2.1. +// force 'org.datanucleus:datanucleus-api-jdo:3.2.6' +// force 'org.datanucleus:datanucleus-core:3.2.10' +// force 'org.datanucleus:datanucleus-rdbms:3.2.9' +// } +// } +// +// dependencies { +// implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') +// api project(':iceberg-api') +// implementation project(':iceberg-common') +// implementation project(':iceberg-core') +// implementation project(':iceberg-data') +// implementation project(':iceberg-orc') +// implementation project(':iceberg-parquet') +// implementation project(':iceberg-arrow') +// implementation project(':iceberg-hive-metastore') +// implementation project(':iceberg-spark-common') +// implementation "com.github.ben-manes.caffeine:caffeine" +// +// compileOnly "org.apache.avro:avro" +// compileOnly("org.apache.spark:spark-hive_2.11") { +// exclude group: 'org.apache.avro', module: 'avro' +// } +// +// testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') +// +// testImplementation("org.apache.hadoop:hadoop-minicluster") { +// exclude group: 'org.apache.avro', module: 'avro' +// } +// testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') +// testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') +// testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') +// } +// +// test { +// // For vectorized reads +// // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds +// systemProperty("arrow.enable_unsafe_memory_access", "true") +// // Disable expensive null check for every get(index) call. +// // Iceberg manages nullability checks itself instead of relying on arrow. +// systemProperty("arrow.enable_null_check_for_get", "false") +// +// // Vectorized reads need more memory +// maxHeapSize '2500m' +// } +// } +// +// // the runtime jar is a self-contained artifact for testing in a notebook +// project(':iceberg-spark-runtime') { +// apply plugin: 'com.github.johnrengelman.shadow' +// +// tasks.jar.dependsOn tasks.shadowJar +// +// configurations { +// implementation { +// exclude group: 'org.apache.spark' +// // included in Spark +// exclude group: 'org.slf4j' +// exclude group: 'org.apache.commons' +// exclude group: 'commons-pool' +// exclude group: 'commons-codec' +// exclude group: 'org.xerial.snappy' +// exclude group: 'javax.xml.bind' +// exclude group: 'javax.annotation' +// } +// } +// +// dependencies { +// implementation project(':iceberg-spark2') +// implementation project(':iceberg-aws') +// implementation(project(':iceberg-nessie')) { +// exclude group: 'com.google.code.findbugs', module: 'jsr305' +// } +// } +// +// shadowJar { +// configurations = [project.configurations.runtimeClasspath] +// +// zip64 true +// +// // include the LICENSE and NOTICE files for the shaded Jar +// from(projectDir) { +// include 'LICENSE' +// include 'NOTICE' +// } +// +// // Relocate dependencies to avoid conflicts +// relocate 'com.google', 'org.apache.iceberg.shaded.com.google' +// relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' +// relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' +// relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' +// relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' +// relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' +// relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' +// relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' +// relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' +// relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' +// relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' +// // relocate Arrow and related deps to shade Iceberg specific version +// relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' +// relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' +// relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' +// relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' +// +// classifier null +// } +// +// jar { +// enabled = false +// } +// } +//} + +project(':iceberg-spark') { apply plugin: 'scala' sourceSets { @@ -1202,7 +1202,7 @@ project(':iceberg-spark3') { check.dependsOn testSpark31 } -project(":iceberg-spark3-extensions") { +project(":iceberg-spark-extensions") { apply plugin: 'java-library' apply plugin: 'scala' apply plugin: 'antlr' @@ -1241,7 +1241,7 @@ project(":iceberg-spark3-extensions") { compileOnly project(':iceberg-core') compileOnly project(':iceberg-common') compileOnly project(':iceberg-spark-common') - compileOnly project(':iceberg-spark3') + compileOnly project(':iceberg-spark') compileOnly project(':iceberg-hive-metastore') compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { exclude group: 'org.apache.avro', module: 'avro' @@ -1253,7 +1253,7 @@ project(":iceberg-spark3-extensions") { testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { exclude group: 'org.apache.avro', module: 'avro' @@ -1281,7 +1281,7 @@ project(":iceberg-spark3-extensions") { check.dependsOn testSpark31 } -project(':iceberg-spark3-runtime') { +project(':iceberg-spark-runtime') { apply plugin: 'com.github.johnrengelman.shadow' tasks.jar.dependsOn tasks.shadowJar @@ -1323,8 +1323,8 @@ project(':iceberg-spark3-runtime') { dependencies { api project(':iceberg-api') - implementation project(':iceberg-spark3') - implementation project(':iceberg-spark3-extensions') + implementation project(':iceberg-spark') + implementation project(':iceberg-spark-extensions') implementation project(':iceberg-aws') implementation(project(':iceberg-nessie')) { exclude group: 'com.google.code.findbugs', module: 'jsr305' @@ -1336,11 +1336,11 @@ project(':iceberg-spark3-runtime') { integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') integrationImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark3', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark3-extensions', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark-extensions', configuration: 'testArtifacts') // Not allowed on our classpath, only the runtime jar is allowed - integrationCompileOnly project(':iceberg-spark3-extensions') - integrationCompileOnly project(':iceberg-spark3') + integrationCompileOnly project(':iceberg-spark-extensions') + integrationCompileOnly project(':iceberg-spark') integrationCompileOnly project(':iceberg-api') spark31Implementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}" diff --git a/jmh.gradle b/jmh.gradle index edcb39c795fb..752e778f4b54 100644 --- a/jmh.gradle +++ b/jmh.gradle @@ -19,9 +19,9 @@ def jmhProjects = [] if (JavaVersion.current() == JavaVersion.VERSION_1_8) { - jmhProjects = [ project("iceberg-spark2"), project("iceberg-spark3") ] + jmhProjects = [ project("iceberg-spark") ] } else if (JavaVersion.current() == JavaVersion.VERSION_11) { - jmhProjects = [ project("iceberg-spark3") ] + jmhProjects = [ project("iceberg-spark") ] } else { throw new GradleException("The JMH benchamrks must be run with JDK 8 or JDK 11") } diff --git a/settings.gradle b/settings.gradle index 5ddf2f137343..5804f4dedbfe 100644 --- a/settings.gradle +++ b/settings.gradle @@ -60,21 +60,21 @@ include 'spark' include 'spark-extensions' include 'spark-runtime' -SparkBuildVersion = System.getProperty("sparkBuild") == null ? System.getProperty("sparkBuild") : '3.1.1' -SparkSrcVersion = System.getProperty("sparkSrc") == null ? System.getProperty("sparkSrc") : '3_0' -ScalaVersion = System.getProperty("scala") == null ? System.getProperty("scala") : '2.12' +var SparkBuildVersion = System.getProperty("sparkBuild") != null ? System.getProperty("sparkBuild") : '3.1.1' +var SparkSrcVersion = System.getProperty("sparkSrc") != null ? System.getProperty("sparkSrc") : '3_0' +var ScalaVersion = System.getProperty("scala") != null ? System.getProperty("scala") : '2.12' if ('2_4' == SparkSrcVersion && JavaVersion.current() == JavaVersion.VERSION_1_8) { throw new Exception("Expected java8 to build Spark 2.4, but got ${SparkSrcVersion}") } project(':spark-common').name = 'iceberg-spark-common' -project(':spark').name = 'iceberg-spark' project(':spark').projectDir = file("spark/${SparkSrcVersion}/core") -project(':spark-extensions').name = 'iceberg-spark-extensions' +project(':spark').name = 'iceberg-spark' project(':spark-extensions').projectDir = file("spark/${SparkSrcVersion}/extensions") -project(':spark-runtime').name = 'iceberg-spark-runtime' +project(':spark-extensions').name = 'iceberg-spark-extensions' project(':spark-runtime').projectDir = file("spark/${SparkSrcVersion}/runtime") +project(':spark-runtime').name = 'iceberg-spark-runtime' if (JavaVersion.current() == JavaVersion.VERSION_1_8) { include 'hive3' From d31847daf62b1a9156256e002ff958cad4596462 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 21:30:32 -0700 Subject: [PATCH 6/8] remove 30 and 31 version variables --- build.gradle | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/build.gradle b/build.gradle index 59f3ab7e758b..afb079dfa15f 100644 --- a/build.gradle +++ b/build.gradle @@ -65,11 +65,9 @@ allprojects { mavenLocal() } project.ext { - Spark30Version = '3.0.3' - Spark31Version = '3.1.1' - SparkBuildVersion = project.hasProperty("sparkBuild") ? project.getProperty("sparkBuild") : '3.1.1' - SparkSrcVersion = project.hasProperty("sparkSrc") ? project.getProperty("sparkSrc") : '3_0' - ScalaVersion = project.hasProperty("scala") ? project.getProperty("scala") : '2.12' + SparkBuildVersion = System.getProperty("sparkBuild") != null ? System.getProperty("sparkBuild") : '3.1.1' + SparkSrcVersion = System.getProperty("sparkSrc") != null ? System.getProperty("sparkSrc") : '3_0' + ScalaVersion = System.getProperty("scala") != null ? System.getProperty("scala") : '2.12' } } @@ -1152,7 +1150,7 @@ project(':iceberg-spark') { implementation project(':iceberg-spark-common') compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { + compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' } @@ -1173,7 +1171,7 @@ project(':iceberg-spark') { testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') testImplementation "org.xerial:sqlite-jdbc" - spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { + spark31Implementation("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' exclude group: 'junit' @@ -1243,7 +1241,7 @@ project(":iceberg-spark-extensions") { compileOnly project(':iceberg-spark-common') compileOnly project(':iceberg-spark') compileOnly project(':iceberg-hive-metastore') - compileOnly("org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}") { + compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' } @@ -1255,7 +1253,7 @@ project(":iceberg-spark-extensions") { testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - spark31Implementation("org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}") { + spark31Implementation("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { exclude group: 'org.apache.avro', module: 'avro' exclude group: 'org.apache.arrow' } @@ -1330,7 +1328,7 @@ project(':iceberg-spark-runtime') { exclude group: 'com.google.code.findbugs', module: 'jsr305' } - integrationImplementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark30Version}" + integrationImplementation "org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}" integrationImplementation 'org.junit.vintage:junit-vintage-engine' integrationImplementation 'org.slf4j:slf4j-simple' integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') @@ -1343,7 +1341,7 @@ project(':iceberg-spark-runtime') { integrationCompileOnly project(':iceberg-spark') integrationCompileOnly project(':iceberg-api') - spark31Implementation "org.apache.spark:spark-hive_2.12:${project.ext.Spark31Version}" + spark31Implementation "org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}" } shadowJar { From 065fea2f9ba49ee9e32e4ce8dfc17ddfe2ac9583 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 22:08:28 -0700 Subject: [PATCH 7/8] spark3 complete --- build.gradle | 85 ++++------------------------------------------------ 1 file changed, 6 insertions(+), 79 deletions(-) diff --git a/build.gradle b/build.gradle index afb079dfa15f..2902c3821a86 100644 --- a/build.gradle +++ b/build.gradle @@ -71,6 +71,9 @@ allprojects { } } +println "Build Spark ${project.ext.SparkBuildVersion} with Scala ${project.ext.ScalaVersion} " + + "against source ${project.ext.SparkSrcVersion}" + subprojects { apply plugin: 'nebula.dependency-recommender' apply plugin: 'java-library' @@ -934,6 +937,7 @@ project(':iceberg-arrow') { } project(':iceberg-spark-common') { + configurations.all { resolutionStrategy { // Spark 2.4.4 can only use the below datanucleus version, the versions introduced @@ -958,7 +962,7 @@ project(':iceberg-spark-common') { compileOnly "com.google.errorprone:error_prone_annotations" compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_2.11") { + compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { exclude group: 'org.apache.avro', module: 'avro' } @@ -1122,21 +1126,6 @@ project(':iceberg-spark-common') { project(':iceberg-spark') { apply plugin: 'scala' - sourceSets { - // Compile test source against Spark 3.1 and main classes compiled against Spark 3.0 - spark31 { - java.srcDir "$projectDir/src/test/java" - resources.srcDir "$projectDir/src/test/resources" - compileClasspath += sourceSets.test.output + sourceSets.main.output - runtimeClasspath += sourceSets.test.output - } - } - - configurations { - spark31Implementation.extendsFrom testImplementation - spark31RuntimeOnly.extendsFrom testRuntimeOnly - } - dependencies { implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') api project(':iceberg-api') @@ -1170,12 +1159,6 @@ project(':iceberg-spark') { testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') testImplementation "org.xerial:sqlite-jdbc" - - spark31Implementation("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - exclude group: 'junit' - } } tasks.withType(Test) { @@ -1189,15 +1172,6 @@ project(':iceberg-spark') { // Vectorized reads need more memory maxHeapSize '2560m' } - - task testSpark31(type: Test) { - dependsOn classes - group = "verification" - description = "Test against Spark 3.1" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + sourceSets.main.output - } - check.dependsOn testSpark31 } project(":iceberg-spark-extensions") { @@ -1205,21 +1179,7 @@ project(":iceberg-spark-extensions") { apply plugin: 'scala' apply plugin: 'antlr' - sourceSets { - // Compile test source against Spark 3.1 and main classes compiled against Spark 3.0 - spark31 { - // Main source is in scala, but test source is only in java - java.srcDir "$projectDir/src/test/java" - resources.srcDir "$projectDir/src/test/resources" - compileClasspath += sourceSets.test.output + sourceSets.main.output - runtimeClasspath += sourceSets.test.output - } - } - configurations { - spark31Implementation.extendsFrom testImplementation - spark31RuntimeOnly.extendsFrom testRuntimeOnly - /* The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate @@ -1253,11 +1213,6 @@ project(":iceberg-spark-extensions") { testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - spark31Implementation("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - } - // Required because we remove antlr plugin dependencies from the compile configuration, see note above // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime runtimeOnly "org.antlr:antlr4-runtime:4.7.1" @@ -1268,15 +1223,6 @@ project(":iceberg-spark-extensions") { maxHeapSize = "64m" arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions'] } - - task testSpark31(type: Test) { - dependsOn classes - group = "verification" - description = "Test against Spark 3.1" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + sourceSets.main.output - } - check.dependsOn testSpark31 } project(':iceberg-spark-runtime') { @@ -1289,12 +1235,6 @@ project(':iceberg-spark-runtime') { java.srcDir "$projectDir/src/integration/java" resources.srcDir "$projectDir/src/integration/resources" } - spark31 { - java.srcDir "$projectDir/src/integration/java" - resources.srcDir "$projectDir/src/integration/resources" - compileClasspath += sourceSets.integration.output - runtimeClasspath += sourceSets.integration.output - } } configurations { @@ -1315,8 +1255,6 @@ project(':iceberg-spark-runtime') { exclude group: 'org.antlr', module: 'ST4' exclude group: 'org.antlr', module: 'antlr4' } - spark31Implementation.extendsFrom integrationImplementation - spark31CompileOnly.extendsFrom integrationCompileOnly } dependencies { @@ -1340,8 +1278,6 @@ project(':iceberg-spark-runtime') { integrationCompileOnly project(':iceberg-spark-extensions') integrationCompileOnly project(':iceberg-spark') integrationCompileOnly project(':iceberg-api') - - spark31Implementation "org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}" } shadowJar { @@ -1387,16 +1323,7 @@ project(':iceberg-spark-runtime') { } integrationTest.dependsOn shadowJar - task spark31IntegrationTest(type: Test) { - dependsOn classes - description = "Test Spark3 Runtime Jar against Spark 3.1" - group = "verification" - testClassesDirs = sourceSets.spark31.output.classesDirs - classpath = sourceSets.spark31.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) - } - spark31IntegrationTest.dependsOn shadowJar - - check.dependsOn integrationTest, spark31IntegrationTest + check.dependsOn integrationTest jar { enabled = false From b6b24e4ef3be859177b01d72b9acde549995b669 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 6 Oct 2021 22:26:21 -0700 Subject: [PATCH 8/8] support spark24 --- build.gradle | 551 ++++++++++++++++++++++++------------------------ settings.gradle | 12 +- 2 files changed, 277 insertions(+), 286 deletions(-) diff --git a/build.gradle b/build.gradle index 2902c3821a86..c3c90ecdda03 100644 --- a/build.gradle +++ b/build.gradle @@ -962,7 +962,7 @@ project(':iceberg-spark-common') { compileOnly "com.google.errorprone:error_prone_annotations" compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { + compileOnly("org.apache.spark:spark-hive_2.11") { exclude group: 'org.apache.avro', module: 'avro' } @@ -1001,227 +1001,161 @@ project(':iceberg-spark-common') { maxHeapSize '2500m' } } -// -//if (jdkVersion == '8') { -// -// project(':iceberg-spark2') { -// configurations.all { -// resolutionStrategy { -// // Spark 2.4.4 can only use the below datanucleus version, the versions introduced -// // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions -// // introduced by Hive 1.2.1. -// force 'org.datanucleus:datanucleus-api-jdo:3.2.6' -// force 'org.datanucleus:datanucleus-core:3.2.10' -// force 'org.datanucleus:datanucleus-rdbms:3.2.9' -// } -// } -// -// dependencies { -// implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') -// api project(':iceberg-api') -// implementation project(':iceberg-common') -// implementation project(':iceberg-core') -// implementation project(':iceberg-data') -// implementation project(':iceberg-orc') -// implementation project(':iceberg-parquet') -// implementation project(':iceberg-arrow') -// implementation project(':iceberg-hive-metastore') -// implementation project(':iceberg-spark-common') -// implementation "com.github.ben-manes.caffeine:caffeine" -// -// compileOnly "org.apache.avro:avro" -// compileOnly("org.apache.spark:spark-hive_2.11") { -// exclude group: 'org.apache.avro', module: 'avro' -// } -// -// testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') -// -// testImplementation("org.apache.hadoop:hadoop-minicluster") { -// exclude group: 'org.apache.avro', module: 'avro' -// } -// testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') -// testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') -// testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') -// } -// -// test { -// // For vectorized reads -// // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds -// systemProperty("arrow.enable_unsafe_memory_access", "true") -// // Disable expensive null check for every get(index) call. -// // Iceberg manages nullability checks itself instead of relying on arrow. -// systemProperty("arrow.enable_null_check_for_get", "false") -// -// // Vectorized reads need more memory -// maxHeapSize '2500m' -// } -// } -// -// // the runtime jar is a self-contained artifact for testing in a notebook -// project(':iceberg-spark-runtime') { -// apply plugin: 'com.github.johnrengelman.shadow' -// -// tasks.jar.dependsOn tasks.shadowJar -// -// configurations { -// implementation { -// exclude group: 'org.apache.spark' -// // included in Spark -// exclude group: 'org.slf4j' -// exclude group: 'org.apache.commons' -// exclude group: 'commons-pool' -// exclude group: 'commons-codec' -// exclude group: 'org.xerial.snappy' -// exclude group: 'javax.xml.bind' -// exclude group: 'javax.annotation' -// } -// } -// -// dependencies { -// implementation project(':iceberg-spark2') -// implementation project(':iceberg-aws') -// implementation(project(':iceberg-nessie')) { -// exclude group: 'com.google.code.findbugs', module: 'jsr305' -// } -// } -// -// shadowJar { -// configurations = [project.configurations.runtimeClasspath] -// -// zip64 true -// -// // include the LICENSE and NOTICE files for the shaded Jar -// from(projectDir) { -// include 'LICENSE' -// include 'NOTICE' -// } -// -// // Relocate dependencies to avoid conflicts -// relocate 'com.google', 'org.apache.iceberg.shaded.com.google' -// relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' -// relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' -// relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' -// relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' -// relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' -// relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' -// relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' -// relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' -// relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' -// relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' -// // relocate Arrow and related deps to shade Iceberg specific version -// relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' -// relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' -// relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' -// relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' -// -// classifier null -// } -// -// jar { -// enabled = false -// } -// } -//} project(':iceberg-spark') { - apply plugin: 'scala' + if (project.ext.SparkSrcVersion == '2_4') { + configurations.all { + resolutionStrategy { + // Spark 2.4.4 can only use the below datanucleus version, the versions introduced + // by Hive 2.3.6 will meet lots of unexpected issues, so here force to use the versions + // introduced by Hive 1.2.1. + force 'org.datanucleus:datanucleus-api-jdo:3.2.6' + force 'org.datanucleus:datanucleus-core:3.2.10' + force 'org.datanucleus:datanucleus-rdbms:3.2.9' + } + } - dependencies { - implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') - api project(':iceberg-api') - implementation project(':iceberg-common') - implementation project(':iceberg-core') - implementation project(':iceberg-data') - implementation project(':iceberg-orc') - implementation project(':iceberg-parquet') - implementation project(':iceberg-arrow') - implementation project(':iceberg-hive-metastore') - implementation project(':iceberg-spark-common') + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark-common') + implementation "com.github.ben-manes.caffeine:caffeine" + + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_2.11") { + exclude group: 'org.apache.avro', module: 'avro' + } - compileOnly "org.apache.avro:avro" - compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') + + testImplementation("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') } - implementation("org.apache.arrow:arrow-vector") { - exclude group: 'io.netty', module: 'netty-buffer' - exclude group: 'io.netty', module: 'netty-common' - exclude group: 'com.google.code.findbugs', module: 'jsr305' + test { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") + + // Vectorized reads need more memory + maxHeapSize '2500m' } + } else { + apply plugin: 'scala' + + dependencies { + implementation project(path: ':iceberg-bundled-guava', configuration: 'shadow') + api project(':iceberg-api') + implementation project(':iceberg-common') + implementation project(':iceberg-core') + implementation project(':iceberg-data') + implementation project(':iceberg-orc') + implementation project(':iceberg-parquet') + implementation project(':iceberg-arrow') + implementation project(':iceberg-hive-metastore') + implementation project(':iceberg-spark-common') - testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') + compileOnly "org.apache.avro:avro" + compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.arrow' + } - testImplementation("org.apache.hadoop:hadoop-minicluster") { - exclude group: 'org.apache.avro', module: 'avro' + implementation("org.apache.arrow:arrow-vector") { + exclude group: 'io.netty', module: 'netty-buffer' + exclude group: 'io.netty', module: 'netty-common' + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } + + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') + + testImplementation("org.apache.hadoop:hadoop-minicluster") { + exclude group: 'org.apache.avro', module: 'avro' + } + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') + testImplementation "org.xerial:sqlite-jdbc" } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts') - testImplementation "org.xerial:sqlite-jdbc" - } - tasks.withType(Test) { - // For vectorized reads - // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds - systemProperty("arrow.enable_unsafe_memory_access", "true") - // Disable expensive null check for every get(index) call. - // Iceberg manages nullability checks itself instead of relying on arrow. - systemProperty("arrow.enable_null_check_for_get", "false") + tasks.withType(Test) { + // For vectorized reads + // Allow unsafe memory access to avoid the costly check arrow does to check if index is within bounds + systemProperty("arrow.enable_unsafe_memory_access", "true") + // Disable expensive null check for every get(index) call. + // Iceberg manages nullability checks itself instead of relying on arrow. + systemProperty("arrow.enable_null_check_for_get", "false") - // Vectorized reads need more memory - maxHeapSize '2560m' + // Vectorized reads need more memory + maxHeapSize '2560m' + } } } -project(":iceberg-spark-extensions") { - apply plugin: 'java-library' - apply plugin: 'scala' - apply plugin: 'antlr' +if (project.ext.SparkSrcVersion != '2_4') { + project(":iceberg-spark-extensions") { + apply plugin: 'java-library' + apply plugin: 'scala' + apply plugin: 'antlr' - configurations { - /* - The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This - bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate - runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with - the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add - the runtime dependency back to only the runtime configuration manually. - */ - implementation { - extendsFrom = extendsFrom.findAll { it != configurations.antlr } + configurations { + /* + The Gradle Antlr plugin erroneously adds both antlr-build and runtime dependencies to the runtime path. This + bug https://github.com/gradle/gradle/issues/820 exists because older versions of Antlr do not have separate + runtime and implementation dependencies and they do not want to break backwards compatibility. So to only end up with + the runtime dependency on the runtime classpath we remove the dependencies added by the plugin here. Then add + the runtime dependency back to only the runtime configuration manually. + */ + implementation { + extendsFrom = extendsFrom.findAll { it != configurations.antlr } + } } - } - dependencies { - compileOnly "org.scala-lang:scala-library" - compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') - compileOnly project(':iceberg-api') - compileOnly project(':iceberg-core') - compileOnly project(':iceberg-common') - compileOnly project(':iceberg-spark-common') - compileOnly project(':iceberg-spark') - compileOnly project(':iceberg-hive-metastore') - compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { - exclude group: 'org.apache.avro', module: 'avro' - exclude group: 'org.apache.arrow' - } + dependencies { + compileOnly "org.scala-lang:scala-library" + compileOnly project(path: ':iceberg-bundled-guava', configuration: 'shadow') + compileOnly project(':iceberg-api') + compileOnly project(':iceberg-core') + compileOnly project(':iceberg-common') + compileOnly project(':iceberg-spark-common') + compileOnly project(':iceberg-spark') + compileOnly project(':iceberg-hive-metastore') + compileOnly("org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}") { + exclude group: 'org.apache.avro', module: 'avro' + exclude group: 'org.apache.arrow' + } - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') - testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') + testImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - // Required because we remove antlr plugin dependencies from the compile configuration, see note above - // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime - runtimeOnly "org.antlr:antlr4-runtime:4.7.1" - antlr "org.antlr:antlr4:4.7.1" - } + // Required because we remove antlr plugin dependencies from the compile configuration, see note above + // We shade this in Spark3 Runtime to avoid issues with Spark's Antlr Runtime + runtimeOnly "org.antlr:antlr4-runtime:4.7.1" + antlr "org.antlr:antlr4:4.7.1" + } - generateGrammarSource { - maxHeapSize = "64m" - arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions'] + generateGrammarSource { + maxHeapSize = "64m" + arguments += ['-visitor', '-package', 'org.apache.spark.sql.catalyst.parser.extensions'] + } } } @@ -1229,101 +1163,156 @@ project(':iceberg-spark-runtime') { apply plugin: 'com.github.johnrengelman.shadow' tasks.jar.dependsOn tasks.shadowJar + if (project.ext.SparkSrcVersion == '2_4') { + configurations { + implementation { + exclude group: 'org.apache.spark' + // included in Spark + exclude group: 'org.slf4j' + exclude group: 'org.apache.commons' + exclude group: 'commons-pool' + exclude group: 'commons-codec' + exclude group: 'org.xerial.snappy' + exclude group: 'javax.xml.bind' + exclude group: 'javax.annotation' + } + } - sourceSets { - integration { - java.srcDir "$projectDir/src/integration/java" - resources.srcDir "$projectDir/src/integration/resources" + dependencies { + implementation project(':iceberg-spark') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } } - } - configurations { - implementation { - exclude group: 'org.apache.spark' - // included in Spark - exclude group: 'org.slf4j' - exclude group: 'org.apache.commons' - exclude group: 'commons-pool' - exclude group: 'commons-codec' - exclude group: 'org.xerial.snappy' - exclude group: 'javax.xml.bind' - exclude group: 'javax.annotation' - exclude group: 'com.github.luben' - exclude group: 'com.ibm.icu' - exclude group: 'org.glassfish' - exclude group: 'org.abego.treelayout' - exclude group: 'org.antlr', module: 'ST4' - exclude group: 'org.antlr', module: 'antlr4' + shadowJar { + configurations = [project.configurations.runtimeClasspath] + + zip64 true + + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } + + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' + relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' + relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' + relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' + relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' + relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' + relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' + relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' + relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' + relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' + // relocate Arrow and related deps to shade Iceberg specific version + relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' + relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' + relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' + relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' + + classifier null + } + } else { + sourceSets { + integration { + java.srcDir "$projectDir/src/integration/java" + resources.srcDir "$projectDir/src/integration/resources" + } } - } - dependencies { - api project(':iceberg-api') - implementation project(':iceberg-spark') - implementation project(':iceberg-spark-extensions') - implementation project(':iceberg-aws') - implementation(project(':iceberg-nessie')) { - exclude group: 'com.google.code.findbugs', module: 'jsr305' + configurations { + implementation { + exclude group: 'org.apache.spark' + // included in Spark + exclude group: 'org.slf4j' + exclude group: 'org.apache.commons' + exclude group: 'commons-pool' + exclude group: 'commons-codec' + exclude group: 'org.xerial.snappy' + exclude group: 'javax.xml.bind' + exclude group: 'javax.annotation' + exclude group: 'com.github.luben' + exclude group: 'com.ibm.icu' + exclude group: 'org.glassfish' + exclude group: 'org.abego.treelayout' + exclude group: 'org.antlr', module: 'ST4' + exclude group: 'org.antlr', module: 'antlr4' + } } - integrationImplementation "org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}" - integrationImplementation 'org.junit.vintage:junit-vintage-engine' - integrationImplementation 'org.slf4j:slf4j-simple' - integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') - integrationImplementation project(path: ':iceberg-spark-extensions', configuration: 'testArtifacts') - // Not allowed on our classpath, only the runtime jar is allowed - integrationCompileOnly project(':iceberg-spark-extensions') - integrationCompileOnly project(':iceberg-spark') - integrationCompileOnly project(':iceberg-api') - } + dependencies { + api project(':iceberg-api') + implementation project(':iceberg-spark') + implementation project(':iceberg-spark-extensions') + implementation project(':iceberg-aws') + implementation(project(':iceberg-nessie')) { + exclude group: 'com.google.code.findbugs', module: 'jsr305' + } - shadowJar { - configurations = [project.configurations.runtimeClasspath] + integrationImplementation "org.apache.spark:spark-hive_${project.ext.ScalaVersion}:${project.ext.SparkBuildVersion}" + integrationImplementation 'org.junit.vintage:junit-vintage-engine' + integrationImplementation 'org.slf4j:slf4j-simple' + integrationImplementation project(path: ':iceberg-api', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark-common', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark', configuration: 'testArtifacts') + integrationImplementation project(path: ':iceberg-spark-extensions', configuration: 'testArtifacts') + // Not allowed on our classpath, only the runtime jar is allowed + integrationCompileOnly project(':iceberg-spark-extensions') + integrationCompileOnly project(':iceberg-spark') + integrationCompileOnly project(':iceberg-api') + } - zip64 true + shadowJar { + configurations = [project.configurations.runtimeClasspath] - // include the LICENSE and NOTICE files for the shaded Jar - from(projectDir) { - include 'LICENSE' - include 'NOTICE' - } + zip64 true - // Relocate dependencies to avoid conflicts - relocate 'com.google', 'org.apache.iceberg.shaded.com.google' - relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' - relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' - relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' - relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' - relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' - relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' - relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' - relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' - relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' - relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' - // relocate Arrow and related deps to shade Iceberg specific version - relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' - relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' - relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' - relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' - // relocate Antlr runtime and related deps to shade Iceberg specific version - relocate 'org.antlr.v4', 'org.apache.iceberg.shaded.org.antlr.v4' + // include the LICENSE and NOTICE files for the shaded Jar + from(projectDir) { + include 'LICENSE' + include 'NOTICE' + } - classifier null - } + // Relocate dependencies to avoid conflicts + relocate 'com.google', 'org.apache.iceberg.shaded.com.google' + relocate 'com.fasterxml', 'org.apache.iceberg.shaded.com.fasterxml' + relocate 'com.github.benmanes', 'org.apache.iceberg.shaded.com.github.benmanes' + relocate 'org.checkerframework', 'org.apache.iceberg.shaded.org.checkerframework' + relocate 'org.apache.avro', 'org.apache.iceberg.shaded.org.apache.avro' + relocate 'avro.shaded', 'org.apache.iceberg.shaded.org.apache.avro.shaded' + relocate 'com.thoughtworks.paranamer', 'org.apache.iceberg.shaded.com.thoughtworks.paranamer' + relocate 'org.apache.parquet', 'org.apache.iceberg.shaded.org.apache.parquet' + relocate 'shaded.parquet', 'org.apache.iceberg.shaded.org.apache.parquet.shaded' + relocate 'org.apache.orc', 'org.apache.iceberg.shaded.org.apache.orc' + relocate 'io.airlift', 'org.apache.iceberg.shaded.io.airlift' + // relocate Arrow and related deps to shade Iceberg specific version + relocate 'io.netty.buffer', 'org.apache.iceberg.shaded.io.netty.buffer' + relocate 'org.apache.arrow', 'org.apache.iceberg.shaded.org.apache.arrow' + relocate 'com.carrotsearch', 'org.apache.iceberg.shaded.com.carrotsearch' + relocate 'org.threeten.extra', 'org.apache.iceberg.shaded.org.threeten.extra' + // relocate Antlr runtime and related deps to shade Iceberg specific version + relocate 'org.antlr.v4', 'org.apache.iceberg.shaded.org.antlr.v4' - task integrationTest(type: Test) { - description = "Test Spark3 Runtime Jar against Spark 3.0" - group = "verification" - testClassesDirs = sourceSets.integration.output.classesDirs - classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) - inputs.file(shadowJar.archiveFile.get().asFile.path) - } - integrationTest.dependsOn shadowJar + classifier null + } - check.dependsOn integrationTest + task integrationTest(type: Test) { + description = "Test Spark3 Runtime Jar against Spark 3.0" + group = "verification" + testClassesDirs = sourceSets.integration.output.classesDirs + classpath = sourceSets.integration.runtimeClasspath + files(shadowJar.archiveFile.get().asFile.path) + inputs.file(shadowJar.archiveFile.get().asFile.path) + } + integrationTest.dependsOn shadowJar + + check.dependsOn integrationTest + } jar { enabled = false diff --git a/settings.gradle b/settings.gradle index 5804f4dedbfe..c4402b5d4d3e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -60,19 +60,21 @@ include 'spark' include 'spark-extensions' include 'spark-runtime' -var SparkBuildVersion = System.getProperty("sparkBuild") != null ? System.getProperty("sparkBuild") : '3.1.1' var SparkSrcVersion = System.getProperty("sparkSrc") != null ? System.getProperty("sparkSrc") : '3_0' -var ScalaVersion = System.getProperty("scala") != null ? System.getProperty("scala") : '2.12' -if ('2_4' == SparkSrcVersion && JavaVersion.current() == JavaVersion.VERSION_1_8) { +if (SparkSrcVersion == '2_4' && JavaVersion.current() != JavaVersion.VERSION_1_8) { throw new Exception("Expected java8 to build Spark 2.4, but got ${SparkSrcVersion}") } project(':spark-common').name = 'iceberg-spark-common' project(':spark').projectDir = file("spark/${SparkSrcVersion}/core") project(':spark').name = 'iceberg-spark' -project(':spark-extensions').projectDir = file("spark/${SparkSrcVersion}/extensions") -project(':spark-extensions').name = 'iceberg-spark-extensions' + +if (SparkSrcVersion != '2_4') { + project(':spark-extensions').projectDir = file("spark/${SparkSrcVersion}/extensions") + project(':spark-extensions').name = 'iceberg-spark-extensions' +} + project(':spark-runtime').projectDir = file("spark/${SparkSrcVersion}/runtime") project(':spark-runtime').name = 'iceberg-spark-runtime'