updating fork

.baseline/checkstyle/checkstyle.xml

@@ -427,6 +427,11 @@
             <property name="illegalPkgs" value="org.hamcrest"/>
             <message key="import.illegal" value="Prefer using org.assertj.core.api.Assertions instead."/>
         </module>
+        <module name="IllegalImport">
+            <property name="id" value="BanJUnit5Assertions"/>
+            <property name="illegalPkgs" value="org.junit.jupiter.api.Assertions"/>
+            <message key="import.illegal" value="Prefer using org.assertj.core.api.Assertions instead."/>
+        </module>
         <module name="RegexpSinglelineJava">
             <property name="ignoreComments" value="true"/>
             <property name="format" value="@Json(S|Des)erialize"/>

.github/ISSUE_TEMPLATE/iceberg_bug_report.yml

@@ -28,7 +28,8 @@ body:
       description: What Apache Iceberg version are you using?
       multiple: false
       options:
-        - "1.6.0 (latest release)"
+        - "1.6.1 (latest release)"
+        - "1.6.0"
         - "1.5.2"
         - "1.5.1"
         - "1.5.0"

.github/workflows/delta-conversion-ci.yml

@@ -35,6 +35,7 @@ on:
       - '.github/workflows/hive-ci.yml'
       - '.github/workflows/java-ci.yml'
       - '.github/workflows/jmh-benchmarks-ci.yml'
+      - '.github/workflows/kafka-connect-ci.yml'
       - '.github/workflows/labeler.yml'
       - '.github/workflows/licence-check.yml'
       - '.github/workflows/open-api.yml'
@@ -51,6 +52,7 @@ on:
       - 'hive3-orc-bundle/**'
       - 'hive-runtime/**'
       - 'flink/**'
+      - 'kafka-connect/**'
       - 'pig/**'
       - 'docs/**'
       - 'site/**'
@@ -88,7 +90,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.12 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:
@@ -117,7 +119,7 @@ jobs:
           key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
           restore-keys: ${{ runner.os }}-gradle-
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
+      - run: ./gradlew -DsparkVersions=3.5 -DscalaVersion=2.13 -DhiveVersions= -DkafkaVersions= -DflinkVersions= :iceberg-delta-lake:check -Pquick=true -x javadoc
       - uses: actions/upload-artifact@v4
         if: failure()
         with:

.github/workflows/flink-ci.yml

@@ -35,6 +35,7 @@ on:
     - '.github/workflows/hive-ci.yml'
     - '.github/workflows/java-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
     - '.github/workflows/licence-check.yml'
     - '.github/workflows/open-api.yml'
@@ -50,6 +51,7 @@ on:
     - 'hive3/**'
     - 'hive3-orc-bundle/**'
     - 'hive-runtime/**'
+    - 'kafka-connect/**'
     - 'spark/**'
     - 'pig/**'
     - 'docs/**'
@@ -91,7 +93,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions= -DkafkaVersions= -DflinkVersions=${{ matrix.flink }} :iceberg-flink:iceberg-flink-${{ matrix.flink }}:check :iceberg-flink:iceberg-flink-runtime-${{ matrix.flink }}:check -Pquick=true -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with:

.github/workflows/hive-ci.yml

@@ -35,6 +35,7 @@ on:
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/java-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
     - '.github/workflows/licence-check.yml'
     - '.github/workflows/open-api.yml'
@@ -49,6 +50,7 @@ on:
     - 'arrow/**'
     - 'spark/**'
     - 'flink/**'
+    - 'kafka-connect/**'
     - 'pig/**'
     - 'docs/**'
     - 'site/**'
@@ -86,7 +88,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions=2 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-mr:check :iceberg-hive-runtime:check -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with:
@@ -115,7 +117,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
+    - run: ./gradlew -DsparkVersions= -DhiveVersions=3 -DflinkVersions= -DkafkaVersions= -Pquick=true :iceberg-hive3-orc-bundle:check :iceberg-hive3:check :iceberg-hive-runtime:check -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with:

.github/workflows/java-ci.yml

@@ -35,6 +35,7 @@ on:
     - '.github/workflows/flink-ci.yml'
     - '.github/workflows/hive-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
     - '.github/workflows/licence-check.yml'
     - '.github/workflows/open-api.yml'
@@ -82,7 +83,7 @@ jobs:
         key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: ${{ runner.os }}-gradle-
     - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
-    - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -Pquick=true -x javadoc
+    - run: ./gradlew check -DsparkVersions= -DhiveVersions= -DflinkVersions= -DkafkaVersions= -Pquick=true -x javadoc
     - uses: actions/upload-artifact@v4
       if: failure()
       with:

.github/workflows/kafka-connect-ci.yml

@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+name: "Kafka Connect CI"
+on:
+  push:
+    branches:
+    - 'main'
+    - '0.*'
+    - '1.*'
+    - '2.*'
+    tags:
+    - 'apache-iceberg-**'
+  pull_request:
+    paths-ignore:
+    - '.github/ISSUE_TEMPLATE/**'
+    - '.github/workflows/api-binary-compatibility.yml'
+    - '.github/workflows/delta-conversion-ci.yml'
+    - '.github/workflows/flink-ci.yml'
+    - '.github/workflows/hive-ci.yml'
+    - '.github/workflows/java-ci.yml'
+    - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/labeler.yml'
+    - '.github/workflows/licence-check.yml'
+    - '.github/workflows/open-api.yml'
+    - '.github/workflows/publish-snapshot.yml'
+    - '.github/workflows/recurring-jmh-benchmarks.yml'
+    - '.github/workflows/site-ci.yml'
+    - '.github/workflows/spark-ci.yml'
+    - '.github/workflows/stale.yml'
+    - '.gitignore'
+    - '.asf.yml'
+    - 'dev/**'
+    - 'mr/**'
+    - 'flink/**'
+    - 'hive3/**'
+    - 'hive3-orc-bundle/**'
+    - 'hive-runtime/**'
+    - 'spark/**'
+    - 'pig/**'
+    - 'docs/**'
+    - 'site/**'
+    - 'open-api/**'
+    - 'format/**'
+    - '.gitattributes'
+    - 'README.md'
+    - 'CONTRIBUTING.md'
+    - 'LICENSE'
+    - 'NOTICE'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
+
+jobs:
+
+  kafka-connect-tests:
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        jvm: [11, 17, 21]
+    env:
+      SPARK_LOCAL_IP: localhost
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: zulu
+        java-version: ${{ matrix.jvm }}
+    - uses: actions/cache@v4
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+        restore-keys: ${{ runner.os }}-gradle-
+    - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
+    - run: |
+        ./gradlew -DsparkVersions= -DhiveVersions= -DflinkVersions= -DkafkaVersions=3 \
+          :iceberg-kafka-connect:iceberg-kafka-connect-events:check \
+          :iceberg-kafka-connect:iceberg-kafka-connect:check \
+          :iceberg-kafka-connect:iceberg-kafka-connect-runtime:check \
+          -Pquick=true -x javadoc
+    - uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: test logs
+        path: |
+          **/build/testlogs

.github/workflows/publish-snapshot.yml

@@ -41,4 +41,4 @@ jobs:
       - run: |
           ./gradlew printVersion
           ./gradlew -DallModules publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
-          ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}
+          ./gradlew -DflinkVersions= -DsparkVersions=3.3,3.4,3.5 -DscalaVersion=2.13 -DkafkaVersions=3 -DhiveVersions= publishApachePublicationToMavenRepository -PmavenUser=${{ secrets.NEXUS_USER }} -PmavenPassword=${{ secrets.NEXUS_PW }}

.github/workflows/spark-ci.yml

@@ -36,6 +36,7 @@ on:
     - '.github/workflows/hive-ci.yml'
     - '.github/workflows/java-ci.yml'
     - '.github/workflows/jmh-benchmarks-ci.yml'
+    - '.github/workflows/kafka-connect-ci.yml'
     - '.github/workflows/labeler.yml'
     - '.github/workflows/licence-check.yml'
     - '.github/workflows/open-api.yml'
@@ -52,6 +53,7 @@ on:
     - 'hive3-orc-bundle/**'
     - 'hive-runtime/**'
     - 'flink/**'
+    - 'kafka-connect/**'
     - 'pig/**'
     - 'docs/**'
     - 'open-api/**'
@@ -101,7 +103,7 @@ jobs:
           tool-cache: false
       - run: echo -e "$(ip addr show eth0 | grep "inet\b" | awk '{print $2}' | cut -d/ -f1)\t$(hostname -f) $(hostname -s)" | sudo tee -a /etc/hosts
       - run: |
-          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= \
+          ./gradlew -DsparkVersions=${{ matrix.spark }} -DscalaVersion=${{ matrix.scala }} -DhiveVersions= -DflinkVersions= -DkafkaVersions= \
             :iceberg-spark:iceberg-spark-${{ matrix.spark }}_${{ matrix.scala }}:check \
             :iceberg-spark:iceberg-spark-extensions-${{ matrix.spark }}_${{ matrix.scala }}:check \
             :iceberg-spark:iceberg-spark-runtime-${{ matrix.spark }}_${{ matrix.scala }}:check \

.gitignore

@@ -37,7 +37,7 @@ spark/v3.4/spark/benchmark/*
 spark/v3.4/spark-extensions/benchmark/*
 spark/v3.5/spark/benchmark/*
 spark/v3.5/spark-extensions/benchmark/*
-data/benchmark/*
+*/benchmark/*
 
 __pycache__/
 *.py[cod]
@@ -66,3 +66,6 @@ metastore_db/
 # Spark/metastore files
 spark-warehouse/
 derby.log
+
+# jenv
+.java-version

.palantir/revapi.yml

@@ -873,6 +873,11 @@ acceptedBreaks:
       new: "method void org.apache.iceberg.encryption.Ciphers::<init>()"
       justification: "Static utility class - should not have public constructor"
   "1.4.0":
+    org.apache.iceberg:iceberg-api:
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.types.Types.NestedField"
+      new: "class org.apache.iceberg.types.Types.NestedField"
+      justification: "Add default value APIs."
     org.apache.iceberg:iceberg-core:
     - code: "java.class.defaultSerializationChanged"
       old: "class org.apache.iceberg.PartitionData"
@@ -1088,6 +1093,10 @@ acceptedBreaks:
       old: "class org.apache.iceberg.GenericManifestFile"
       new: "class org.apache.iceberg.GenericManifestFile"
       justification: "Serialization across versions is not supported"
+    - code: "java.class.defaultSerializationChanged"
+      old: "class org.apache.iceberg.io.WriteResult"
+      new: "class org.apache.iceberg.io.WriteResult"
+      justification: "Serialization across versions is not supported"
     - code: "java.class.removed"
       old: "enum org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus"
       justification: "Removing deprecated code"

LICENSE

@@ -298,6 +298,7 @@ License: https://www.apache.org/licenses/LICENSE-2.0
 This product includes code from Delta Lake.
 
 * AssignmentAlignmentSupport is an independent development but UpdateExpressionsSupport in Delta was used as a reference.
+* RoaringPositionBitmap is a Java implementation of RoaringBitmapArray in Delta.
 
 Copyright: 2020 The Delta Lake Project Authors.
 Home page: https://delta.io/

aliyun/src/test/java/org/apache/iceberg/aliyun/TestUtility.java

@@ -65,7 +65,7 @@ public static AliyunOSSExtension initialize() {
     } else {
       LOG.info(
           "Initializing AliyunOSSExtension implementation with default AliyunOSSMockExtension");
-      extension = AliyunOSSMockExtension.builder().silent().build();
+      extension = AliyunOSSMockExtension.builder().build();
     }
 
     return extension;

aliyun/src/test/java/org/apache/iceberg/aliyun/oss/TestOSSOutputStream.java

@@ -70,7 +70,7 @@ public void testWrite() throws IOException {
       reset(ossMock);
 
       // Write large file.
-      writeAndVerify(ossMock, uri, randomData(32 * 1024 * 1024), arrayWrite);
+      writeAndVerify(ossMock, uri, randomData(32 * 1024), arrayWrite);
       verify(ossMock, times(1)).putObject(any());
       reset(ossMock);
     }