diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index 11935e815e76..8aa5fbed1076 100644 --- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -327,6 +327,9 @@ private Result rebuildMetadata() { Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); // rebuild manifest files + // The size map holds one entry per distinct rewritten delete-file location. It is collected to + // the driver and broadcast to the manifest-rewrite tasks, so its footprint scales with the + // number of distinct delete files being rewritten rather than the table's total file count. RewriteContentFileResult rewriteManifestResult = rewriteManifests( deltaSnapshots, diff --git a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index 11935e815e76..8aa5fbed1076 100644 --- a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -327,6 +327,9 @@ private Result rebuildMetadata() { Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); // rebuild manifest files + // The size map holds one entry per distinct rewritten delete-file location. It is collected to + // the driver and broadcast to the manifest-rewrite tasks, so its footprint scales with the + // number of distinct delete files being rewritten rather than the table's total file count. RewriteContentFileResult rewriteManifestResult = rewriteManifests( deltaSnapshots, diff --git a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java index 11935e815e76..8aa5fbed1076 100644 --- a/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java +++ b/spark/v4.1/spark/src/main/java/org/apache/iceberg/spark/actions/RewriteTablePathSparkAction.java @@ -327,6 +327,9 @@ private Result rebuildMetadata() { Map rewrittenDeleteFileSizes = rewritePositionDeletes(deleteFilesToRewrite); // rebuild manifest files + // The size map holds one entry per distinct rewritten delete-file location. It is collected to + // the driver and broadcast to the manifest-rewrite tasks, so its footprint scales with the + // number of distinct delete files being rewritten rather than the table's total file count. RewriteContentFileResult rewriteManifestResult = rewriteManifests( deltaSnapshots,