Manifest list encryption

.palantir/revapi.yml

@@ -1364,6 +1364,11 @@ acceptedBreaks:
         \ org.apache.iceberg.data.parquet.GenericParquetWriter::createStructWriter(java.util.List<org.apache.iceberg.parquet.ParquetValueWriter<?>>)"
       justification: "Removing deprecations for 1.10.0"
   "1.10.0":
+    org.apache.iceberg:iceberg-api:
+      - code: "java.class.defaultSerializationChanged"
+        old: "class org.apache.iceberg.encryption.EncryptingFileIO"
+        new: "class org.apache.iceberg.encryption.EncryptingFileIO"
+        justification: "New method for Manifest List reading"
     org.apache.iceberg:iceberg-core:
       - code: "java.field.constantValueChanged"
         old: "field org.apache.iceberg.rest.ResourcePaths.V1_TABLE_SCAN_PLAN"

api/src/main/java/org/apache/iceberg/ManifestListFile.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.nio.ByteBuffer;
+import org.apache.iceberg.encryption.EncryptionManager;
+
+public interface ManifestListFile {
+
+  /** Location of manifest list file. */
+  String location();
+
+  /** The manifest list key metadata can be encrypted. Returns ID of encryption key */
+  String encryptionKeyID();
+
+  /** Decrypt and return the manifest list key metadata */
+  ByteBuffer decryptKeyMetadata(EncryptionManager em);
+}

api/src/main/java/org/apache/iceberg/encryption/EncryptingFileIO.java

@@ -28,6 +28,7 @@
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFile;
 import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestListFile;
 import org.apache.iceberg.io.FileIO;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
@@ -108,13 +109,21 @@ public InputFile newInputFile(ManifestFile manifest) {
     }
   }
 
+  @Override
+  public InputFile newInputFile(ManifestListFile manifestList) {
+    if (manifestList.encryptionKeyID() != null) {
+      ByteBuffer keyMetadata = manifestList.decryptKeyMetadata(em);
+      return newDecryptingInputFile(manifestList.location(), keyMetadata);
+    } else {
+      return newInputFile(manifestList.location());
+    }
+  }
+
   public InputFile newDecryptingInputFile(String path, ByteBuffer buffer) {
     return em.decrypt(wrap(io.newInputFile(path), buffer));
   }
 
   public InputFile newDecryptingInputFile(String path, long length, ByteBuffer buffer) {
-    // TODO: is the length correct for the encrypted file? It may be the length of the plaintext
-    // stream
     return em.decrypt(wrap(io.newInputFile(path, length), buffer));
   }
 

api/src/main/java/org/apache/iceberg/io/FileIO.java

@@ -24,6 +24,7 @@
 import org.apache.iceberg.DataFile;
 import org.apache.iceberg.DeleteFile;
 import org.apache.iceberg.ManifestFile;
+import org.apache.iceberg.ManifestListFile;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 
 /**
@@ -70,6 +71,15 @@ default InputFile newInputFile(ManifestFile manifest) {
     return newInputFile(manifest.path(), manifest.length());
   }
 
+  default InputFile newInputFile(ManifestListFile manifestList) {
+    Preconditions.checkArgument(
+        manifestList.encryptionKeyID() == null,
+        "Cannot decrypt manifest list: %s (use EncryptingFileIO)",
+        manifestList.location());
+    // cannot pass length because it is not tracked outside of key metadata
+    return newInputFile(manifestList.location());
+  }
+
   /** Get a {@link OutputFile} instance to write bytes to the file at the given path. */
   OutputFile newOutputFile(String path);
 

core/src/jmh/java/org/apache/iceberg/ManifestReadBenchmark.java

@@ -32,6 +32,7 @@
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
+import org.apache.iceberg.encryption.PlaintextEncryptionManager;
 import org.apache.iceberg.io.CloseableIterator;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -72,7 +73,13 @@ public void before() {
 
     try (ManifestListWriter listWriter =
         ManifestLists.write(
-            1, org.apache.iceberg.Files.localOutput(manifestListFile), 0, 1L, 0, 0L)) {
+            1,
+            org.apache.iceberg.Files.localOutput(manifestListFile),
+            PlaintextEncryptionManager.instance(),
+            0,
+            1L,
+            0,
+            0L)) {
       for (int i = 0; i < NUM_FILES; i++) {
         OutputFile manifestFile =
             org.apache.iceberg.Files.localOutput(

core/src/jmh/java/org/apache/iceberg/ManifestWriteBenchmark.java

@@ -27,6 +27,7 @@
 import java.util.UUID;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.io.FileUtils;
+import org.apache.iceberg.encryption.PlaintextEncryptionManager;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.collect.Maps;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -103,6 +104,7 @@ public void writeManifestFile(BenchmarkState state) throws IOException {
         ManifestLists.write(
             state.getFormatVersion(),
             org.apache.iceberg.Files.localOutput(manifestListFile),
+            PlaintextEncryptionManager.instance(),
             0,
             1L,
             0,

core/src/main/java/org/apache/iceberg/BaseManifestListFile.java

@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg;
+
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.EncryptionUtil;
+
+class BaseManifestListFile implements ManifestListFile, Serializable {
+  private final String location;
+  private final String encryptionKeyID;
+
+  BaseManifestListFile(String location, String encryptionKeyID) {
+    this.location = location;
+    this.encryptionKeyID = encryptionKeyID;
+  }
+
+  @Override
+  public String location() {
+    return location;
+  }
+
+  @Override
+  public String encryptionKeyID() {
+    return encryptionKeyID;
+  }
+
+  @Override
+  public ByteBuffer decryptKeyMetadata(EncryptionManager em) {
+    return EncryptionUtil.decryptManifestListKeyMetadata(this, em);
+  }
+}

core/src/main/java/org/apache/iceberg/BaseSnapshot.java

@@ -182,7 +182,9 @@ private void cacheManifests(FileIO fileIO) {
 
     if (allManifests == null) {
       // if manifests isn't set, then the snapshotFile is set and should be read to get the list
-      this.allManifests = ManifestLists.read(fileIO.newInputFile(manifestListLocation));
+      this.allManifests =
+          ManifestLists.read(
+              fileIO.newInputFile(new BaseManifestListFile(manifestListLocation, keyId)));
     }
 
     if (dataManifests == null || deleteManifests == null) {

core/src/main/java/org/apache/iceberg/ManifestListWriter.java

@@ -21,6 +21,10 @@
 import java.io.IOException;
 import java.util.Iterator;
 import java.util.Map;
+import org.apache.iceberg.encryption.EncryptionManager;
+import org.apache.iceberg.encryption.NativeEncryptionKeyMetadata;
+import org.apache.iceberg.encryption.NativeEncryptionOutputFile;
+import org.apache.iceberg.encryption.StandardEncryptionManager;
 import org.apache.iceberg.exceptions.RuntimeIOException;
 import org.apache.iceberg.io.FileAppender;
 import org.apache.iceberg.io.OutputFile;
@@ -29,9 +33,25 @@
 
 abstract class ManifestListWriter implements FileAppender<ManifestFile> {
   private final FileAppender<ManifestFile> writer;
+  private final StandardEncryptionManager standardEncryptionManager;
+  private final NativeEncryptionKeyMetadata manifestListKeyMetadata;
+  private final OutputFile outputFile;
+
+  private ManifestListWriter(
+      OutputFile file, EncryptionManager encryptionManager, Map<String, String> meta) {
+    if (encryptionManager instanceof StandardEncryptionManager) {
+      // ability to encrypt the manifest list key is introduced for standard encryption.
+      this.standardEncryptionManager = (StandardEncryptionManager) encryptionManager;
+      NativeEncryptionOutputFile encryptedFile = this.standardEncryptionManager.encrypt(file);
+      this.outputFile = encryptedFile.encryptingOutputFile();
+      this.manifestListKeyMetadata = encryptedFile.keyMetadata();
+    } else {
+      this.standardEncryptionManager = null;
+      this.outputFile = file;
+      this.manifestListKeyMetadata = null;
+    }
 
-  private ManifestListWriter(OutputFile file, Map<String, String> meta) {
-    this.writer = newAppender(file, meta);
+    this.writer = newAppender(outputFile, meta);
   }
 
   protected abstract ManifestFile prepare(ManifestFile manifest);
@@ -73,18 +93,31 @@ public Long nextRowId() {
     return null;
   }
 
+  public ManifestListFile toManifestListFile() {
+    if (manifestListKeyMetadata != null && manifestListKeyMetadata.encryptionKey() != null) {
+      manifestListKeyMetadata.copyWithLength(writer.length());
+      String manifestListKeyID =
+          standardEncryptionManager.addManifestListKeyMetadata(manifestListKeyMetadata);
+      return new BaseManifestListFile(outputFile.location(), manifestListKeyID);
+    } else {
+      return new BaseManifestListFile(outputFile.location(), null);
+    }
+  }
+
   static class V4Writer extends ManifestListWriter {
     private final V4Metadata.ManifestFileWrapper wrapper;
     private Long nextRowId;
 
     V4Writer(
         OutputFile snapshotFile,
+        EncryptionManager encryptionManager,
         long snapshotId,
         Long parentSnapshotId,
         long sequenceNumber,
         long firstRowId) {
       super(
           snapshotFile,
+          encryptionManager,
           ImmutableMap.of(
               "snapshot-id", String.valueOf(snapshotId),
               "parent-snapshot-id", String.valueOf(parentSnapshotId),
@@ -137,12 +170,14 @@ static class V3Writer extends ManifestListWriter {
 
     V3Writer(
         OutputFile snapshotFile,
+        EncryptionManager encryptionManager,
         long snapshotId,
         Long parentSnapshotId,
         long sequenceNumber,
         long firstRowId) {
       super(
           snapshotFile,
+          encryptionManager,
           ImmutableMap.of(
               "snapshot-id", String.valueOf(snapshotId),
               "parent-snapshot-id", String.valueOf(parentSnapshotId),
@@ -192,9 +227,15 @@ public Long nextRowId() {
   static class V2Writer extends ManifestListWriter {
     private final V2Metadata.ManifestFileWrapper wrapper;
 
-    V2Writer(OutputFile snapshotFile, long snapshotId, Long parentSnapshotId, long sequenceNumber) {
+    V2Writer(
+        OutputFile snapshotFile,
+        EncryptionManager encryptionManager,
+        long snapshotId,
+        Long parentSnapshotId,
+        long sequenceNumber) {
       super(
           snapshotFile,
+          encryptionManager,
           ImmutableMap.of(
               "snapshot-id", String.valueOf(snapshotId),
               "parent-snapshot-id", String.valueOf(parentSnapshotId),
@@ -228,9 +269,14 @@ protected FileAppender<ManifestFile> newAppender(OutputFile file, Map<String, St
   static class V1Writer extends ManifestListWriter {
     private final V1Metadata.ManifestFileWrapper wrapper = new V1Metadata.ManifestFileWrapper();
 
-    V1Writer(OutputFile snapshotFile, long snapshotId, Long parentSnapshotId) {
+    V1Writer(
+        OutputFile snapshotFile,
+        EncryptionManager encryptionManager,
+        long snapshotId,
+        Long parentSnapshotId) {
       super(
           snapshotFile,
+          encryptionManager,
           ImmutableMap.of(
               "snapshot-id", String.valueOf(snapshotId),
               "parent-snapshot-id", String.valueOf(parentSnapshotId),

core/src/main/java/org/apache/iceberg/ManifestLists.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.util.List;
+import org.apache.iceberg.encryption.EncryptionManager;
 import org.apache.iceberg.exceptions.RuntimeIOException;
 import org.apache.iceberg.io.CloseableIterable;
 import org.apache.iceberg.io.InputFile;
@@ -50,6 +51,7 @@ static List<ManifestFile> read(InputFile manifestList) {
   static ManifestListWriter write(
       int formatVersion,
       OutputFile manifestListFile,
+      EncryptionManager encryptionManager,
       long snapshotId,
       Long parentSnapshotId,
       long sequenceNumber,
@@ -60,16 +62,27 @@ static ManifestListWriter write(
             sequenceNumber == TableMetadata.INITIAL_SEQUENCE_NUMBER,
             "Invalid sequence number for v1 manifest list: %s",
             sequenceNumber);
-        return new ManifestListWriter.V1Writer(manifestListFile, snapshotId, parentSnapshotId);
+        return new ManifestListWriter.V1Writer(
+            manifestListFile, encryptionManager, snapshotId, parentSnapshotId);
       case 2:
         return new ManifestListWriter.V2Writer(
-            manifestListFile, snapshotId, parentSnapshotId, sequenceNumber);
+            manifestListFile, encryptionManager, snapshotId, parentSnapshotId, sequenceNumber);
       case 3:
         return new ManifestListWriter.V3Writer(
-            manifestListFile, snapshotId, parentSnapshotId, sequenceNumber, firstRowId);
+            manifestListFile,
+            encryptionManager,
+            snapshotId,
+            parentSnapshotId,
+            sequenceNumber,
+            firstRowId);
       case 4:
         return new ManifestListWriter.V4Writer(
-            manifestListFile, snapshotId, parentSnapshotId, sequenceNumber, firstRowId);
+            manifestListFile,
+            encryptionManager,
+            snapshotId,
+            parentSnapshotId,
+            sequenceNumber,
+            firstRowId);
     }
     throw new UnsupportedOperationException(
         "Cannot write manifest list for table version: " + formatVersion);