feat(auth): implement SigV4 authentication for REST catalog

.github/workflows/aws_test.yml

@@ -0,0 +1,174 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: AWS Tests
+
+on:
+  push:
+    branches:
+      - '**'
+      - '!dependabot/**'
+    tags:
+      - '**'
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+concurrency:
+  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+env:
+  ICEBERG_HOME: /tmp/iceberg
+
+jobs:
+  aws:
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    name: AWS (${{ matrix.title }})
+    runs-on: ${{ matrix.runs-on }}
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - title: Ubuntu 24.04, S3 + SigV4, bundled AWS SDK
+            runs-on: ubuntu-24.04
+            CC: gcc-14
+            CXX: g++-14
+            s3: "ON"
+            sigv4: "ON"
+            bundle_awssdk: "ON"
+          - title: Ubuntu 24.04, S3 + SigV4, system AWS SDK
+            runs-on: ubuntu-24.04
+            CC: gcc-14
+            CXX: g++-14
+            s3: "ON"
+            sigv4: "ON"
+            bundle_awssdk: "OFF"
+            aws-sdk-features: core,config,s3,identity-management,sts,transfer
+          - title: macOS 26 ARM64, S3, bundled AWS SDK
+            runs-on: macos-26
+            s3: "ON"
+            sigv4: "OFF"
+            bundle_awssdk: "ON"
+    env:
+      ICEBERG_TEST_S3_URI: s3://iceberg-test
+      AWS_ACCESS_KEY_ID: minio
+      AWS_SECRET_ACCESS_KEY: minio123
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ENDPOINT_URL: http://127.0.0.1:9000
+      AWS_EC2_METADATA_DISABLED: "TRUE"
+    steps:
+      - name: Checkout iceberg-cpp
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Install dependencies on Ubuntu
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
+        shell: bash
+        run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Cache vcpkg packages
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        id: vcpkg-cache
+        with:
+          path: /usr/local/share/vcpkg/installed
+          key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }}
+      - name: Install AWS SDK via vcpkg
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
+        shell: bash
+        # Retry to ride out transient GitHub/mirror download failures (504s).
+        run: |
+          for attempt in 1 2 3; do
+            if vcpkg install "aws-sdk-cpp[${{ matrix.aws-sdk-features }}]:x64-linux"; then
+              exit 0
+            fi
+            echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s"
+            sleep 30
+          done
+          echo "::error::vcpkg install failed after 3 attempts"
+          exit 1
+      - name: Set Ubuntu Compilers
+        if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
+        run: |
+          echo "CC=${{ matrix.CC }}" >> $GITHUB_ENV
+          echo "CXX=${{ matrix.CXX }}" >> $GITHUB_ENV
+      - name: Start MinIO
+        if: ${{ matrix.s3 == 'ON' }}
+        shell: bash
+        run: bash ci/scripts/start_minio.sh
+      - name: Build and test Iceberg
+        shell: bash
+        env:
+          CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
+        run: ci/scripts/build_iceberg.sh "$(pwd)" OFF OFF ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
+
+  # Exercise the Meson build with SigV4 enabled (resolves aws-cpp-sdk-core via
+  # its CMake config, not pkg-config whose Cflags force -std=c++11).
+  meson-sigv4:
+    if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
+    name: Meson SigV4 (AMD64 Ubuntu 24.04)
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
+        with:
+          python-version: '3.x'
+      - name: Checkout iceberg-cpp
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Install build dependencies
+        shell: bash
+        run: |
+          sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements.txt
+      - name: Cache vcpkg packages
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+        id: vcpkg-cache
+        with:
+          path: /usr/local/share/vcpkg/installed
+          key: vcpkg-x64-linux-aws-sdk-cpp-core-${{ hashFiles('.github/workflows/aws_test.yml') }}
+      - name: Install AWS SDK via vcpkg
+        if: ${{ steps.vcpkg-cache.outputs.cache-hit != 'true' }}
+        shell: bash
+        # Retry to ride out transient GitHub/mirror download failures (504s).
+        run: |
+          for attempt in 1 2 3; do
+            if vcpkg install aws-sdk-cpp[core]:x64-linux; then
+              exit 0
+            fi
+            echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s"
+            sleep 30
+          done
+          echo "::error::vcpkg install failed after 3 attempts"
+          exit 1
+      - name: Set Ubuntu Compilers
+        run: |
+          echo "CC=gcc-14" >> $GITHUB_ENV
+          echo "CXX=g++-14" >> $GITHUB_ENV
+      - name: Build and test Iceberg
+        shell: bash
+        env:
+          CMAKE_PREFIX_PATH: /usr/local/share/vcpkg/installed/x64-linux
+        run: |
+          meson setup builddir -Dsigv4=enabled
+          meson compile -C builddir
+          meson test -C builddir --timeout-multiplier 0 --print-errorlogs

CMakeLists.txt

@@ -53,6 +53,8 @@ option(ICEBERG_SQL_SQLITE "Build the SQLite connector for the SQL catalog" OFF)
 option(ICEBERG_SQL_POSTGRESQL "Build the PostgreSQL connector for the SQL catalog" OFF)
 option(ICEBERG_SQL_MYSQL "Build the MySQL connector for the SQL catalog" OFF)
 option(ICEBERG_S3 "Build with S3 support" OFF)
+option(ICEBERG_SIGV4 "Build with SigV4 support" OFF)
+option(ICEBERG_BUNDLE_AWSSDK "Bundle AWS SDK for S3/SigV4 support" ON)
 option(ICEBERG_ENABLE_ASAN "Enable Address Sanitizer" OFF)
 option(ICEBERG_ENABLE_UBSAN "Enable Undefined Behavior Sanitizer" OFF)
 
@@ -76,12 +78,6 @@ if(ICEBERG_BUILD_REST_INTEGRATION_TESTS AND WIN32)
   message(WARNING "Cannot build rest integration test on Windows, turning it off.")
 endif()
 
-# ICEBERG_S3 requires ICEBERG_BUILD_BUNDLE
-if(NOT ICEBERG_BUILD_BUNDLE AND ICEBERG_S3)
-  set(ICEBERG_S3 OFF)
-  message(STATUS "ICEBERG_S3 is disabled because ICEBERG_BUILD_BUNDLE is OFF")
-endif()
-
 include(CMakeParseArguments)
 include(IcebergBuildUtils)
 include(IcebergSanitizer)

ci/scripts/build_iceberg.sh

@@ -17,7 +17,7 @@
 # specific language governing permissions and limitations
 # under the License.
 #
-# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF]
+# Usage: build_iceberg.sh <source_dir> [rest_integration_tests=OFF] [sccache=OFF] [s3=OFF] [sigv4=OFF] [bundle_awssdk=ON]
 
 set -eux
 
@@ -26,6 +26,8 @@ build_dir=${1}/build
 build_rest_integration_test=${2:-OFF}
 build_enable_sccache=${3:-OFF}
 build_enable_s3=${4:-OFF}
+build_enable_sigv4=${5:-OFF}
+build_bundle_awssdk=${6:-ON}
 run_tests=${ICEBERG_RUN_TESTS:-ON}
 
 mkdir ${build_dir}
@@ -49,10 +51,26 @@ else
     CMAKE_ARGS+=("-DICEBERG_S3=OFF")
 fi
 
+if [[ "${build_enable_sigv4}" == "ON" ]]; then
+    CMAKE_ARGS+=("-DICEBERG_SIGV4=ON")
+else
+    CMAKE_ARGS+=("-DICEBERG_SIGV4=OFF")
+fi
+
+if [[ "${build_bundle_awssdk}" == "ON" ]]; then
+    CMAKE_ARGS+=("-DICEBERG_BUNDLE_AWSSDK=ON")
+else
+    CMAKE_ARGS+=("-DICEBERG_BUNDLE_AWSSDK=OFF")
+fi
+
 if is_windows; then
     CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake")
     CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Release")
 else
+    # Pass an externally provided toolchain (e.g. vcpkg for the SigV4 job)
+    if [[ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]]; then
+        CMAKE_ARGS+=("-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
+    fi
     CMAKE_ARGS+=("-DCMAKE_BUILD_TYPE=Debug")
 fi
 

cmake_modules/IcebergThirdpartyToolchain.cmake

@@ -19,6 +19,48 @@
 # third party libraries.
 set(ICEBERG_SYSTEM_DEPENDENCIES)
 set(ICEBERG_ARROW_INSTALL_INTERFACE_LIBS)
+set(ICEBERG_AWSSDK_BUNDLED FALSE)
+if(ICEBERG_S3 AND ICEBERG_BUNDLE_AWSSDK)
+  if(NOT ICEBERG_BUILD_BUNDLE)
+    message(FATAL_ERROR "ICEBERG_BUNDLE_AWSSDK requires ICEBERG_BUILD_BUNDLE to be ON")
+  endif()
+  set(ICEBERG_AWSSDK_BUNDLED TRUE)
+endif()
+
+set(ICEBERG_AWSSDK_COMPONENTS)
+if(NOT ICEBERG_AWSSDK_BUNDLED)
+  if(ICEBERG_S3)
+    list(APPEND
+         ICEBERG_AWSSDK_COMPONENTS
+         core
+         config
+         s3
+         transfer
+         identity-management
+         sts)
+  elseif(ICEBERG_SIGV4)
+    list(APPEND ICEBERG_AWSSDK_COMPONENTS core)
+  endif()
+endif()
+
+# ----------------------------------------------------------------------
+# AWS SDK for C++
+
+function(resolve_aws_sdk_dependency)
+  if(NOT ICEBERG_AWSSDK_COMPONENTS)
+    return()
+  endif()
+  find_package(AWSSDK REQUIRED COMPONENTS ${ICEBERG_AWSSDK_COMPONENTS})
+  list(APPEND ICEBERG_SYSTEM_DEPENDENCIES AWSSDK)
+  set(ICEBERG_SYSTEM_DEPENDENCIES
+      ${ICEBERG_SYSTEM_DEPENDENCIES}
+      PARENT_SCOPE)
+  # Forwarded to find_dependency(AWSSDK ...) in iceberg-config.cmake.in so
+  # downstream installed builds load the same AWS SDK targets.
+  set(ICEBERG_FIND_EXTRA_ARGS_AWSSDK
+      "COMPONENTS;${ICEBERG_AWSSDK_COMPONENTS}"
+      PARENT_SCOPE)
+endfunction()
 
 # ----------------------------------------------------------------------
 # Versions and URLs for toolchain builds
@@ -111,6 +153,9 @@ function(resolve_arrow_dependency)
   set(ARROW_POSITION_INDEPENDENT_CODE ON)
   set(ARROW_DEPENDENCY_SOURCE "BUNDLED")
   set(ARROW_WITH_ZLIB ON)
+  if(ICEBERG_S3 AND NOT ICEBERG_AWSSDK_BUNDLED)
+    set(AWSSDK_SOURCE "SYSTEM")
+  endif()
   set(ZLIB_SOURCE "SYSTEM")
   set(ARROW_VERBOSE_THIRDPARTY_BUILD OFF)
   set(CMAKE_CXX_STANDARD 20)
@@ -620,6 +665,13 @@ resolve_nanoarrow_dependency()
 resolve_croaring_dependency()
 resolve_nlohmann_json_dependency()
 
+if(ICEBERG_S3 OR ICEBERG_SIGV4)
+  if(ICEBERG_SIGV4 AND NOT ICEBERG_BUILD_REST)
+    message(FATAL_ERROR "ICEBERG_SIGV4 requires ICEBERG_BUILD_REST to be ON")
+  endif()
+  resolve_aws_sdk_dependency()
+endif()
+
 if(ICEBERG_BUILD_BUNDLE)
   resolve_arrow_dependency()
   resolve_avro_dependency()