Skip to content

[improve][offload] Support credentials from offload policies for S3 and Aliyun OSS drivers #26232

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lhotari merged 2 commits into from
Jul 25, 2026
Merged

[improve][offload] Support credentials from offload policies for S3 and Aliyun OSS drivers #26232

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ff07ea3
[improve][offload] Support credentials from offload policies for S3 a…
KannarFr Jul 23, 2026
7941ebd
[improve][offload] Carry offload-policy credentials for aliyun-oss vi…
KannarFr Jul 23, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 16 additions & 8 deletions pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/OffloadPoliciesImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,14 @@ public class OffloadPoliciesImpl implements Serializable, OffloadPolicies {
CONFIGURATION_FIELDS = Collections.unmodifiableList(temp);
}

public static final List<String> INTERNAL_SUPPORTED_DRIVER = List.of("S3",
"aws-s3", "google-cloud-storage", "filesystem", "azureblob", "aliyun-oss");
public static final String DRIVER_S3 = "S3";
public static final String DRIVER_AWS_S3 = "aws-s3";
public static final String DRIVER_GOOGLE_CLOUD_STORAGE = "google-cloud-storage";
public static final String DRIVER_FILESYSTEM = "filesystem";
public static final String DRIVER_AZUREBLOB = "azureblob";
public static final String DRIVER_ALIYUN_OSS = "aliyun-oss";
public static final List<String> INTERNAL_SUPPORTED_DRIVER = List.of(DRIVER_S3,
DRIVER_AWS_S3, DRIVER_GOOGLE_CLOUD_STORAGE, DRIVER_FILESYSTEM, DRIVER_AZUREBLOB, DRIVER_ALIYUN_OSS);
public static final List<String> DRIVER_NAMES;
static {
String extraDrivers = System.getProperty("pulsar.extra.offload.drivers", "");
Expand Down Expand Up @@ -221,7 +227,8 @@ public static OffloadPoliciesImpl create(String driver, String region, String bu
.managedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes)
.managedLedgerOffloadedReadPriority(readPriority);

if (driver.equalsIgnoreCase(DRIVER_NAMES.get(0)) || driver.equalsIgnoreCase(DRIVER_NAMES.get(1))) {
if (driver.equalsIgnoreCase(DRIVER_S3) || driver.equalsIgnoreCase(DRIVER_AWS_S3)
|| driver.equalsIgnoreCase(DRIVER_ALIYUN_OSS)) {
if (role != null) {
builder.s3ManagedLedgerOffloadRole(role);
}
Expand All @@ -240,7 +247,7 @@ public static OffloadPoliciesImpl create(String driver, String region, String bu
.s3ManagedLedgerOffloadServiceEndpoint(endpoint)
.s3ManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes)
.s3ManagedLedgerOffloadReadBufferSizeInBytes(readBufferSizeInBytes);
} else if (driver.equalsIgnoreCase(DRIVER_NAMES.get(2))) {
} else if (driver.equalsIgnoreCase(DRIVER_GOOGLE_CLOUD_STORAGE)) {
builder.gcsManagedLedgerOffloadRegion(region)
.gcsManagedLedgerOffloadBucket(bucket)
.gcsManagedLedgerOffloadMaxBlockSizeInBytes(maxBlockSizeInBytes)
Expand Down Expand Up @@ -310,22 +317,23 @@ public boolean isS3Driver() {
if (managedLedgerOffloadDriver == null) {
return false;
}
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(0))
|| managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(1));
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_S3)
|| managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_AWS_S3)
|| managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_ALIYUN_OSS);
}

public boolean isGcsDriver() {
if (managedLedgerOffloadDriver == null) {
return false;
}
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(2));
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_GOOGLE_CLOUD_STORAGE);
}

public boolean isFileSystemDriver() {
if (managedLedgerOffloadDriver == null) {
return false;
}
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_NAMES.get(3));
return managedLedgerOffloadDriver.equalsIgnoreCase(DRIVER_FILESYSTEM);
}

public boolean bucketValid() {
Expand Down
46 changes: 46 additions & 0 deletions pulsar-common/src/test/java/org/apache/pulsar/common/policies/data/OffloadPoliciesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,52 @@ public void testS3Configuration() {
Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadThresholdInSeconds(), offloadThresholdInSeconds);
}

@Test
public void testAliyunOssConfiguration() {
final String driver = "aliyun-oss";
final String region = "test-region";
final String bucket = "test-bucket";
final String role = "test-role";
final String roleSessionName = "test-role-session-name";
final String credentialId = "test-credential-id";
final String credentialSecret = "test-credential-secret";
final String endPoint = "test-endpoint";
final Integer maxBlockSizeInBytes = 5 * M;
final Integer readBufferSizeInBytes = 2 * M;
final Long offloadThresholdInBytes = 10L * M;
final Long offloadThresholdInSeconds = 1000L;
final Long offloadDeletionLagInMillis = 5L * MIN;

OffloadPoliciesImpl offloadPolicies = OffloadPoliciesImpl.create(
driver,
region,
bucket,
endPoint,
role,
roleSessionName,
credentialId,
credentialSecret,
maxBlockSizeInBytes,
readBufferSizeInBytes,
offloadThresholdInBytes,
offloadThresholdInSeconds,
offloadDeletionLagInMillis,
OffloadedReadPriority.TIERED_STORAGE_FIRST
);

Assert.assertTrue(offloadPolicies.isS3Driver());
Assert.assertEquals(offloadPolicies.getManagedLedgerOffloadDriver(), driver);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadRegion(), region);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadBucket(), bucket);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadServiceEndpoint(), endPoint);
// The CLI create() path must carry the credentials under the s3-prefixed keys the
// S3-compatible aliyun-oss offloader reads, not silently drop them.
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadCredentialId(), credentialId);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadCredentialSecret(), credentialSecret);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadMaxBlockSizeInBytes(), maxBlockSizeInBytes);
Assert.assertEquals(offloadPolicies.getS3ManagedLedgerOffloadReadBufferSizeInBytes(), readBufferSizeInBytes);
}

@Test
public void testGcsConfiguration() {
final String driver = "google-cloud-storage";
Expand Down
19 changes: 19 additions & 0 deletions ...n/java/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,6 +433,25 @@ public String getAWSSecretKey() {
};

static final CredentialBuilder S3_CREDENTIAL_BUILDER = (TieredStorageConfiguration config) -> {
if (config.getCredentials() != null) {
return;
}
// Credentials provided in the tiered storage configuration take
// precedence over the environment variables. Offload policies carry
// credentials under the s3-prefixed keys for every S3-compatible
// driver (see OffloadPoliciesImpl), so those are the keys accepted.
String configId = config.getConfigProperty(S3_ID_FIELD);
String configSecret = config.getConfigProperty(S3_SECRET_FIELD);
if (StringUtils.isNotBlank(configId) || StringUtils.isNotBlank(configSecret)) {
if (StringUtils.isBlank(configId) || StringUtils.isBlank(configSecret)) {
throw new IllegalArgumentException(
"Both " + S3_ID_FIELD + " and " + S3_SECRET_FIELD
+ " must be set when providing offload credentials in the configuration");
}
Credentials credentials = new Credentials(configId, configSecret);
config.setProviderCredentials(() -> credentials);
return;
}
String accountName = System.getenv().getOrDefault("ACCESS_KEY_ID", "");
// For forward compatibility
if (StringUtils.isEmpty(accountName.trim())) {
Expand Down
46 changes: 46 additions & 0 deletions ...va/org/apache/bookkeeper/mledger/offload/jcloud/provider/JCloudBlobStoreProviderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@
package org.apache.bookkeeper.mledger.offload.jcloud.provider;

import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import java.util.HashMap;
import java.util.Map;
import org.jclouds.domain.Credentials;
import org.testng.annotations.Test;

public class JCloudBlobStoreProviderTest {
Expand Down Expand Up @@ -130,4 +132,48 @@ public void s3ValidationBucketMissed() {
TieredStorageConfiguration configuration = new TieredStorageConfiguration(map);
configuration.getProvider().validate(configuration);
}

// Offload policies (OffloadPoliciesImpl) carry credentials under the
// s3-prefixed keys for every S3-compatible driver
private TieredStorageConfiguration credentialConfig(String driver, String id, String secret) {
Map<String, String> map = new HashMap<>();
map.put("managedLedgerOffloadDriver", driver);
map.put("managedLedgerOffloadServiceEndpoint", "http://storage.service");
map.put("managedLedgerOffloadBucket", "test-bucket");
if (id != null) {
map.put("s3ManagedLedgerOffloadCredentialId", id);
}
if (secret != null) {
map.put("s3ManagedLedgerOffloadCredentialSecret", secret);
}
return new TieredStorageConfiguration(map);
}

private void assertCredentialsFromConfig(String driver) {
TieredStorageConfiguration configuration =
credentialConfig(driver, "config-access-id", "config-access-secret");
configuration.getProvider().buildCredentials(configuration);
assertNotNull(configuration.getProviderCredentials());
Credentials credentials = configuration.getProviderCredentials().get();
assertEquals(credentials.identity, "config-access-id");
assertEquals(credentials.credential, "config-access-secret");
}

@Test
public void s3CredentialsFromConfigTest() {
assertCredentialsFromConfig("S3");
}

@Test
public void aliyunOssCredentialsFromConfigTest() {
assertCredentialsFromConfig("aliyun-oss");
}

@Test(expectedExceptions = IllegalArgumentException.class,
expectedExceptionsMessageRegExp = "Both s3ManagedLedgerOffloadCredentialId and "
+ "s3ManagedLedgerOffloadCredentialSecret must be set.*")
public void s3PartialCredentialsFromConfigTest() {
TieredStorageConfiguration configuration = credentialConfig("S3", "config-access-id", null);
configuration.getProvider().buildCredentials(configuration);
}
}
Loading