diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1970ea2144..9087de5e58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -26,7 +26,7 @@ jobs: DAPR_RUNTIME_VER: 1.0.0-rc.2 DAPR_INSTALL_URL: https://raw.githubusercontent.com/dapr/cli/3dacfb672d55f1436c249057aaebbe597e1066f3/install/install.sh DAPR_CLI_REF: - DAPR_REF: 4678e477562e7e35a1d6ba04a9b17b6c9c00a025 + DAPR_REF: f4327d91c3cd921b26077d99abe0f237c9a38f5d OSSRH_USER_TOKEN: ${{ secrets.OSSRH_USER_TOKEN }} OSSRH_PWD_TOKEN: ${{ secrets.OSSRH_PWD_TOKEN }} GPG_KEY: ${{ secrets.GPG_KEY }} @@ -91,8 +91,6 @@ jobs: run: | docker-compose -f ./sdk-tests/deploy/local-test-vault.yml up -d docker ps - - name: Setup Vault's test token - run: echo myroot > /tmp/.hashicorp_vault_token - name: Clean up files run: mvn clean - name: Build sdk diff --git a/pom.xml b/pom.xml index 76e32ddbdf..03f8dc25ca 100644 --- a/pom.xml +++ b/pom.xml @@ -16,7 +16,7 @@ UTF-8 1.33.1 3.13.0 - https://raw.githubusercontent.com/dapr/dapr/33dc3361e99f75e88628e0abfc8b04d3d7e8bc5f/dapr/proto + https://raw.githubusercontent.com/dapr/dapr/4a6369caaba9cf46eae9bfa4fa6e76b474854c89/dapr/proto 1.6.2 3.1.1 1.8 diff --git a/sdk-tests/.hashicorp_vault_token b/sdk-tests/.hashicorp_vault_token new file mode 100644 index 0000000000..18309abed3 --- /dev/null +++ b/sdk-tests/.hashicorp_vault_token @@ -0,0 +1 @@ +myroot diff --git a/sdk-tests/components/vault.yaml b/sdk-tests/components/vault.yaml index 633c6426d3..7f52257170 100644 --- a/sdk-tests/components/vault.yaml +++ b/sdk-tests/components/vault.yaml @@ -10,6 +10,6 @@ spec: - name: skipVerify value : true - name: vaultTokenMountPath - value : "/tmp/.hashicorp_vault_token" + value : ".hashicorp_vault_token" - name: vaultKVPrefix value : "dapr" diff --git a/sdk-tests/src/test/java/io/dapr/it/secrets/SecretsClientIT.java b/sdk-tests/src/test/java/io/dapr/it/secrets/SecretsClientIT.java index d87024f5ec..45cea437c0 100644 --- a/sdk-tests/src/test/java/io/dapr/it/secrets/SecretsClientIT.java +++ b/sdk-tests/src/test/java/io/dapr/it/secrets/SecretsClientIT.java @@ -21,10 +21,12 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.UUID; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Test Secrets Store APIs using Harshicorp's vault. @@ -104,6 +106,26 @@ public void getSecret() throws Exception { assertEquals("The Metrics IV", data.get("title")); } + @Test + public void getBulkSecret() throws Exception { + String key1 = UUID.randomUUID().toString(); + writeSecret(key1, new HashMap<>() {{ + put("title", "The Metrics IV"); + put("year", "2020"); + }}); + String key2 = UUID.randomUUID().toString(); + writeSecret(key2, "name", "Jon Doe"); + + Map> data = daprClient.getBulkSecret(SECRETS_STORE_NAME).block(); + // There can be other keys from other runs or test cases, so we are good with at least two. + assertTrue(data.size() >= 2); + assertEquals(2, data.get(key1).size()); + assertEquals("The Metrics IV", data.get(key1).get("title")); + assertEquals("2020", data.get(key1).get("year")); + assertEquals(1, data.get(key2).size()); + assertEquals("Jon Doe", data.get(key2).get("name")); + } + @Test(expected = RuntimeException.class) public void getSecretKeyNotFound() { daprClient.getSecret(SECRETS_STORE_NAME, "unknownKey").block(); @@ -115,7 +137,10 @@ public void getSecretStoreNotFound() throws Exception { } private static void writeSecret(String secretName, String key, String value) throws Exception { - Map secrets = Collections.singletonMap(key, value); + writeSecret(secretName, Collections.singletonMap(key, value)); + } + + private static void writeSecret(String secretName, Map secrets) throws Exception { vault.logical().write("secret/" + PREFIX + "/" + secretName, secrets); } diff --git a/sdk/src/main/java/io/dapr/client/AbstractDaprClient.java b/sdk/src/main/java/io/dapr/client/AbstractDaprClient.java index 3665f062de..f435d2e00d 100644 --- a/sdk/src/main/java/io/dapr/client/AbstractDaprClient.java +++ b/sdk/src/main/java/io/dapr/client/AbstractDaprClient.java @@ -9,6 +9,8 @@ import io.dapr.client.domain.DeleteStateRequestBuilder; import io.dapr.client.domain.ExecuteStateTransactionRequest; import io.dapr.client.domain.ExecuteStateTransactionRequestBuilder; +import io.dapr.client.domain.GetBulkSecretRequest; +import io.dapr.client.domain.GetBulkSecretRequestBuilder; import io.dapr.client.domain.GetBulkStateRequestBuilder; import io.dapr.client.domain.GetSecretRequest; import io.dapr.client.domain.GetSecretRequestBuilder; @@ -395,4 +397,23 @@ public Mono> getSecret(String storeName, String secretName) return this.getSecret(storeName, secretName, null); } + /** + * {@inheritDoc} + */ + @Override + public Mono>> getBulkSecret(String storeName) { + return this.getBulkSecret(storeName, null); + } + + /** + * {@inheritDoc} + */ + @Override + public Mono>> getBulkSecret(String storeName, Map metadata) { + GetBulkSecretRequest request = new GetBulkSecretRequestBuilder(storeName) + .withMetadata(metadata) + .build(); + return this.getBulkSecret(request).map(r -> r.getObject() == null ? Collections.EMPTY_MAP : r.getObject()); + } + } \ No newline at end of file diff --git a/sdk/src/main/java/io/dapr/client/DaprClient.java b/sdk/src/main/java/io/dapr/client/DaprClient.java index 20f7a6e260..12feb99c21 100644 --- a/sdk/src/main/java/io/dapr/client/DaprClient.java +++ b/sdk/src/main/java/io/dapr/client/DaprClient.java @@ -7,6 +7,7 @@ import io.dapr.client.domain.DeleteStateRequest; import io.dapr.client.domain.ExecuteStateTransactionRequest; +import io.dapr.client.domain.GetBulkSecretRequest; import io.dapr.client.domain.GetBulkStateRequest; import io.dapr.client.domain.GetSecretRequest; import io.dapr.client.domain.GetStateRequest; @@ -525,4 +526,29 @@ Mono executeStateTransaction(String storeName, * @return Key-value pairs for the secret. */ Mono>> getSecret(GetSecretRequest request); + + /** + * Fetches all secrets from the configured vault. + * + * @param storeName Name of vault component in Dapr. + * @return Key-value pairs for all the secrets in the state store. + */ + Mono>> getBulkSecret(String storeName); + + /** + * Fetches all secrets from the configured vault. + * + * @param storeName Name of vault component in Dapr. + * @param metadata Optional metadata. + * @return Key-value pairs for all the secrets in the state store. + */ + Mono>> getBulkSecret(String storeName, Map metadata); + + /** + * Fetches all secrets from the configured vault. + * + * @param request Request to fetch secret. + * @return Key-value pairs for the secret. + */ + Mono>>> getBulkSecret(GetBulkSecretRequest request); } \ No newline at end of file diff --git a/sdk/src/main/java/io/dapr/client/DaprClientGrpc.java b/sdk/src/main/java/io/dapr/client/DaprClientGrpc.java index 9e673c8cea..47176a4041 100644 --- a/sdk/src/main/java/io/dapr/client/DaprClientGrpc.java +++ b/sdk/src/main/java/io/dapr/client/DaprClientGrpc.java @@ -11,6 +11,7 @@ import com.google.protobuf.Empty; import io.dapr.client.domain.DeleteStateRequest; import io.dapr.client.domain.ExecuteStateTransactionRequest; +import io.dapr.client.domain.GetBulkSecretRequest; import io.dapr.client.domain.GetBulkStateRequest; import io.dapr.client.domain.GetSecretRequest; import io.dapr.client.domain.GetStateRequest; @@ -55,9 +56,11 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -469,7 +472,7 @@ private CommonProtos.StateItem.Builder buildStateRequest(State state) thr CommonProtos.StateItem.Builder stateBuilder = CommonProtos.StateItem.newBuilder(); if (state.getEtag() != null) { - stateBuilder.setEtag(state.getEtag()); + stateBuilder.setEtag(CommonProtos.Etag.newBuilder().setValue(state.getEtag()).build()); } if (state.getMetadata() != null) { stateBuilder.putAllMetadata(state.getMetadata()); @@ -532,7 +535,7 @@ public Mono> deleteState(DeleteStateRequest request) { builder.putAllMetadata(metadata); } if (etag != null) { - builder.setEtag(etag); + builder.setEtag(CommonProtos.Etag.newBuilder().setValue(etag).build()); } if (optionBuilder != null) { @@ -626,6 +629,44 @@ public Mono>> getSecret(GetSecretRequest request) { ).map(it -> new Response<>(context, it.getDataMap())); } + /** + * {@inheritDoc} + */ + @Override + public Mono>>> getBulkSecret(GetBulkSecretRequest request) { + try { + final String storeName = request.getStoreName(); + final Map metadata = request.getMetadata(); + final Context context = request.getContext(); + if ((storeName == null) || (storeName.trim().isEmpty())) { + throw new IllegalArgumentException("Secret store name cannot be null or empty."); + } + + DaprProtos.GetBulkSecretRequest.Builder builder = DaprProtos.GetBulkSecretRequest.newBuilder() + .setStoreName(storeName); + if (metadata != null) { + builder.putAllMetadata(metadata); + } + + DaprProtos.GetBulkSecretRequest envelope = builder.build(); + + return this.createMono(context, it -> asyncStub.getBulkSecret(envelope, it)) + .map(it -> { + Map secretsMap = it.getDataMap(); + if (secretsMap == null) { + return Collections.EMPTY_MAP; + } + return secretsMap + .entrySet() + .stream() + .collect(Collectors.toMap(s -> s.getKey(), s -> s.getValue().getSecretsMap())); + }) + .map(s -> new Response<>(context, s)); + } catch (Exception ex) { + return DaprException.wrapMono(ex); + } + } + /** * Closes the ManagedChannel for GRPC. * @see io.grpc.ManagedChannel#shutdown() diff --git a/sdk/src/main/java/io/dapr/client/DaprClientHttp.java b/sdk/src/main/java/io/dapr/client/DaprClientHttp.java index d8dbbfbf5b..9e5a5b9d69 100644 --- a/sdk/src/main/java/io/dapr/client/DaprClientHttp.java +++ b/sdk/src/main/java/io/dapr/client/DaprClientHttp.java @@ -9,6 +9,7 @@ import com.google.common.base.Strings; import io.dapr.client.domain.DeleteStateRequest; import io.dapr.client.domain.ExecuteStateTransactionRequest; +import io.dapr.client.domain.GetBulkSecretRequest; import io.dapr.client.domain.GetBulkStateRequest; import io.dapr.client.domain.GetSecretRequest; import io.dapr.client.domain.GetStateRequest; @@ -591,6 +592,42 @@ public Mono>> getSecret(GetSecretRequest request) { .map(m -> new Response<>(context, m)); } + /** + * {@inheritDoc} + */ + @Override + public Mono>>> getBulkSecret(GetBulkSecretRequest request) { + String secretStoreName = request.getStoreName(); + Map metadata = request.getMetadata(); + Context context = request.getContext(); + try { + if ((secretStoreName == null) || (secretStoreName.trim().isEmpty())) { + throw new IllegalArgumentException("Secret store name cannot be null or empty."); + } + } catch (Exception e) { + return DaprException.wrapMono(e); + } + + Map queryArgs = metadataToQueryArgs(metadata); + String[] pathSegments = new String[]{ DaprHttp.API_VERSION, "secrets", secretStoreName, "bulk"}; + return this.client + .invokeApi(DaprHttp.HttpMethods.GET.name(), pathSegments, queryArgs, (String)null, null, context) + .flatMap(response -> { + try { + Map m = INTERNAL_SERIALIZER.deserialize(response.getBody(), Map.class); + if (m == null) { + return Mono.just(Collections.EMPTY_MAP); + } + + return Mono.just(m); + } catch (IOException e) { + return DaprException.wrapMono(e); + } + }) + .map(m -> (Map>)m) + .map(m -> new Response<>(context, m)); + } + /** * {@inheritDoc} */ diff --git a/sdk/src/main/java/io/dapr/client/DaprClientProxy.java b/sdk/src/main/java/io/dapr/client/DaprClientProxy.java index dcab274958..5d1b701726 100644 --- a/sdk/src/main/java/io/dapr/client/DaprClientProxy.java +++ b/sdk/src/main/java/io/dapr/client/DaprClientProxy.java @@ -7,6 +7,7 @@ import io.dapr.client.domain.DeleteStateRequest; import io.dapr.client.domain.ExecuteStateTransactionRequest; +import io.dapr.client.domain.GetBulkSecretRequest; import io.dapr.client.domain.GetBulkStateRequest; import io.dapr.client.domain.GetSecretRequest; import io.dapr.client.domain.GetStateRequest; @@ -465,6 +466,30 @@ public Mono>> getSecret(GetSecretRequest request) { return client.getSecret(request); } + /** + * {@inheritDoc} + */ + @Override + public Mono>> getBulkSecret(String storeName) { + return client.getBulkSecret(storeName); + } + + /** + * {@inheritDoc} + */ + @Override + public Mono>> getBulkSecret(String storeName, Map metadata) { + return client.getBulkSecret(storeName, metadata); + } + + /** + * {@inheritDoc} + */ + @Override + public Mono>>> getBulkSecret(GetBulkSecretRequest request) { + return client.getBulkSecret(request); + } + /** * {@inheritDoc} */ diff --git a/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequest.java b/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequest.java new file mode 100644 index 0000000000..addd28b7df --- /dev/null +++ b/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequest.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT License. + */ + +package io.dapr.client.domain; + +import io.opentelemetry.context.Context; + +import java.util.Map; + +/** + * A request to get a secret by key. + */ +public class GetBulkSecretRequest { + + private String storeName; + + private Map metadata; + + private Context context; + + public String getStoreName() { + return storeName; + } + + void setStoreName(String storeName) { + this.storeName = storeName; + } + + public Map getMetadata() { + return metadata; + } + + void setMetadata(Map metadata) { + this.metadata = metadata; + } + + public Context getContext() { + return context; + } + + void setContext(Context context) { + this.context = context; + } +} diff --git a/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequestBuilder.java b/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequestBuilder.java new file mode 100644 index 0000000000..320d9cc725 --- /dev/null +++ b/sdk/src/main/java/io/dapr/client/domain/GetBulkSecretRequestBuilder.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) Microsoft Corporation. + * Licensed under the MIT License. + */ + +package io.dapr.client.domain; + +import io.opentelemetry.context.Context; + +import java.util.Collections; +import java.util.Map; + +/** + * Builds a request to fetch all secrets of a secret store. + */ +public class GetBulkSecretRequestBuilder { + + private final String storeName; + + private Map metadata; + + private Context context; + + public GetBulkSecretRequestBuilder(String storeName) { + this.storeName = storeName; + } + + public GetBulkSecretRequestBuilder withMetadata(Map metadata) { + this.metadata = metadata == null ? null : Collections.unmodifiableMap(metadata); + return this; + } + + public GetBulkSecretRequestBuilder withContext(Context context) { + this.context = context; + return this; + } + + /** + * Builds a request object. + * @return Request object. + */ + public GetBulkSecretRequest build() { + GetBulkSecretRequest request = new GetBulkSecretRequest(); + request.setStoreName(this.storeName); + request.setMetadata(this.metadata); + request.setContext(this.context); + return request; + } + +} diff --git a/sdk/src/test/java/io/dapr/client/DaprClientGrpcTest.java b/sdk/src/test/java/io/dapr/client/DaprClientGrpcTest.java index 3f1179e05d..525b32c51d 100644 --- a/sdk/src/test/java/io/dapr/client/DaprClientGrpcTest.java +++ b/sdk/src/test/java/io/dapr/client/DaprClientGrpcTest.java @@ -50,6 +50,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Collectors; import static io.dapr.utils.TestUtils.assertThrowsDaprException; import static io.dapr.utils.TestUtils.findFreePort; @@ -1927,6 +1928,74 @@ public void getSecretsWithMetadata() { assertEquals(expectedValue, result.get(expectedKey)); } + @Test + public void getBulkSecrets() { + DaprProtos.GetBulkSecretResponse responseEnvelope = buildGetBulkSecretResponse( + new HashMap>() {{ + put("one", Collections.singletonMap("mysecretkey", "mysecretvalue")); + put("two", new HashMap() {{ + put("a", "1"); + put("b", "2"); + }}); + }}); + + doAnswer((Answer) invocation -> { + DaprProtos.GetBulkSecretRequest req = invocation.getArgument(0); + assertEquals(SECRET_STORE_NAME, req.getStoreName()); + assertEquals(0, req.getMetadataCount()); + + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + observer.onNext(responseEnvelope); + observer.onCompleted(); + return null; + }).when(daprStub).getBulkSecret(any(DaprProtos.GetBulkSecretRequest.class), any()); + + Map> secrets = client.getBulkSecret(SECRET_STORE_NAME).block(); + + assertEquals(2, secrets.size()); + assertEquals(1, secrets.get("one").size()); + assertEquals("mysecretvalue", secrets.get("one").get("mysecretkey")); + assertEquals(2, secrets.get("two").size()); + assertEquals("1", secrets.get("two").get("a")); + assertEquals("2", secrets.get("two").get("b")); + } + + @Test + public void getBulkSecretsWithMetadata() { + DaprProtos.GetBulkSecretResponse responseEnvelope = buildGetBulkSecretResponse( + new HashMap>() {{ + put("one", Collections.singletonMap("mysecretkey", "mysecretvalue")); + put("two", new HashMap() {{ + put("a", "1"); + put("b", "2"); + }}); + }}); + + doAnswer((Answer) invocation -> { + DaprProtos.GetBulkSecretRequest req = invocation.getArgument(0); + assertEquals(SECRET_STORE_NAME, req.getStoreName()); + assertEquals(1, req.getMetadataCount()); + assertEquals("metavalue", req.getMetadataOrThrow("metakey")); + + StreamObserver observer = + (StreamObserver) invocation.getArguments()[1]; + observer.onNext(responseEnvelope); + observer.onCompleted(); + return null; + }).when(daprStub).getBulkSecret(any(DaprProtos.GetBulkSecretRequest.class), any()); + + Map> secrets = client.getBulkSecret( + SECRET_STORE_NAME, Collections.singletonMap("metakey", "metavalue")).block(); + + assertEquals(2, secrets.size()); + assertEquals(1, secrets.get("one").size()); + assertEquals("mysecretvalue", secrets.get("one").get("mysecretkey")); + assertEquals(2, secrets.get("two").size()); + assertEquals("1", secrets.get("two").get("a")); + assertEquals("2", secrets.get("two").get("b")); + } + /* If this test is failing, it means that a new value was added to StateOptions.Consistency * enum, without creating a mapping to one of the proto defined gRPC enums */ @@ -1992,6 +2061,16 @@ private DaprProtos.GetSecretResponse buildGetSecretResponse() { return DaprProtos.GetSecretResponse.newBuilder().build(); } + private DaprProtos.GetBulkSecretResponse buildGetBulkSecretResponse(Map> res) { + Map map = res.entrySet().stream().collect( + Collectors.toMap( + e -> e.getKey(), + e -> DaprProtos.SecretResponse.newBuilder().putAllSecrets(e.getValue()).build())); + return DaprProtos.GetBulkSecretResponse.newBuilder() + .putAllData(map) + .build(); + } + private StateOptions buildStateOptions(StateOptions.Consistency consistency, StateOptions.Concurrency concurrency) { StateOptions options = null; if (consistency != null || concurrency != null) { diff --git a/sdk/src/test/java/io/dapr/client/DaprClientHttpTest.java b/sdk/src/test/java/io/dapr/client/DaprClientHttpTest.java index 824b5fd29f..146846b457 100644 --- a/sdk/src/test/java/io/dapr/client/DaprClientHttpTest.java +++ b/sdk/src/test/java/io/dapr/client/DaprClientHttpTest.java @@ -1098,6 +1098,39 @@ public void getSecretsWithMetadata() { } } + @Test + public void getBulkSecrets() { + mockInterceptor.addRule() + .get("http://127.0.0.1:3000/v1.0/secrets/MySecretStore/bulk") + .respond("{ \"one\": { \"mysecretkey\": \"mysecretvalue\"}, \"two\": { \"a\": \"1\", \"b\": \"2\"}}"); + + Map> secrets = daprClientHttp.getBulkSecret(SECRET_STORE_NAME).block(); + + assertEquals(2, secrets.size()); + assertEquals(1, secrets.get("one").size()); + assertEquals("mysecretvalue", secrets.get("one").get("mysecretkey")); + assertEquals(2, secrets.get("two").size()); + assertEquals("1", secrets.get("two").get("a")); + assertEquals("2", secrets.get("two").get("b")); + } + + @Test + public void getBulkSecretsWithMetadata() { + mockInterceptor.addRule() + .get("http://127.0.0.1:3000/v1.0/secrets/MySecretStore/bulk?metadata.metakey=metavalue") + .respond("{ \"one\": { \"mysecretkey\": \"mysecretvalue\"}, \"two\": { \"a\": \"1\", \"b\": \"2\"}}"); + + Map> secrets = + daprClientHttp.getBulkSecret(SECRET_STORE_NAME, Collections.singletonMap("metakey", "metavalue")).block(); + + assertEquals(2, secrets.size()); + assertEquals(1, secrets.get("one").size()); + assertEquals("mysecretvalue", secrets.get("one").get("mysecretkey")); + assertEquals(2, secrets.get("two").size()); + assertEquals("1", secrets.get("two").get("a")); + assertEquals("2", secrets.get("two").get("b")); + } + @Test public void closeException() { DaprHttp daprHttp = Mockito.mock(DaprHttp.class);