From 94e1e854a685a2772fc4cec990b6f4551f3cdbcc Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Fri, 7 Oct 2022 18:18:54 +0000 Subject: [PATCH 1/3] feat(storage): support `Autoclass` feature This implements support for REST and gRPC, as well as the unit tests and the samples. --- google/cloud/storage/bucket_autoclass.cc | 35 +++++ google/cloud/storage/bucket_autoclass.h | 64 ++++++++ google/cloud/storage/bucket_metadata.cc | 16 ++ google/cloud/storage/bucket_metadata.h | 22 +++ google/cloud/storage/bucket_metadata_test.cc | 74 ++++++++++ google/cloud/storage/examples/CMakeLists.txt | 1 + .../storage_bucket_autoclass_samples.cc | 139 ++++++++++++++++++ .../storage/examples/storage_examples.bzl | 1 + .../storage/google_cloud_cpp_storage.bzl | 2 + .../storage/google_cloud_cpp_storage.cmake | 2 + .../internal/bucket_metadata_parser.cc | 20 +++ .../internal/grpc_bucket_metadata_parser.cc | 25 ++++ .../internal/grpc_bucket_metadata_parser.h | 5 + .../grpc_bucket_metadata_parser_test.cc | 27 +++- .../internal/grpc_bucket_request_parser.cc | 17 +++ .../grpc_bucket_request_parser_test.cc | 22 ++- google/cloud/testing_util/example_driver.cc | 5 + 17 files changed, 470 insertions(+), 7 deletions(-) create mode 100644 google/cloud/storage/bucket_autoclass.cc create mode 100644 google/cloud/storage/bucket_autoclass.h create mode 100644 google/cloud/storage/examples/storage_bucket_autoclass_samples.cc diff --git a/google/cloud/storage/bucket_autoclass.cc b/google/cloud/storage/bucket_autoclass.cc new file mode 100644 index 0000000000000..189d1db67df93 --- /dev/null +++ b/google/cloud/storage/bucket_autoclass.cc @@ -0,0 +1,35 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/storage/bucket_autoclass.h" +#include "google/cloud/internal/format_time_point.h" +#include "google/cloud/internal/ios_flags_saver.h" +#include +#include + +namespace google { +namespace cloud { +namespace storage { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN + +std::ostream& operator<<(std::ostream& os, BucketAutoclass const& rhs) { + google::cloud::internal::IosFlagsSaver flags(os); + return os << "{enabled=" << std::boolalpha << rhs.enabled << ", toggle_time=" + << google::cloud::internal::FormatRfc3339(rhs.toggle_time) << "}"; +} + +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace storage +} // namespace cloud +} // namespace google diff --git a/google/cloud/storage/bucket_autoclass.h b/google/cloud/storage/bucket_autoclass.h new file mode 100644 index 0000000000000..1671e45e0eb71 --- /dev/null +++ b/google/cloud/storage/bucket_autoclass.h @@ -0,0 +1,64 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_AUTOCLASS_H +#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_AUTOCLASS_H + +#include "google/cloud/storage/version.h" +#include +#include +#include + +namespace google { +namespace cloud { +namespace storage { +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN + +/** + * The autoclass configuration for a Bucket. + * + * @par Example + * + * @snippet storage_bucket_autoclass_samples.cc get-autoclass + * + * @par Example + * + * @snippet storage_bucket_autoclass_samples.cc set-autoclass + */ +struct BucketAutoclass { + explicit BucketAutoclass(bool e) : enabled(e) {} + explicit BucketAutoclass(bool e, std::chrono::system_clock::time_point tp) + : enabled(e), toggle_time(tp) {} + + bool enabled; + std::chrono::system_clock::time_point toggle_time; +}; + +inline bool operator==(BucketAutoclass const& lhs, BucketAutoclass const& rhs) { + return std::tie(lhs.enabled, lhs.toggle_time) == + std::tie(rhs.enabled, rhs.toggle_time); +} + +inline bool operator!=(BucketAutoclass const& lhs, BucketAutoclass const& rhs) { + return !(lhs == rhs); +} + +std::ostream& operator<<(std::ostream& os, BucketAutoclass const& rhs); + +GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END +} // namespace storage +} // namespace cloud +} // namespace google + +#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_AUTOCLASS_H diff --git a/google/cloud/storage/bucket_metadata.cc b/google/cloud/storage/bucket_metadata.cc index a2b0466dfa756..2e2396d89fd90 100644 --- a/google/cloud/storage/bucket_metadata.cc +++ b/google/cloud/storage/bucket_metadata.cc @@ -83,6 +83,7 @@ nlohmann::json ActionAsPatch(LifecycleRuleAction const& a) { bool operator==(BucketMetadata const& lhs, BucketMetadata const& rhs) { return lhs.acl_ == rhs.acl_ // + && lhs.autoclass_ == rhs.autoclass_ // && lhs.billing_ == rhs.billing_ // && lhs.cors_ == rhs.cors_ // && lhs.custom_placement_config_ == rhs.custom_placement_config_ // @@ -121,6 +122,9 @@ std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs) { os << absl::StrJoin(rhs.acl(), ", ", absl::StreamFormatter()); os << "]"; + if (rhs.has_autoclass()) { + os << ", autoclass=" << rhs.autoclass(); + } if (rhs.has_billing()) { auto previous_flags = os.flags(); os << ", billing.requesterPays=" << std::boolalpha @@ -249,6 +253,18 @@ BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::ResetAcl() { return *this; } +BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::SetAutoclass( + BucketAutoclass const& v) { + impl_.AddSubPatch( + "autoclass", internal::PatchBuilder().SetBoolField("enabled", v.enabled)); + return *this; +} + +BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::ResetAutoclass() { + impl_.RemoveField("autoclass"); + return *this; +} + BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::SetBilling( BucketBilling const& v) { impl_.AddSubPatch("billing", internal::PatchBuilder().SetBoolField( diff --git a/google/cloud/storage/bucket_metadata.h b/google/cloud/storage/bucket_metadata.h index bbffbc4bda455..1862c1bbbd458 100644 --- a/google/cloud/storage/bucket_metadata.h +++ b/google/cloud/storage/bucket_metadata.h @@ -16,6 +16,7 @@ #define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_METADATA_H #include "google/cloud/storage/bucket_access_control.h" +#include "google/cloud/storage/bucket_autoclass.h" #include "google/cloud/storage/bucket_billing.h" #include "google/cloud/storage/bucket_cors_entry.h" #include "google/cloud/storage/bucket_custom_placement_config.h" @@ -69,6 +70,23 @@ class BucketMetadata { } ///@} + /// @name Accessors and modifiers for Autoclass configuration. + ///@{ + bool has_autoclass() const { return autoclass_.has_value(); } + BucketAutoclass const& autoclass() const { return *autoclass_; } + absl::optional const& autoclass_as_optional() const { + return autoclass_; + } + BucketMetadata& set_autoclass(BucketAutoclass v) { + autoclass_ = std::move(v); + return *this; + } + BucketMetadata& reset_autoclass() { + autoclass_.reset(); + return *this; + } + ///@} + /** * @name Get and set billing configuration for the Bucket. * @@ -549,6 +567,7 @@ class BucketMetadata { friend std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs); // Keep the fields in alphabetical order. std::vector acl_; + absl::optional autoclass_; absl::optional billing_; std::vector cors_; absl::optional custom_placement_config_; @@ -607,6 +626,9 @@ class BucketMetadataPatchBuilder { */ BucketMetadataPatchBuilder& ResetAcl(); + BucketMetadataPatchBuilder& SetAutoclass(BucketAutoclass const& v); + BucketMetadataPatchBuilder& ResetAutoclass(); + BucketMetadataPatchBuilder& SetBilling(BucketBilling const& v); BucketMetadataPatchBuilder& ResetBilling(); diff --git a/google/cloud/storage/bucket_metadata_test.cc b/google/cloud/storage/bucket_metadata_test.cc index 6eeb6441dc502..f89606fc4b56e 100644 --- a/google/cloud/storage/bucket_metadata_test.cc +++ b/google/cloud/storage/bucket_metadata_test.cc @@ -68,6 +68,10 @@ BucketMetadata CreateBucketMetadataForTest() { "etag": "AYX=" } ], + "autoclass": { + "enabled": true, + "toggleTime": "2022-10-07T01:02:03Z" + }, "billing": { "requesterPays": true }, @@ -178,6 +182,14 @@ TEST(BucketMetadataTest, Parse) { EXPECT_EQ(2, actual.acl().size()); EXPECT_EQ("acl-id-0", actual.acl().at(0).id()); EXPECT_EQ("acl-id-1", actual.acl().at(1).id()); + + auto const expected_autoclass_toggle = + google::cloud::internal::ParseRfc3339("2022-10-07T01:02:03Z"); + ASSERT_STATUS_OK(expected_autoclass_toggle); + ASSERT_TRUE(actual.has_autoclass()); + EXPECT_EQ(actual.autoclass(), + BucketAutoclass(true, *expected_autoclass_toggle)); + EXPECT_TRUE(actual.billing().requester_pays); EXPECT_EQ(2, actual.cors().size()); auto expected_cors_0 = @@ -302,6 +314,12 @@ TEST(BucketMetadataTest, IOStream) { // acl() EXPECT_THAT(actual, HasSubstr("acl-id-0")); EXPECT_THAT(actual, HasSubstr("acl-id-1")); + + // autoclass() + EXPECT_THAT( + actual, + HasSubstr("autoclass={enabled=true, toggle_time=2022-10-07T01:02:03Z}")); + // billing() EXPECT_THAT(actual, HasSubstr("enabled=true")); @@ -383,6 +401,15 @@ TEST(BucketMetadataTest, ToJsonString) { EXPECT_EQ("user-test-user", actual["acl"][0].value("entity", "")); EXPECT_EQ("user-test-user2", actual["acl"][1].value("entity", "")); + // autoclass() + ASSERT_TRUE(actual.contains("autoclass")); + auto const expected_autoclass = nlohmann::json{ + {"enabled", true}, + // "toggleTime" is OUTPUT_ONLY and thus not included in the + // JSON string for create/update. + }; + EXPECT_EQ(actual["autoclass"], expected_autoclass); + // billing() ASSERT_EQ(1U, actual.count("billing")) << actual; EXPECT_TRUE(actual["billing"].value("requesterPays", false)); @@ -573,6 +600,32 @@ TEST(BucketMetadataTest, SetAcl) { EXPECT_EQ("READER", copy.acl().at(0).role()); } +/// @test Verify we can change the autoclass configuration in BucketMetadata. +TEST(BucketMetadataTest, SetAutoclass) { + auto expected = CreateBucketMetadataForTest(); + auto copy = expected; + ASSERT_TRUE(copy.has_autoclass()); + ASSERT_TRUE(copy.autoclass().enabled); + copy.set_autoclass( + BucketAutoclass{false, std::chrono::system_clock::time_point()}); + EXPECT_NE(expected, copy); + ASSERT_TRUE(copy.has_autoclass()); + ASSERT_FALSE(copy.autoclass().enabled); +} + +/// @test Verify we can reset the autoclass configuration in BucketMetadata. +TEST(BucketMetadataTest, ResetAutoclass) { + auto expected = CreateBucketMetadataForTest(); + EXPECT_TRUE(expected.has_autoclass()); + auto copy = expected; + copy.reset_autoclass(); + EXPECT_FALSE(copy.has_autoclass()); + EXPECT_NE(expected, copy); + std::ostringstream os; + os << copy; + EXPECT_THAT(os.str(), Not(HasSubstr("autoclass"))); +} + /// @test Verify we can change the billing configuration in BucketMetadata. TEST(BucketMetadataTest, SetBilling) { auto expected = CreateBucketMetadataForTest(); @@ -945,6 +998,27 @@ TEST(BucketMetadataPatchBuilder, ResetAcl) { ASSERT_TRUE(json["acl"].is_null()) << json; } +TEST(BucketMetadataPatchBuilder, SetAutoclass) { + BucketMetadataPatchBuilder builder; + builder.SetAutoclass(BucketAutoclass(true)); + + auto actual = builder.BuildPatch(); + auto const json = nlohmann::json::parse(actual); + ASSERT_TRUE(json.contains("autoclass")) << json; + auto const expected_autoclass = nlohmann::json{{"enabled", true}}; + EXPECT_EQ(expected_autoclass, json["autoclass"]); +} + +TEST(BucketMetadataPatchBuilder, ResetAutoclass) { + BucketMetadataPatchBuilder builder; + builder.ResetAutoclass(); + + auto actual = builder.BuildPatch(); + auto json = nlohmann::json::parse(actual); + ASSERT_TRUE(json.contains("autoclass")) << json; + ASSERT_TRUE(json["autoclass"].is_null()) << json; +} + TEST(BucketMetadataPatchBuilder, SetBilling) { BucketMetadataPatchBuilder builder; builder.SetBilling(BucketBilling{true}); diff --git a/google/cloud/storage/examples/CMakeLists.txt b/google/cloud/storage/examples/CMakeLists.txt index 775cb6fdfabb6..ed3af7ac0cde7 100644 --- a/google/cloud/storage/examples/CMakeLists.txt +++ b/google/cloud/storage/examples/CMakeLists.txt @@ -23,6 +23,7 @@ if (BUILD_TESTING) set(storage_examples # cmake-format: sort + storage_bucket_autoclass_samples.cc storage_bucket_acl_samples.cc storage_bucket_cors_samples.cc storage_bucket_default_kms_key_samples.cc diff --git a/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc b/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc new file mode 100644 index 0000000000000..f7750e4e2cd3d --- /dev/null +++ b/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc @@ -0,0 +1,139 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "google/cloud/storage/client.h" +#include "google/cloud/storage/examples/storage_examples_common.h" +#include "google/cloud/internal/getenv.h" +#include +#include +#include + +namespace { + +void GetAutoclass(google::cloud::storage::Client client, + std::vector const& argv) { + //! [get-autoclass] [START storage_get_autoclass] + namespace gcs = ::google::cloud::storage; + [](gcs::Client client, std::string const& bucket_name) { + auto metadata = client.GetBucketMetadata(bucket_name); + if (!metadata) throw google::cloud::Status(std::move(metadata).status()); + + if (!metadata->has_autoclass()) { + std::cout << "The bucket " << metadata->name() << " does not have an" + << " autoclass configuration.\n"; + return; + } + + std::cout << "Autoclass is " + << (metadata->autoclass().enabled ? "enabled" : "disabled") + << " for bucket " << metadata->name() << ". " + << " The bucket's full autoclass configuration is " + << metadata->autoclass() << "\n"; + } + //! [get-autoclass] [END storage_get_autoclass] + (std::move(client), argv.at(0)); +} + +void SetAutoclass(google::cloud::storage::Client client, + std::vector const& argv) { + using ::google::cloud::storage::examples::Usage; + if (argv.at(1) != "true" && argv.at(1) != "false") { + throw Usage{"enabled must be either 'true' or 'false'"}; + } + auto const enabled = argv.at(1) == "true"; + //! [set-autoclass] [START storage_set_autoclass] + namespace gcs = ::google::cloud::storage; + [](gcs::Client client, std::string const& bucket_name, bool enabled) { + auto metadata = client.PatchBucket( + bucket_name, gcs::BucketMetadataPatchBuilder().SetAutoclass( + gcs::BucketAutoclass{enabled})); + if (!metadata) throw google::cloud::Status(std::move(metadata).status()); + + std::cout << "The autoclass configuration for bucket " << bucket_name + << " was successfully updated."; + if (!metadata->has_autoclass()) { + std::cout << " The bucket no longer has an autoclass configuration.\n"; + return; + } + std::cout << " The new configuration is " << metadata->autoclass() << "\n"; + } + //! [set-autoclass] [END storage_set_autoclass] + (std::move(client), argv.at(0), enabled); +} + +void RunAll(std::vector const& argv) { + namespace examples = ::google::cloud::storage::examples; + namespace gcs = ::google::cloud::storage; + + if (!argv.empty()) throw examples::Usage{"auto"}; + examples::CheckEnvironmentVariablesAreSet({ + "GOOGLE_CLOUD_PROJECT", + }); + auto const project_id = + google::cloud::internal::GetEnv("GOOGLE_CLOUD_PROJECT").value(); + auto generator = google::cloud::internal::DefaultPRNG(std::random_device{}()); + auto const bucket_name_enabled = examples::MakeRandomBucketName(generator); + auto const bucket_name_disabled = examples::MakeRandomBucketName(generator); + auto const object_name = + examples::MakeRandomObjectName(generator, "object-") + ".txt"; + auto client = gcs::Client(); + + std::cout << "\nCreating buckets to run the example:" + << "\nEnabled Autoclass: " << bucket_name_enabled + << "\nDisabled Autoclass: " << bucket_name_disabled << std::endl; + // In GCS a single project cannot create or delete buckets more often than + // once every two seconds. We will pause until that time before deleting the + // bucket. + auto constexpr kBucketPeriod = std::chrono::seconds(2); + auto pause = std::chrono::steady_clock::now() + kBucketPeriod; + (void)client + .CreateBucketForProject( + bucket_name_enabled, project_id, + gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{true})) + .value(); + if (!examples::UsingEmulator()) std::this_thread::sleep_until(pause); + pause = std::chrono::steady_clock::now() + kBucketPeriod; + (void)client + .CreateBucketForProject( + bucket_name_disabled, project_id, + gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{false})) + .value(); + + std::cout << "\nRunning GetAutoclass() example [enabled]" << std::endl; + GetAutoclass(client, {bucket_name_enabled}); + + std::cout << "\nRunning GetAutoclass() example [disabled]" << std::endl; + GetAutoclass(client, {bucket_name_disabled}); + + std::cout << "\nRunning SetAutoclass() example" << std::endl; + SetAutoclass(client, {bucket_name_enabled, "false"}); + + if (!examples::UsingEmulator()) std::this_thread::sleep_until(pause); + (void)examples::RemoveBucketAndContents(client, bucket_name_enabled); + (void)examples::RemoveBucketAndContents(client, bucket_name_disabled); +} + +} // namespace + +int main(int argc, char* argv[]) { + namespace examples = ::google::cloud::storage::examples; + examples::Example example({ + examples::CreateCommandEntry("get-autoclass", {""}, + GetAutoclass), + examples::CreateCommandEntry( + "set-autoclass", {"", ""}, SetAutoclass), + {"auto", RunAll}, + }); + return example.Run(argc, argv); +} diff --git a/google/cloud/storage/examples/storage_examples.bzl b/google/cloud/storage/examples/storage_examples.bzl index 2031c11bb91c5..3a81c139e850b 100644 --- a/google/cloud/storage/examples/storage_examples.bzl +++ b/google/cloud/storage/examples/storage_examples.bzl @@ -17,6 +17,7 @@ """Automatically generated unit tests list - DO NOT EDIT.""" storage_examples = [ + "storage_bucket_autoclass_samples.cc", "storage_bucket_acl_samples.cc", "storage_bucket_cors_samples.cc", "storage_bucket_default_kms_key_samples.cc", diff --git a/google/cloud/storage/google_cloud_cpp_storage.bzl b/google/cloud/storage/google_cloud_cpp_storage.bzl index 9a33c01fca3b5..2b706e691c496 100644 --- a/google/cloud/storage/google_cloud_cpp_storage.bzl +++ b/google/cloud/storage/google_cloud_cpp_storage.bzl @@ -19,6 +19,7 @@ google_cloud_cpp_storage_hdrs = [ "auto_finalize.h", "bucket_access_control.h", + "bucket_autoclass.h", "bucket_billing.h", "bucket_cors_entry.h", "bucket_custom_placement_config.h", @@ -152,6 +153,7 @@ google_cloud_cpp_storage_hdrs = [ google_cloud_cpp_storage_srcs = [ "auto_finalize.cc", "bucket_access_control.cc", + "bucket_autoclass.cc", "bucket_cors_entry.cc", "bucket_custom_placement_config.cc", "bucket_iam_configuration.cc", diff --git a/google/cloud/storage/google_cloud_cpp_storage.cmake b/google/cloud/storage/google_cloud_cpp_storage.cmake index 993aeb4e682a5..e5157348972d1 100644 --- a/google/cloud/storage/google_cloud_cpp_storage.cmake +++ b/google/cloud/storage/google_cloud_cpp_storage.cmake @@ -25,6 +25,8 @@ add_library( auto_finalize.h bucket_access_control.cc bucket_access_control.h + bucket_autoclass.cc + bucket_autoclass.h bucket_billing.h bucket_cors_entry.cc bucket_cors_entry.h diff --git a/google/cloud/storage/internal/bucket_metadata_parser.cc b/google/cloud/storage/internal/bucket_metadata_parser.cc index 52e851a5f0940..6b6768ba681a1 100644 --- a/google/cloud/storage/internal/bucket_metadata_parser.cc +++ b/google/cloud/storage/internal/bucket_metadata_parser.cc @@ -81,6 +81,17 @@ Status ParseAcl(BucketMetadata& meta, nlohmann::json const& json) { return Status{}; } +Status ParseAutoclass(BucketMetadata& meta, nlohmann::json const& json) { + auto f = json.find("autoclass"); + if (f == json.end()) return Status{}; + auto enabled = internal::ParseBoolField(*f, "enabled"); + if (!enabled) return std::move(enabled).status(); + auto toggle = internal::ParseTimestampField(*f, "toggleTime"); + if (!toggle) return std::move(toggle).status(); + meta.set_autoclass(BucketAutoclass{*enabled, *toggle}); + return Status{}; +} + Status ParseBilling(BucketMetadata& meta, nlohmann::json const& json) { if (!json.contains("billing")) return Status{}; auto const& b = json["billing"]; @@ -287,6 +298,13 @@ void ToJsonCors(nlohmann::json& json, BucketMetadata const& meta) { json["cors"] = std::move(value); } +void ToJsonAutoclass(nlohmann::json& json, BucketMetadata const& meta) { + if (!meta.has_autoclass()) return; + json["autoclass"] = nlohmann::json{ + {"enabled", meta.autoclass().enabled}, + }; +} + void ToJsonBilling(nlohmann::json& json, BucketMetadata const& meta) { if (!meta.has_billing()) return; json["billing"] = nlohmann::json{ @@ -453,6 +471,7 @@ StatusOr BucketMetadataParser::FromJson( using Parser = std::function; Parser parsers[] = { ParseAcl, + ParseAutoclass, ParseBilling, ParseCorsList, ParseCustomPlacementConfig, @@ -550,6 +569,7 @@ std::string ToJsonString(absl::CivilDay date) { std::string BucketMetadataToJsonString(BucketMetadata const& meta) { nlohmann::json json; ToJsonAcl(json, meta); + ToJsonAutoclass(json, meta); ToJsonBilling(json, meta); ToJsonCors(json, meta); ToJsonDefaultEventBasedHold(json, meta); diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc b/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc index 64b160d566f4f..d0452cbdd6275 100644 --- a/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc +++ b/google/cloud/storage/internal/grpc_bucket_metadata_parser.cc @@ -102,6 +102,9 @@ google::storage::v2::Bucket ToProto(storage::BucketMetadata const& rhs) { *result.mutable_custom_placement_config() = ToProto(rhs.custom_placement_config()); } + if (rhs.has_autoclass()) { + *result.mutable_autoclass() = ToProto(rhs.autoclass()); + } return result; } @@ -181,10 +184,32 @@ storage::BucketMetadata FromProto(google::storage::v2::Bucket const& rhs) { metadata.set_custom_placement_config( FromProto(rhs.custom_placement_config())); } + if (rhs.has_autoclass()) { + metadata.set_autoclass(FromProto(rhs.autoclass())); + } return metadata; } +google::storage::v2::Bucket::Autoclass ToProto( + storage::BucketAutoclass const& rhs) { + google::storage::v2::Bucket::Autoclass result; + result.set_enabled(rhs.enabled); + *result.mutable_toggle_time() = + google::cloud::internal::ToProtoTimestamp(rhs.toggle_time); + return result; +} + +storage::BucketAutoclass FromProto( + google::storage::v2::Bucket::Autoclass const& rhs) { + storage::BucketAutoclass result{rhs.enabled()}; + if (rhs.has_toggle_time()) { + result.toggle_time = + google::cloud::internal::ToChronoTimePoint(rhs.toggle_time()); + } + return result; +} + google::storage::v2::Bucket::Billing ToProto( storage::BucketBilling const& rhs) { google::storage::v2::Bucket::Billing result; diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser.h b/google/cloud/storage/internal/grpc_bucket_metadata_parser.h index c01cbc301af86..f8ac92005258a 100644 --- a/google/cloud/storage/internal/grpc_bucket_metadata_parser.h +++ b/google/cloud/storage/internal/grpc_bucket_metadata_parser.h @@ -27,6 +27,11 @@ GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN google::storage::v2::Bucket ToProto(storage::BucketMetadata const& rhs); storage::BucketMetadata FromProto(google::storage::v2::Bucket const& rhs); +google::storage::v2::Bucket::Autoclass ToProto( + storage::BucketAutoclass const& rhs); +storage::BucketAutoclass FromProto( + google::storage::v2::Bucket::Autoclass const& rhs); + google::storage::v2::Bucket::Billing ToProto(storage::BucketBilling const& rhs); storage::BucketBilling FromProto( google::storage::v2::Bucket::Billing const& rhs); diff --git a/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc b/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc index 55c41d76fd631..5628b82dd9924 100644 --- a/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc +++ b/google/cloud/storage/internal/grpc_bucket_metadata_parser_test.cc @@ -119,6 +119,10 @@ TEST(GrpcBucketMetadataParser, BucketAllFieldsRoundtrip) { public_access_prevention: "inherited" } etag: "test-etag" + autoclass { + enabled: true + toggle_time { seconds: 1665108184 nanos: 123456000 } + } )pb"; EXPECT_TRUE(google::protobuf::TextFormat::ParseFromString(kText, &input)); @@ -224,7 +228,11 @@ TEST(GrpcBucketMetadataParser, BucketAllFieldsRoundtrip) { }, "publicAccessPrevention": "inherited" }, - "etag": "test-etag" + "etag": "test-etag", + "autoclass": { + "enabled": true, + "toggleTime": "2022-10-07T02:03:04.123456000Z" + } })"""); ASSERT_THAT(expected, IsOk()); @@ -239,6 +247,23 @@ TEST(GrpcBucketMetadataParser, BucketAllFieldsRoundtrip) { EXPECT_THAT(actual, IsProtoEqual(input)); } +TEST(GrpcBucketMetadataParser, BucketAutoclassRoundtrip) { + auto constexpr kText = R"pb( + enabled: true + toggle_time { seconds: 1665108184 nanos: 123456000 } + )pb"; + google::storage::v2::Bucket::Autoclass start; + EXPECT_TRUE(google::protobuf::TextFormat::ParseFromString(kText, &start)); + auto const expected_toggle = + google::cloud::internal::ParseRfc3339("2022-10-07T02:03:04.123456000Z"); + ASSERT_STATUS_OK(expected_toggle); + auto const expected = storage::BucketAutoclass{true, *expected_toggle}; + auto const middle = FromProto(start); + EXPECT_EQ(middle, expected); + auto const end = ToProto(middle); + EXPECT_THAT(end, IsProtoEqual(start)); +} + TEST(GrpcBucketMetadataParser, BucketBillingRoundtrip) { auto constexpr kText = R"pb( requester_pays: true diff --git a/google/cloud/storage/internal/grpc_bucket_request_parser.cc b/google/cloud/storage/internal/grpc_bucket_request_parser.cc index 231b952a9d80e..0de6810b2a5a5 100644 --- a/google/cloud/storage/internal/grpc_bucket_request_parser.cc +++ b/google/cloud/storage/internal/grpc_bucket_request_parser.cc @@ -199,6 +199,15 @@ Status PatchIamConfig(Bucket& b, nlohmann::json const& i) { return Status{}; } +Status PatchAutoclass(Bucket& bucket, nlohmann::json const& p) { + if (p.is_null()) { + bucket.clear_autoclass(); + } else { + bucket.mutable_autoclass()->set_enabled(p.value("enabled", false)); + } + return Status{}; +} + void UpdateAcl(Bucket& bucket, storage::BucketMetadata const& metadata) { for (auto const& a : metadata.acl()) { auto& acl = *bucket.add_acl(); @@ -301,6 +310,11 @@ void UpdateIamConfig(Bucket& bucket, storage::BucketMetadata const& metadata) { } } +void UpdateAutoclass(Bucket& bucket, storage::BucketMetadata const& metadata) { + if (!metadata.has_autoclass()) return; + bucket.mutable_autoclass()->set_enabled(metadata.autoclass().enabled); +} + } // namespace google::storage::v2::DeleteBucketRequest ToProto( @@ -523,6 +537,7 @@ StatusOr ToProto( {"billing", "", PatchBilling}, {"retentionPolicy", "retention_policy", PatchRetentionPolicy}, {"iamConfiguration", "iam_config", PatchIamConfig}, + {"autoclass", "", PatchAutoclass}, }; auto const& patch = @@ -596,6 +611,8 @@ google::storage::v2::UpdateBucketRequest ToProto( UpdateRetentionPolicy(bucket, metadata); result.mutable_update_mask()->add_paths("iam_config"); UpdateIamConfig(bucket, metadata); + result.mutable_update_mask()->add_paths("autoclass"); + UpdateAutoclass(bucket, metadata); if (request.HasOption()) { result.set_if_metageneration_match( diff --git a/google/cloud/storage/internal/grpc_bucket_request_parser_test.cc b/google/cloud/storage/internal/grpc_bucket_request_parser_test.cc index 07a1bfea8be36..244c0d6df82d9 100644 --- a/google/cloud/storage/internal/grpc_bucket_request_parser_test.cc +++ b/google/cloud/storage/internal/grpc_bucket_request_parser_test.cc @@ -85,6 +85,10 @@ TEST(GrpcBucketRequestParser, CreateBucketMetadataAllOptions) { storage_class: "NEARLINE" rpo: "ASYNC_TURBO" acl { entity: "allUsers" role: "READER" } + autoclass { + enabled: true + toggle_time {} + } billing { requester_pays: true } default_object_acl { entity: "user:test-user@example.com" @@ -131,6 +135,7 @@ TEST(GrpcBucketRequestParser, CreateBucketMetadataAllOptions) { .set_name("test-bucket") .set_storage_class("NEARLINE") .set_location("us-central1") + .set_autoclass(storage::BucketAutoclass{true}) .set_billing(storage::BucketBilling(true)) .set_rpo(storage::RpoAsyncTurbo()) .set_versioning(storage::BucketVersioning(true)) @@ -427,6 +432,7 @@ TEST(GrpcBucketRequestParser, PatchBucketRequestAllOptions) { log_object_prefix: "test-log-prefix" } encryption { default_kms_key: "test-only-kms-key" } + autoclass { enabled: true } billing { requester_pays: true } retention_policy { retention_period: 123000 } iam_config { @@ -501,6 +507,7 @@ TEST(GrpcBucketRequestParser, PatchBucketRequestAllOptions) { storage::BucketLogging{"test-log-bucket", "test-log-prefix"}) .SetEncryption(storage::BucketEncryption{ /*.default_kms_key=*/"test-only-kms-key"}) + .SetAutoclass(storage::BucketAutoclass{true}) .SetBilling(storage::BucketBilling{/*.requester_pays=*/true}) .SetRetentionPolicy(std::chrono::seconds(123000)) .SetIamConfiguration(storage::BucketIamConfiguration{ @@ -523,8 +530,8 @@ TEST(GrpcBucketRequestParser, PatchBucketRequestAllOptions) { UnorderedElementsAre( "storage_class", "rpo", "acl", "default_object_acl", "lifecycle", "cors", "default_event_based_hold", "labels", - "website", "versioning", "logging", "encryption", "billing", - "retention_policy", "iam_config")); + "website", "versioning", "logging", "encryption", "autoclass", + "billing", "retention_policy", "iam_config")); // Clear the paths, which we already compared, and compare the proto. actual->mutable_update_mask()->clear_paths(); @@ -544,6 +551,7 @@ TEST(GrpcBucketRequestParser, PatchBucketRequestAllResets) { "bucket-name", storage::BucketMetadataPatchBuilder{} .SetStorageClass("NEARLINE") .ResetAcl() + .ResetAutoclass() .ResetBilling() .ResetCors() .ResetDefaultEventBasedHold() @@ -567,8 +575,8 @@ TEST(GrpcBucketRequestParser, PatchBucketRequestAllResets) { UnorderedElementsAre( "storage_class", "rpo", "acl", "default_object_acl", "lifecycle", "cors", "default_event_based_hold", "labels", - "website", "versioning", "logging", "encryption", "billing", - "retention_policy", "iam_config")); + "website", "versioning", "logging", "encryption", "autoclass", + "billing", "retention_policy", "iam_config")); // Clear the paths, which we already compared, and compare the proto. actual->mutable_update_mask()->clear_paths(); @@ -625,6 +633,7 @@ TEST(GrpcBucketRequestParser, UpdateBucketRequestAllOptions) { log_object_prefix: "test-log-prefix" } encryption { default_kms_key: "test-only-kms-key" } + autoclass { enabled: true } billing { requester_pays: true } retention_policy { retention_period: 123000 } iam_config { @@ -697,6 +706,7 @@ TEST(GrpcBucketRequestParser, UpdateBucketRequestAllOptions) { storage::BucketLogging{"test-log-bucket", "test-log-prefix"}) .set_encryption(storage::BucketEncryption{ /*.default_kms_key=*/"test-only-kms-key"}) + .set_autoclass(storage::BucketAutoclass{true}) .set_billing(storage::BucketBilling{/*.requester_pays=*/true}) .set_retention_policy(std::chrono::seconds(123000)) .set_iam_configuration(storage::BucketIamConfiguration{ @@ -718,8 +728,8 @@ TEST(GrpcBucketRequestParser, UpdateBucketRequestAllOptions) { UnorderedElementsAre( "storage_class", "rpo", "acl", "default_object_acl", "lifecycle", "cors", "default_event_based_hold", "labels", - "website", "versioning", "logging", "encryption", "billing", - "retention_policy", "iam_config")); + "website", "versioning", "logging", "encryption", "autoclass", + "billing", "retention_policy", "iam_config")); // Clear the paths, which we already compared, and test the rest actual.mutable_update_mask()->clear_paths(); diff --git a/google/cloud/testing_util/example_driver.cc b/google/cloud/testing_util/example_driver.cc index f71df5cdebfb7..626a1e0b96660 100644 --- a/google/cloud/testing_util/example_driver.cc +++ b/google/cloud/testing_util/example_driver.cc @@ -15,6 +15,7 @@ #include "google/cloud/testing_util/example_driver.h" #include "google/cloud/internal/getenv.h" #include "google/cloud/log.h" +#include "google/cloud/status.h" #include #if GOOGLE_CLOUD_CPP_HAVE_EXCEPTIONS @@ -71,6 +72,10 @@ int Example::Run(int argc, char const* const argv[]) try { } catch (Usage const& u) { PrintUsage(argv[0], u.what()); return 1; +} catch (google::cloud::Status const& status) { + std::cerr << "google::cloud::Status thrown: " << status << "\n"; + google::cloud::LogSink::Instance().Flush(); + throw; } catch (std::exception const& ex) { std::cerr << "Standard exception raised: " << ex.what() << "\n"; google::cloud::LogSink::Instance().Flush(); From b210695679f886f25c9cf7d94405c86c88f53d01 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Fri, 7 Oct 2022 23:35:18 +0000 Subject: [PATCH 2/3] Fix formatting --- google/cloud/storage/examples/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/storage/examples/CMakeLists.txt b/google/cloud/storage/examples/CMakeLists.txt index ed3af7ac0cde7..be375e79ae4a8 100644 --- a/google/cloud/storage/examples/CMakeLists.txt +++ b/google/cloud/storage/examples/CMakeLists.txt @@ -23,8 +23,8 @@ if (BUILD_TESTING) set(storage_examples # cmake-format: sort - storage_bucket_autoclass_samples.cc storage_bucket_acl_samples.cc + storage_bucket_autoclass_samples.cc storage_bucket_cors_samples.cc storage_bucket_default_kms_key_samples.cc storage_bucket_iam_samples.cc From 9fd04104af0f7d109502682007c1f57fb52b9039 Mon Sep 17 00:00:00 2001 From: Carlos O'Ryan Date: Sat, 8 Oct 2022 18:40:15 +0000 Subject: [PATCH 3/3] Address review comments --- .../storage/examples/storage_bucket_autoclass_samples.cc | 6 +++--- google/cloud/storage/examples/storage_examples.bzl | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc b/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc index f7750e4e2cd3d..4fcd81729ce74 100644 --- a/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc +++ b/google/cloud/storage/examples/storage_bucket_autoclass_samples.cc @@ -96,19 +96,18 @@ void RunAll(std::vector const& argv) { // once every two seconds. We will pause until that time before deleting the // bucket. auto constexpr kBucketPeriod = std::chrono::seconds(2); - auto pause = std::chrono::steady_clock::now() + kBucketPeriod; (void)client .CreateBucketForProject( bucket_name_enabled, project_id, gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{true})) .value(); - if (!examples::UsingEmulator()) std::this_thread::sleep_until(pause); - pause = std::chrono::steady_clock::now() + kBucketPeriod; + if (!examples::UsingEmulator()) std::this_thread::sleep_for(kBucketPeriod); (void)client .CreateBucketForProject( bucket_name_disabled, project_id, gcs::BucketMetadata{}.set_autoclass(gcs::BucketAutoclass{false})) .value(); + auto const pause = std::chrono::steady_clock::now() + kBucketPeriod; std::cout << "\nRunning GetAutoclass() example [enabled]" << std::endl; GetAutoclass(client, {bucket_name_enabled}); @@ -121,6 +120,7 @@ void RunAll(std::vector const& argv) { if (!examples::UsingEmulator()) std::this_thread::sleep_until(pause); (void)examples::RemoveBucketAndContents(client, bucket_name_enabled); + if (!examples::UsingEmulator()) std::this_thread::sleep_for(kBucketPeriod); (void)examples::RemoveBucketAndContents(client, bucket_name_disabled); } diff --git a/google/cloud/storage/examples/storage_examples.bzl b/google/cloud/storage/examples/storage_examples.bzl index 3a81c139e850b..0105b7ecb2457 100644 --- a/google/cloud/storage/examples/storage_examples.bzl +++ b/google/cloud/storage/examples/storage_examples.bzl @@ -17,8 +17,8 @@ """Automatically generated unit tests list - DO NOT EDIT.""" storage_examples = [ - "storage_bucket_autoclass_samples.cc", "storage_bucket_acl_samples.cc", + "storage_bucket_autoclass_samples.cc", "storage_bucket_cors_samples.cc", "storage_bucket_default_kms_key_samples.cc", "storage_bucket_iam_samples.cc",