feat(bigquery): expose dataset MaxTimeTravelHours (#7706)

Author: shollyman

bigquery/dataset.go

@@ -58,6 +58,10 @@ type DatasetMetadata struct {
 	// More information: https://cloud.google.com/bigquery/docs/reference/standard-sql/collation-concepts
 	DefaultCollation string
 
+	// MaxTimeTravelHours represents the number of hours for the max time travel for all tables
+	// in the dataset.  Durations are rounded towards zero for the nearest hourly value.
+	MaxTimeTravelHours time.Duration
+
 	// Storage billing model to be used for all tables in the dataset.
 	// Can be set to PHYSICAL. Default is LOGICAL.
 	// Once you create a dataset with storage billing model set to physical bytes, you can't change it back to using logical bytes again.
@@ -131,6 +135,10 @@ type DatasetMetadataToUpdate struct {
 	// created in the dataset.
 	DefaultCollation optional.String
 
+	// MaxTimeTravelHours represents the number of hours for the max time travel for all tables
+	// in the dataset.  Durations are rounded towards zero for the nearest hourly value.
+	MaxTimeTravelHours optional.Duration
+
 	// Storage billing model to be used for all tables in the dataset.
 	// Can be set to PHYSICAL. Default is LOGICAL.
 	// Once you change a dataset's storage billing model to use physical bytes, you can't change it back to using logical bytes again.
@@ -208,6 +216,7 @@ func (dm *DatasetMetadata) toBQ() (*bq.Dataset, error) {
 	ds.DefaultTableExpirationMs = int64(dm.DefaultTableExpiration / time.Millisecond)
 	ds.DefaultPartitionExpirationMs = int64(dm.DefaultPartitionExpiration / time.Millisecond)
 	ds.DefaultCollation = dm.DefaultCollation
+	ds.MaxTimeTravelHours = int64(dm.MaxTimeTravelHours / time.Hour)
 	ds.StorageBillingModel = string(dm.StorageBillingModel)
 	ds.Labels = dm.Labels
 	var err error
@@ -295,6 +304,7 @@ func bqToDatasetMetadata(d *bq.Dataset, c *Client) (*DatasetMetadata, error) {
 		DefaultTableExpiration:     time.Duration(d.DefaultTableExpirationMs) * time.Millisecond,
 		DefaultPartitionExpiration: time.Duration(d.DefaultPartitionExpirationMs) * time.Millisecond,
 		DefaultCollation:           d.DefaultCollation,
+		MaxTimeTravelHours:         time.Duration(d.MaxTimeTravelHours) * time.Hour,
 		StorageBillingModel:        d.StorageBillingModel,
 		DefaultEncryptionConfig:    bqToEncryptionConfig(d.DefaultEncryptionConfiguration),
 		Description:                d.Description,
@@ -385,6 +395,15 @@ func (dm *DatasetMetadataToUpdate) toBQ() (*bq.Dataset, error) {
 		ds.DefaultCollation = optional.ToString(dm.DefaultCollation)
 		forceSend("DefaultCollation")
 	}
+	if dm.MaxTimeTravelHours != nil {
+		dur := optional.ToDuration(dm.MaxTimeTravelHours)
+		if dur == 0 {
+			// Send a null to delete the field.
+			ds.NullFields = append(ds.NullFields, "MaxTimeTravelHours")
+		} else {
+			ds.MaxTimeTravelHours = int64(dur / time.Hour)
+		}
+	}
 	if dm.StorageBillingModel != nil {
 		ds.StorageBillingModel = optional.ToString(dm.StorageBillingModel)
 		forceSend("StorageBillingModel")

bigquery/dataset_integration_test.go

@@ -159,7 +159,7 @@ func TestIntegration_DatasetUpdateETags(t *testing.T) {
 	check(md3, "", "")
 }
 
-func TestIntegration_DatasetUpdateDefaultExpiration(t *testing.T) {
+func TestIntegration_DatasetUpdateDefaultExpirationAndMaxTimeTravel(t *testing.T) {
 	if client == nil {
 		t.Skip("Integration tests skipped")
 	}
@@ -168,13 +168,21 @@ func TestIntegration_DatasetUpdateDefaultExpiration(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	wantExpiration := time.Hour
+	wantTimeTravel := 48 * time.Hour
 	// Set the default expiration time.
-	md, err := dataset.Update(ctx, DatasetMetadataToUpdate{DefaultTableExpiration: time.Hour}, "")
+	md, err := dataset.Update(ctx, DatasetMetadataToUpdate{
+		DefaultTableExpiration: wantExpiration,
+		MaxTimeTravelHours:     wantTimeTravel,
+	}, "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	if md.DefaultTableExpiration != time.Hour {
-		t.Fatalf("got %s, want 1h", md.DefaultTableExpiration)
+	if got := md.DefaultTableExpiration; got != wantExpiration {
+		t.Fatalf("DefaultTableExpiration want %s got %s", wantExpiration, md.DefaultTableExpiration)
+	}
+	if got := md.MaxTimeTravelHours; got != wantTimeTravel {
+		t.Fatalf("MaxTimeTravelHours want %s got %s", wantTimeTravel, md.MaxTimeTravelHours)
 	}
 	// Omitting DefaultTableExpiration doesn't change it.
 	md, err = dataset.Update(ctx, DatasetMetadataToUpdate{Name: "xyz"}, "")

bigquery/dataset_test.go

@@ -322,6 +322,7 @@ func TestDatasetToBQ(t *testing.T) {
 			Name:                       "name",
 			Description:                "desc",
 			DefaultTableExpiration:     time.Hour,
+			MaxTimeTravelHours:         time.Duration(181 * time.Minute),
 			DefaultPartitionExpiration: 24 * time.Hour,
 			DefaultEncryptionConfig: &EncryptionConfig{
 				KMSKeyName: "some_key",
@@ -342,6 +343,7 @@ func TestDatasetToBQ(t *testing.T) {
 			FriendlyName:                 "name",
 			Description:                  "desc",
 			DefaultTableExpirationMs:     60 * 60 * 1000,
+			MaxTimeTravelHours:           3,
 			DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
 			DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
 				KmsKeyName: "some_key",
@@ -397,6 +399,7 @@ func TestBQToDatasetMetadata(t *testing.T) {
 		FriendlyName:                 "name",
 		Description:                  "desc",
 		DefaultTableExpirationMs:     60 * 60 * 1000,
+		MaxTimeTravelHours:           3,
 		DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
 		DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{
 			KmsKeyName: "some_key",
@@ -428,6 +431,7 @@ func TestBQToDatasetMetadata(t *testing.T) {
 		Name:                       "name",
 		Description:                "desc",
 		DefaultTableExpiration:     time.Hour,
+		MaxTimeTravelHours:         time.Duration(3 * time.Hour),
 		DefaultPartitionExpiration: 24 * time.Hour,
 		DefaultEncryptionConfig: &EncryptionConfig{
 			KMSKeyName: "some_key",
@@ -467,6 +471,7 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
 		Name:                       "name",
 		DefaultTableExpiration:     time.Hour,
 		DefaultPartitionExpiration: 24 * time.Hour,
+		MaxTimeTravelHours:         time.Duration(181 * time.Minute),
 		StorageBillingModel:        PhysicalStorageBillingModel,
 		DefaultEncryptionConfig: &EncryptionConfig{
 			KMSKeyName: "some_key",
@@ -483,6 +488,7 @@ func TestDatasetMetadataToUpdateToBQ(t *testing.T) {
 		Description:                  "desc",
 		FriendlyName:                 "name",
 		DefaultTableExpirationMs:     60 * 60 * 1000,
+		MaxTimeTravelHours:           3,
 		DefaultPartitionExpirationMs: 24 * 60 * 60 * 1000,
 		StorageBillingModel:          string(PhysicalStorageBillingModel),
 		DefaultEncryptionConfiguration: &bq.EncryptionConfiguration{