From ea5486827ef9748af64f5ad0fd8388142d5d88f1 Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Fri, 20 Nov 2020 17:31:41 -0500 Subject: [PATCH 1/7] support service account in dataflow flex template job --- ...resource_dataflow_flex_template_job.go.erb | 14 ++ ...rce_dataflow_flex_template_job_test.go.erb | 120 ++++++++++++++++-- 2 files changed, 126 insertions(+), 8 deletions(-) diff --git a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb index 4f5805acd827..93ddd00ac345 100644 --- a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb +++ b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb @@ -87,6 +87,12 @@ func resourceDataflowFlexTemplateJob() *schema.Resource { Type: schema.TypeString, Computed: true, }, + + "service_account_email": { + Type: schema.TypeString, + Optional: true, + Description: `The Service Account email used to create the job.`, + }, }, } } @@ -109,11 +115,16 @@ func resourceDataflowFlexTemplateJobCreate(d *schema.ResourceData, meta interfac return err } + env := dataflow.FlexTemplateRuntimeEnvironment{ + ServiceAccountEmail: d.Get("service_account_email").(string), + } + request := dataflow.LaunchFlexTemplateRequest{ LaunchParameter: &dataflow.LaunchFlexTemplateParameter{ ContainerSpecGcsPath: d.Get("container_spec_gcs_path").(string), JobName: d.Get("name").(string), Parameters: expandStringMap(d, "parameters"), + Environment: &env, }, } @@ -168,6 +179,9 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{ if err := d.Set("labels", job.Labels); err != nil { return fmt.Errorf("Error setting labels: %s", err) } + if err := d.Set("service_account_email", job.Environment.ServiceAccountEmail); err != nil { + return fmt.Errorf("Error setting service_account_email: %s", err) + } if _, ok := dataflowTerminalStatesMap[job.CurrentState]; ok { log.Printf("[DEBUG] Removing resource '%s' because it is in state %s.\n", job.Name, job.CurrentState) diff --git a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb index d1775c73f1ed..c1357e320f5c 100644 --- a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb +++ b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb @@ -10,9 +10,9 @@ import ( ) func TestAccDataflowFlexTemplateJob_basic(t *testing.T) { - // This resource uses custom retry logic that cannot be sped up without - // modifying the actual resource - skipIfVcr(t) + // This resource uses custom retry logic that cannot be sped up without + // modifying the actual resource + skipIfVcr(t) t.Parallel() randStr := randString(t, 10) @@ -25,7 +25,7 @@ func TestAccDataflowFlexTemplateJob_basic(t *testing.T) { CheckDestroy: testAccCheckDataflowJobDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccDataflowFlowFlexTemplateJob_basic(bucket, job), + Config: testAccDataflowFlexTemplateJob_basic(bucket, job), Check: resource.ComposeTestCheckFunc( testAccDataflowJobExists(t, "google_dataflow_flex_template_job.big_data"), ), @@ -34,8 +34,35 @@ func TestAccDataflowFlexTemplateJob_basic(t *testing.T) { }) } +func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { + // Dataflow responses include serialized java classes and bash commands + // This makes body comparison infeasible + skipIfVcr(t) + t.Parallel() + + randStr := randString(t, 10) + bucket := "tf-test-dataflow-gcs-" + randStr + job := "tf-test-dataflow-job-" + randStr + accountId := "tf-test-dataflow-sa" + randStr + + vcrTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + Providers: testAccProviders, + CheckDestroy: testAccCheckDataflowJobDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId), + Check: resource.ComposeTestCheckFunc( + testAccDataflowJobExists(t, "google_dataflow_flex_template_job.big_data"), + testAccDataflowJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId), + ), + }, + }, + }) +} + // note: this config creates a job that doesn't actually do anything -func testAccDataflowFlowFlexTemplateJob_basic(bucket, job string) string { +func testAccDataflowFlexTemplateJob_basic(bucket, job string) string { return fmt.Sprintf(` resource "google_storage_bucket" "temp" { name = "%s" @@ -43,9 +70,9 @@ resource "google_storage_bucket" "temp" { } resource "google_storage_bucket_object" "flex_template" { - name = "flex_template.json" - bucket = google_storage_bucket.temp.name - content = < From 65a824918e24649ced3a7971c69e3ff401492405 Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Tue, 24 Nov 2020 19:12:39 -0500 Subject: [PATCH 2/7] add doc and fix fetching compute instances --- ...resource_dataflow_flex_template_job.go.erb | 11 ++++ ...rce_dataflow_flex_template_job_test.go.erb | 65 +++++++++++++++++-- .../dataflow_flex_template_job.html.markdown | 14 ++-- 3 files changed, 82 insertions(+), 8 deletions(-) diff --git a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb index 93ddd00ac345..412d087c55ac 100644 --- a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb +++ b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb @@ -41,6 +41,14 @@ func resourceDataflowFlexTemplateJob() *schema.Resource { ForceNew: true, }, + "zone": { + Type: schema.TypeString, + Optional: true, + // ForceNew applies to both stream and batch jobs + ForceNew: true, + Description: `The zone in which the created job should run. If it is not provided, the provider zone is used.`, + }, + "region": { Type: schema.TypeString, Optional: true, @@ -115,8 +123,11 @@ func resourceDataflowFlexTemplateJobCreate(d *schema.ResourceData, meta interfac return err } + zone, _ := getZone(d, config) + env := dataflow.FlexTemplateRuntimeEnvironment{ ServiceAccountEmail: d.Get("service_account_email").(string), + Zone: zone, } request := dataflow.LaunchFlexTemplateRequest{ diff --git a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb index c1357e320f5c..9e736465f5b3 100644 --- a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb +++ b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb @@ -44,6 +44,7 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { bucket := "tf-test-dataflow-gcs-" + randStr job := "tf-test-dataflow-job-" + randStr accountId := "tf-test-dataflow-sa" + randStr + zone := "us-central1-b" vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -51,16 +52,71 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { CheckDestroy: testAccCheckDataflowJobDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId), + Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone), Check: resource.ComposeTestCheckFunc( testAccDataflowJobExists(t, "google_dataflow_flex_template_job.big_data"), - testAccDataflowJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId), + testAccDataflowFlexTemplateJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId, zone), ), }, }, }) } +func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expectedId, zone string) resource.TestCheckFunc { + return func(s *terraform.State) error { + instance, err := testAccDataflowFlexTemplateJobGetGeneratedInstance(t, s, res, zone) + if err != nil { + return fmt.Errorf("Error getting dataflow job instance: %s", err) + } + accounts := instance.ServiceAccounts + if len(accounts) != 1 { + return fmt.Errorf("Found multiple service accounts (%d) for dataflow job %q, expected 1", len(accounts), res) + } + actualId := strings.Split(accounts[0].Email, "@")[0] + if expectedId != actualId { + return fmt.Errorf("service account mismatch, expected account ID = %q, actual email = %q", expectedId, accounts[0].Email) + } + return nil + } +} + +func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terraform.State, res, zone string) (*compute.Instance, error) { + rs, ok := s.RootModule().Resources[res] + if !ok { + return nil, fmt.Errorf("resource %q not in state", res) + } + if rs.Primary.ID == "" { + return nil, fmt.Errorf("resource %q does not have an ID set", res) + } + filter := fmt.Sprintf("labels.goog-dataflow-job-id = %s", rs.Primary.ID) + + config := googleProviderConfig(t) + + var instance *compute.Instance + + err := resource.Retry(1*time.Minute, func() *resource.RetryError { + instances, rerr := config.NewComputeClient(config.userAgent).Instances. + List(config.Project, zone). + Filter(filter). + MaxResults(2). + Do() + if rerr != nil { + return resource.NonRetryableError(rerr) + } + if len(instances.Items) == 0 { + return resource.RetryableError(fmt.Errorf("no instance found for dataflow job %q", rs.Primary.ID)) + } + if len(instances.Items) > 1 { + return resource.NonRetryableError(fmt.Errorf("Wrong number of matching instances for dataflow job: %s, %d", rs.Primary.ID, len(instances.Items))) + } + return nil + }) + if err != nil { + return nil, err + } + return instance, nil +} + // note: this config creates a job that doesn't actually do anything func testAccDataflowFlexTemplateJob_basic(bucket, job string) string { return fmt.Sprintf(` @@ -118,7 +174,7 @@ resource "google_dataflow_flex_template_job" "big_data" { } // note: this config creates a job that doesn't actually do anything -func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId string) string { +func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone string) string { return fmt.Sprintf(` resource "google_storage_bucket" "temp" { name = "%s" @@ -186,12 +242,13 @@ resource "google_dataflow_flex_template_job" "big_data" { outputTable = "my-project:my-dataset.my-table" } service_account_email = google_service_account.dataflow-sa.email + zone = "%s" depends_on = [ google_storage_bucket_iam_member.dataflow-gcs, google_project_iam_member.dataflow-worker ] } -`, bucket, accountId, job) +`, bucket, accountId, job, zone) } <% end -%> diff --git a/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown b/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown index f209114c3ec9..44e5c614ff4d 100644 --- a/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown +++ b/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown @@ -29,14 +29,14 @@ resource "google_dataflow_flex_template_job" "big_data_job" { ## Note on "destroy" / "apply" There are many types of Dataflow jobs. Some Dataflow jobs run constantly, -getting new data from (e.g.) a GCS bucket, and outputting data continuously. +getting new data from (e.g.) a GCS bucket, and outputting data continuously. Some jobs process a set amount of data then terminate. All jobs can fail while running due to programming errors or other issues. In this way, Dataflow jobs are different from most other Terraform / Google resources. The Dataflow resource is considered 'existing' while it is in a nonterminal state. If it reaches a terminal state (e.g. 'FAILED', 'COMPLETE', -'CANCELLED'), it will be recreated on the next 'apply'. This is as expected for +'CANCELLED'), it will be recreated on the next 'apply'. This is as expected for jobs which run continuously, but may surprise users who use this resource for other kinds of Dataflow jobs. @@ -65,10 +65,10 @@ used in the template). * `labels` - (Optional) User labels to be specified for the job. Keys and values should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions) page. **Note**: This field is marked as deprecated in Terraform as the API does not currently -support adding labels. +support adding labels. **NOTE**: Google-provided Dataflow templates often provide default labels that begin with `goog-dataflow-provided`. Unless explicitly set in config, these -labels will be ignored to prevent diffs on re-apply. +labels will be ignored to prevent diffs on re-apply. * `on_delete` - (Optional) One of "drain" or "cancel". Specifies behavior of deletion during `terraform destroy`. See above note. @@ -76,6 +76,12 @@ deletion during `terraform destroy`. See above note. * `project` - (Optional) The project in which the resource belongs. If it is not provided, the provider project is used. +* `zone` - (Optional) The zone in which the created job should run. If it is not provided, the provider zone is used. + +* `region` - (Optional) The region in which the created job should run. + +* `service_account_email` - (Optional) The Service Account email used to create the job. + ## Attributes Reference In addition to the arguments listed above, the following computed attributes are exported: From 23b945dd3468577711d99018cb78306374e6f826 Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Tue, 24 Nov 2020 20:03:47 -0500 Subject: [PATCH 3/7] fix null instance --- .../tests/resource_dataflow_flex_template_job_test.go.erb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb index 9e736465f5b3..e2dbb3da0dd4 100644 --- a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb +++ b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb @@ -109,6 +109,10 @@ func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terrafo if len(instances.Items) > 1 { return resource.NonRetryableError(fmt.Errorf("Wrong number of matching instances for dataflow job: %s, %d", rs.Primary.ID, len(instances.Items))) } + instance = instances.Items[0] + if instance == nil { + return resource.NonRetryableError(fmt.Errorf("invalid instance")) + } return nil }) if err != nil { From 5ce54124fb0ef1661e7d83a95f92fbabc14ea0a6 Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Tue, 24 Nov 2020 22:13:19 -0500 Subject: [PATCH 4/7] fix --- .../resources/resource_dataflow_flex_template_job.go.erb | 3 --- 1 file changed, 3 deletions(-) diff --git a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb index 412d087c55ac..1b462543d01d 100644 --- a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb +++ b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb @@ -190,9 +190,6 @@ func resourceDataflowFlexTemplateJobRead(d *schema.ResourceData, meta interface{ if err := d.Set("labels", job.Labels); err != nil { return fmt.Errorf("Error setting labels: %s", err) } - if err := d.Set("service_account_email", job.Environment.ServiceAccountEmail); err != nil { - return fmt.Errorf("Error setting service_account_email: %s", err) - } if _, ok := dataflowTerminalStatesMap[job.CurrentState]; ok { log.Printf("[DEBUG] Removing resource '%s' because it is in state %s.\n", job.Name, job.CurrentState) From fde5e0e9c8a73ba9a2eaeacc5f6e6bc8fd35992b Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Wed, 25 Nov 2020 15:08:51 -0500 Subject: [PATCH 5/7] address comments --- .../resources/resource_dataflow_flex_template_job.go.erb | 3 +++ 1 file changed, 3 insertions(+) diff --git a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb index 1b462543d01d..92ae4c087755 100644 --- a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb +++ b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb @@ -44,6 +44,7 @@ func resourceDataflowFlexTemplateJob() *schema.Resource { "zone": { Type: schema.TypeString, Optional: true, + Computed: true, // ForceNew applies to both stream and batch jobs ForceNew: true, Description: `The zone in which the created job should run. If it is not provided, the provider zone is used.`, @@ -123,6 +124,8 @@ func resourceDataflowFlexTemplateJobCreate(d *schema.ResourceData, meta interfac return err } + // Zone is optional in the dataflow API, so it is ok to ignore the error if zone + // cannot be determinied in either resource config or provider. zone, _ := getZone(d, config) env := dataflow.FlexTemplateRuntimeEnvironment{ From 567cadbb9c5e126ea1bbe9af2ef10af3b3afb71f Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Tue, 1 Dec 2020 16:34:44 -0500 Subject: [PATCH 6/7] use params instead --- ...resource_dataflow_flex_template_job.go.erb | 25 ------------------- ...rce_dataflow_flex_template_job_test.go.erb | 20 +++++++-------- .../dataflow_flex_template_job.html.markdown | 7 ++---- 3 files changed, 11 insertions(+), 41 deletions(-) diff --git a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb index 92ae4c087755..4f5805acd827 100644 --- a/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb +++ b/third_party/terraform/resources/resource_dataflow_flex_template_job.go.erb @@ -41,15 +41,6 @@ func resourceDataflowFlexTemplateJob() *schema.Resource { ForceNew: true, }, - "zone": { - Type: schema.TypeString, - Optional: true, - Computed: true, - // ForceNew applies to both stream and batch jobs - ForceNew: true, - Description: `The zone in which the created job should run. If it is not provided, the provider zone is used.`, - }, - "region": { Type: schema.TypeString, Optional: true, @@ -96,12 +87,6 @@ func resourceDataflowFlexTemplateJob() *schema.Resource { Type: schema.TypeString, Computed: true, }, - - "service_account_email": { - Type: schema.TypeString, - Optional: true, - Description: `The Service Account email used to create the job.`, - }, }, } } @@ -124,21 +109,11 @@ func resourceDataflowFlexTemplateJobCreate(d *schema.ResourceData, meta interfac return err } - // Zone is optional in the dataflow API, so it is ok to ignore the error if zone - // cannot be determinied in either resource config or provider. - zone, _ := getZone(d, config) - - env := dataflow.FlexTemplateRuntimeEnvironment{ - ServiceAccountEmail: d.Get("service_account_email").(string), - Zone: zone, - } - request := dataflow.LaunchFlexTemplateRequest{ LaunchParameter: &dataflow.LaunchFlexTemplateParameter{ ContainerSpecGcsPath: d.Get("container_spec_gcs_path").(string), JobName: d.Get("name").(string), Parameters: expandStringMap(d, "parameters"), - Environment: &env, }, } diff --git a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb index e2dbb3da0dd4..2e706f3d1d9f 100644 --- a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb +++ b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb @@ -44,7 +44,6 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { bucket := "tf-test-dataflow-gcs-" + randStr job := "tf-test-dataflow-job-" + randStr accountId := "tf-test-dataflow-sa" + randStr - zone := "us-central1-b" vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -52,19 +51,19 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { CheckDestroy: testAccCheckDataflowJobDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone), + Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId), Check: resource.ComposeTestCheckFunc( testAccDataflowJobExists(t, "google_dataflow_flex_template_job.big_data"), - testAccDataflowFlexTemplateJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId, zone), + testAccDataflowFlexTemplateJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId), ), }, }, }) } -func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expectedId, zone string) resource.TestCheckFunc { +func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expectedId string) resource.TestCheckFunc { return func(s *terraform.State) error { - instance, err := testAccDataflowFlexTemplateJobGetGeneratedInstance(t, s, res, zone) + instance, err := testAccDataflowFlexTemplateJobGetGeneratedInstance(t, s, res) if err != nil { return fmt.Errorf("Error getting dataflow job instance: %s", err) } @@ -80,7 +79,7 @@ func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expected } } -func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terraform.State, res, zone string) (*compute.Instance, error) { +func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terraform.State, res string) (*compute.Instance, error) { rs, ok := s.RootModule().Resources[res] if !ok { return nil, fmt.Errorf("resource %q not in state", res) @@ -96,7 +95,7 @@ func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terrafo err := resource.Retry(1*time.Minute, func() *resource.RetryError { instances, rerr := config.NewComputeClient(config.userAgent).Instances. - List(config.Project, zone). + List(config.Project, config.Zone). Filter(filter). MaxResults(2). Do() @@ -178,7 +177,7 @@ resource "google_dataflow_flex_template_job" "big_data" { } // note: this config creates a job that doesn't actually do anything -func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone string) string { +func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId string) string { return fmt.Sprintf(` resource "google_storage_bucket" "temp" { name = "%s" @@ -244,15 +243,14 @@ resource "google_dataflow_flex_template_job" "big_data" { parameters = { inputSubscription = "my-subscription" outputTable = "my-project:my-dataset.my-table" + serviceAccount = google_service_account.dataflow-sa.email } - service_account_email = google_service_account.dataflow-sa.email - zone = "%s" depends_on = [ google_storage_bucket_iam_member.dataflow-gcs, google_project_iam_member.dataflow-worker ] } -`, bucket, accountId, job, zone) +`, bucket, accountId, job) } <% end -%> diff --git a/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown b/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown index 44e5c614ff4d..598dda663d38 100644 --- a/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown +++ b/third_party/terraform/website/docs/r/dataflow_flex_template_job.html.markdown @@ -60,7 +60,8 @@ Template. - - - * `parameters` - (Optional) Key/Value pairs to be passed to the Dataflow job (as -used in the template). +used in the template). Additional [pipeline options](https://cloud.google.com/dataflow/docs/guides/specifying-exec-params#setting-other-cloud-dataflow-pipeline-options) +such as `serviceAccount`, `workerMachineType`, etc can be specified here. * `labels` - (Optional) User labels to be specified for the job. Keys and values should follow the restrictions specified in the [labeling restrictions](https://cloud.google.com/compute/docs/labeling-resources#restrictions) @@ -76,12 +77,8 @@ deletion during `terraform destroy`. See above note. * `project` - (Optional) The project in which the resource belongs. If it is not provided, the provider project is used. -* `zone` - (Optional) The zone in which the created job should run. If it is not provided, the provider zone is used. - * `region` - (Optional) The region in which the created job should run. -* `service_account_email` - (Optional) The Service Account email used to create the job. - ## Attributes Reference In addition to the arguments listed above, the following computed attributes are exported: From 2e84ccd7ae2dccc6fcb96d2c2b4e1a1f00c69237 Mon Sep 17 00:00:00 2001 From: Xin Gao Date: Tue, 1 Dec 2020 17:02:39 -0500 Subject: [PATCH 7/7] specify zone --- ...urce_dataflow_flex_template_job_test.go.erb | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb index 2e706f3d1d9f..962ff13393bd 100644 --- a/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb +++ b/third_party/terraform/tests/resource_dataflow_flex_template_job_test.go.erb @@ -44,6 +44,7 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { bucket := "tf-test-dataflow-gcs-" + randStr job := "tf-test-dataflow-job-" + randStr accountId := "tf-test-dataflow-sa" + randStr + zone := "us-central1-b" vcrTest(t, resource.TestCase{ PreCheck: func() { testAccPreCheck(t) }, @@ -51,19 +52,19 @@ func TestAccDataflowFlexTemplateJob_withServiceAccount(t *testing.T) { CheckDestroy: testAccCheckDataflowJobDestroyProducer(t), Steps: []resource.TestStep{ { - Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId), + Config: testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone), Check: resource.ComposeTestCheckFunc( testAccDataflowJobExists(t, "google_dataflow_flex_template_job.big_data"), - testAccDataflowFlexTemplateJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId), + testAccDataflowFlexTemplateJobHasServiceAccount(t, "google_dataflow_flex_template_job.big_data", accountId, zone), ), }, }, }) } -func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expectedId string) resource.TestCheckFunc { +func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expectedId, zone string) resource.TestCheckFunc { return func(s *terraform.State) error { - instance, err := testAccDataflowFlexTemplateJobGetGeneratedInstance(t, s, res) + instance, err := testAccDataflowFlexTemplateJobGetGeneratedInstance(t, s, res, zone) if err != nil { return fmt.Errorf("Error getting dataflow job instance: %s", err) } @@ -79,7 +80,7 @@ func testAccDataflowFlexTemplateJobHasServiceAccount(t *testing.T, res, expected } } -func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terraform.State, res string) (*compute.Instance, error) { +func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terraform.State, res, zone string) (*compute.Instance, error) { rs, ok := s.RootModule().Resources[res] if !ok { return nil, fmt.Errorf("resource %q not in state", res) @@ -95,7 +96,7 @@ func testAccDataflowFlexTemplateJobGetGeneratedInstance(t *testing.T, s *terrafo err := resource.Retry(1*time.Minute, func() *resource.RetryError { instances, rerr := config.NewComputeClient(config.userAgent).Instances. - List(config.Project, config.Zone). + List(config.Project, zone). Filter(filter). MaxResults(2). Do() @@ -177,7 +178,7 @@ resource "google_dataflow_flex_template_job" "big_data" { } // note: this config creates a job that doesn't actually do anything -func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId string) string { +func testAccDataflowFlexTemplateJob_serviceAccount(bucket, job, accountId, zone string) string { return fmt.Sprintf(` resource "google_storage_bucket" "temp" { name = "%s" @@ -244,13 +245,14 @@ resource "google_dataflow_flex_template_job" "big_data" { inputSubscription = "my-subscription" outputTable = "my-project:my-dataset.my-table" serviceAccount = google_service_account.dataflow-sa.email + zone = "%s" } depends_on = [ google_storage_bucket_iam_member.dataflow-gcs, google_project_iam_member.dataflow-worker ] } -`, bucket, accountId, job) +`, bucket, accountId, job, zone) } <% end -%>