From eabc5bf537f34c12408675240ee496e5c4139b4d Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Wed, 3 Jul 2024 14:59:52 -0300 Subject: [PATCH 01/46] first commit --- modules/tf_cloudbuild_workspace/cb.tf | 30 +++++++++++++++++++- modules/tf_cloudbuild_workspace/variables.tf | 12 ++++++-- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/modules/tf_cloudbuild_workspace/cb.tf b/modules/tf_cloudbuild_workspace/cb.tf index 9aed57c0..865f5582 100644 --- a/modules/tf_cloudbuild_workspace/cb.tf +++ b/modules/tf_cloudbuild_workspace/cb.tf @@ -31,6 +31,10 @@ locals { gh_owner = local.is_gh_repo ? local.gh_repo_url_split[length(local.gh_repo_url_split) - 2] : "" gh_name = local.is_gh_repo ? local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] : "" + is_cb_v2_repo = var.tf_repo_type == "CLOUDBUILD_V2_REPOSITORY" + # Generic repo name extracted from format projects/{{project}}/locations/{{location}}/connections/{{name}} + cb_v2_repo_name = local.is_cb_v2_repo ? element(split("/", cloudbuildv2_repository_id), length(split("/", cloudbuildv2_repository_id)) - 1) : "" + # default build steps default_entrypoint = "terraform" default_build_plan = [ @@ -48,7 +52,7 @@ locals { } # default prefix computed from repo name and dir if specified of form ${repo}-${dir?}-${plan/apply} - repo = local.is_source_repo ? local.source_repo_name : local.gh_name + repo = local.is_source_repo ? local.source_repo_name : local.is_cb_v2_repo ? local.cb_v2_repo_name : local.gh_name default_prefix = var.prefix != "" ? var.prefix : replace(var.tf_repo_dir != "" ? "${local.repo}-${var.tf_repo_dir}" : local.repo, "/", "-") # default substitutions @@ -78,6 +82,30 @@ resource "google_cloudbuild_trigger" "triggers" { } } + # Generic Cloud Build 2nd Gen Repository + dynamic "repository_event_config" { + for_each = local.is_cb_v2_repo ? [1] : [] + content { + repository = var.cloudbuildv2_repository_id + # plan for PRs targeting apply branches + dynamic "pull_request" { + for_each = each.key != "apply" ? [1] : [] + content { + branch = local.apply_branches_regex + invert_regex = false + } + } + # apply for pushes to apply branches + dynamic "push" { + for_each = each.key != "apply" ? [] : [1] + content { + branch = local.apply_branches_regex + invert_regex = false + } + } + } + } + # GH repo dynamic "github" { for_each = local.is_gh_repo ? [1] : [] diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index 6450116c..eb69d1e9 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -168,12 +168,12 @@ variable "tf_repo_dir" { } variable "tf_repo_type" { - description = "Type of repo" + description = "Type of repo. When the repo type is CLOUDBUILD_V2_REPOSITORY, it will use the generig Cloudbuild 2nd gen Repository API." type = string default = "CLOUD_SOURCE_REPOSITORIES" validation { - condition = contains(["CLOUD_SOURCE_REPOSITORIES", "GITHUB"], var.tf_repo_type) - error_message = "Must be one of CLOUD_SOURCE_REPOSITORIES or GITHUB." + condition = contains(["CLOUD_SOURCE_REPOSITORIES", "GITHUB", "CLOUDBUILD_V2_REPOSITORY"], var.tf_repo_type) + error_message = "Must be one of CLOUD_SOURCE_REPOSITORIES, GITHUB, CLOUDBUILD_V2_REPOSITORY" } } @@ -188,3 +188,9 @@ variable "worker_pool_id" { type = string default = "" } + +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{name}}'." + type = string + default = "" +} From c1e39cb8639bfd1ff7e77c0883b414a19a913804 Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Thu, 4 Jul 2024 08:54:28 -0300 Subject: [PATCH 02/46] more info on variables --- modules/tf_cloudbuild_workspace/variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index eb69d1e9..394cdd7b 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -190,7 +190,7 @@ variable "worker_pool_id" { } variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{name}}'." + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." type = string default = "" } From 5a51af21991b9b152916825dea37aef4ec9b7f8e Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Fri, 5 Jul 2024 15:08:09 -0300 Subject: [PATCH 03/46] update modules --- modules/tf_cloudbuild_builder/cb.tf | 25 ++++++++++++++++---- modules/tf_cloudbuild_builder/variables.tf | 9 ++++++- modules/tf_cloudbuild_workspace/cb.tf | 8 +++---- modules/tf_cloudbuild_workspace/variables.tf | 2 +- 4 files changed, 34 insertions(+), 10 deletions(-) diff --git a/modules/tf_cloudbuild_builder/cb.tf b/modules/tf_cloudbuild_builder/cb.tf index 25f779d5..22006011 100644 --- a/modules/tf_cloudbuild_builder/cb.tf +++ b/modules/tf_cloudbuild_builder/cb.tf @@ -38,6 +38,8 @@ locals { source_repo_url_split = local.is_source_repo ? split("/", var.dockerfile_repo_uri) : [] source_repo_project = local.is_source_repo ? local.source_repo_url_split[4] : "" source_repo_name = local.is_source_repo ? local.source_repo_url_split[6] : "" + + use_repo_id = var.dockerfile_repo_id != "" } resource "google_cloudbuild_trigger" "build_trigger" { @@ -45,10 +47,25 @@ resource "google_cloudbuild_trigger" "build_trigger" { location = var.trigger_location name = var.trigger_name description = "Builds a Terraform runner image. Managed by Terraform." - source_to_build { - uri = var.dockerfile_repo_uri - ref = var.dockerfile_repo_ref - repo_type = var.dockerfile_repo_type + + + # repository accepts Generic Cloud Build 2nd Gen Repository + dynamic "source_to_build" { + for_each = local.use_repo_id ? [1] : [] + content { + repository = var.dockerfile_repo_id + ref = var.dockerfile_repo_ref + repo_type = var.dockerfile_repo_type + } + } + + dynamic "source_to_build" { + for_each = local.use_repo_id ? [] : [1] + content { + uri = var.dockerfile_repo_uri + ref = var.dockerfile_repo_ref + repo_type = var.dockerfile_repo_type + } } # todo(bharathkkb): switch to yaml after /~https://github.com/hashicorp/terraform-provider-google/issues/9818 diff --git a/modules/tf_cloudbuild_builder/variables.tf b/modules/tf_cloudbuild_builder/variables.tf index 38870bf6..4250901b 100644 --- a/modules/tf_cloudbuild_builder/variables.tf +++ b/modules/tf_cloudbuild_builder/variables.tf @@ -98,8 +98,15 @@ variable "trigger_location" { } variable "dockerfile_repo_uri" { - description = "The URI of the repo where the Dockerfile for Terraform builder is stored" + description = "The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories." type = string + default = "" +} + +variable "dockerfile_repo_id" { + description = "The repository id where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_uri`." + type = string + default = "" } variable "dockerfile_repo_ref" { diff --git a/modules/tf_cloudbuild_workspace/cb.tf b/modules/tf_cloudbuild_workspace/cb.tf index 865f5582..5619cc09 100644 --- a/modules/tf_cloudbuild_workspace/cb.tf +++ b/modules/tf_cloudbuild_workspace/cb.tf @@ -33,7 +33,7 @@ locals { is_cb_v2_repo = var.tf_repo_type == "CLOUDBUILD_V2_REPOSITORY" # Generic repo name extracted from format projects/{{project}}/locations/{{location}}/connections/{{name}} - cb_v2_repo_name = local.is_cb_v2_repo ? element(split("/", cloudbuildv2_repository_id), length(split("/", cloudbuildv2_repository_id)) - 1) : "" + cb_v2_repo_name = local.is_cb_v2_repo ? element(split("/", var.cloudbuildv2_repository_id), length(split("/", var.cloudbuildv2_repository_id)) - 1) : "" # default build steps default_entrypoint = "terraform" @@ -87,12 +87,12 @@ resource "google_cloudbuild_trigger" "triggers" { for_each = local.is_cb_v2_repo ? [1] : [] content { repository = var.cloudbuildv2_repository_id - # plan for PRs targeting apply branches - dynamic "pull_request" { + # plan for non apply branch + dynamic "push" { for_each = each.key != "apply" ? [1] : [] content { branch = local.apply_branches_regex - invert_regex = false + invert_regex = true } } # apply for pushes to apply branches diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index 394cdd7b..e67efed2 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -190,7 +190,7 @@ variable "worker_pool_id" { } variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." type = string default = "" } From ceb06be61e0d0de81dc6c47597aa9dabd0239541 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Fri, 5 Jul 2024 17:14:54 -0300 Subject: [PATCH 04/46] update documentation --- modules/tf_cloudbuild_builder/README.md | 3 ++- modules/tf_cloudbuild_workspace/README.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 5fc19d45..933c862e 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -38,9 +38,10 @@ This module creates: | cb\_logs\_bucket\_force\_destroy | When deleting the bucket for storing CloudBuild logs, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | | cloudbuild\_sa | Custom SA email to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | dockerfile\_repo\_dir | The directory inside the repo where the Dockerfile is located. If empty defaults to repo root. | `string` | `""` | no | +| dockerfile\_repo\_id | The repository id where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_uri`. | `string` | `""` | no | | dockerfile\_repo\_ref | The branch or tag to use. Use refs/heads/branchname for branches or refs/tags/tagname for tags. | `string` | `"refs/heads/main"` | no | | dockerfile\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | -| dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored | `string` | n/a | yes | +| dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories. | `string` | `""` | no | | enable\_worker\_pool | Set to true to use a private worker pool in the Cloud Build Trigger. | `bool` | `false` | no | | gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | `"us"` | no | | gar\_repo\_name | Name of the Google Artifact Repository where the Terraform builder images are stored. | `string` | `"tf-runners"` | no | diff --git a/modules/tf_cloudbuild_workspace/README.md b/modules/tf_cloudbuild_workspace/README.md index 8965ca9d..727006fd 100644 --- a/modules/tf_cloudbuild_workspace/README.md +++ b/modules/tf_cloudbuild_workspace/README.md @@ -46,6 +46,7 @@ This module creates: | cloudbuild\_plan\_filename | Optional Cloud Build YAML definition used for terraform plan. Defaults to using inline definition. | `string` | `null` | no | | cloudbuild\_sa | Custom SA id of form projects/{{project}}/serviceAccounts/{{email}} to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | cloudbuild\_sa\_roles | Optional to assign to custom CloudBuild SA. Map of project name or any static key to object with project\_id and list of roles. |
map(object({
project_id = string
roles = list(string)
}))
| `{}` | no | +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | | create\_cloudbuild\_sa | Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified. | `bool` | `true` | no | | create\_cloudbuild\_sa\_name | Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty | `string` | `""` | no | | create\_state\_bucket | Create a GCS bucket for storing state. If false `state_bucket_self_link` has to be specified. | `bool` | `true` | no | @@ -61,7 +62,7 @@ This module creates: | tf\_apply\_branches | List of git branches configured to run terraform apply Cloud Build trigger. All other branches will run plan by default. | `list(string)` |
[
"main"
]
| no | | tf\_cloudbuilder | Name of the Cloud Builder image used for running build steps. | `string` | `"hashicorp/terraform:1.3.10"` | no | | tf\_repo\_dir | The directory inside the repo where the Terrafrom root config is located. If empty defaults to repo root. | `string` | `""` | no | -| tf\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | +| tf\_repo\_type | Type of repo. When the repo type is CLOUDBUILD\_V2\_REPOSITORY, it will use the generig Cloudbuild 2nd gen Repository API. | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | | tf\_repo\_uri | The URI of the repo where Terraform configs are stored. | `string` | n/a | yes | | trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"global"` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | From 152dc87677d6b38f0900080069dcf418bc8dd5b9 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 10 Jul 2024 11:08:55 -0300 Subject: [PATCH 05/46] add fixture for cloudbuild workspace for github --- .../README.md | 22 +++++ .../apis.tf | 30 ++++++ .../main.tf | 33 +++++++ .../outputs.tf | 54 ++++++++++ .../variables.tf | 25 +++++ modules/tf_cloudbuild_workspace/variables.tf | 12 +-- .../main.tf | 99 +++++++++++++++++++ .../outputs.tf | 15 +++ .../variables.tf | 31 ++++++ 9 files changed, 315 insertions(+), 6 deletions(-) create mode 100644 examples/tf_cloudbuild_workspace_simple_github/README.md create mode 100644 examples/tf_cloudbuild_workspace_simple_github/apis.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_github/main.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_github/outputs.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_github/variables.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf diff --git a/examples/tf_cloudbuild_workspace_simple_github/README.md b/examples/tf_cloudbuild_workspace_simple_github/README.md new file mode 100644 index 00000000..7c167808 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/README.md @@ -0,0 +1,22 @@ + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | +| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| artifacts\_bucket | Bucket for storing TF plans | +| cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply | +| cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan | +| cloudbuild\_sa | SA used by Cloud Build triggers | +| github\_repo\_id | CSR repo for storing TF configs | +| logs\_bucket | Bucket for storing TF logs | +| project\_id | n/a | +| state\_bucket | Bucket for storing TF state | + + diff --git a/examples/tf_cloudbuild_workspace_simple_github/apis.tf b/examples/tf_cloudbuild_workspace_simple_github/apis.tf new file mode 100644 index 00000000..38049631 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/apis.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 15.0" + + project_id = var.project_id + disable_services_on_destroy = false + + activate_apis = [ + "iam.googleapis.com", + "secretmanager.googleapis.com", + "compute.googleapis.com", + "cloudbuild.googleapis.com", + ] +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf new file mode 100644 index 00000000..67104ffd --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -0,0 +1,33 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "tf_workspace" { + source = "../../modules/tf_cloudbuild_workspace" + + project_id = module.enabled_google_apis.project_id + cloudbuildv2_repository_id = var.cloudbuildv2_repository_id + tf_repo_uri = "" + + # allow log/state buckets to be destroyed + buckets_force_destroy = true + cloudbuild_sa_roles = { (module.enabled_google_apis.project_id) = { + project_id = module.enabled_google_apis.project_id, + roles = ["roles/compute.networkAdmin"] + } + } + cloudbuild_env_vars = ["TF_VAR_project_id=${var.project_id}"] + +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/outputs.tf b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf new file mode 100644 index 00000000..17765327 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf @@ -0,0 +1,54 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloudbuild_plan_trigger_id" { + description = "Trigger used for running TF plan" + value = module.tf_workspace.cloudbuild_plan_trigger_id +} + +output "cloudbuild_apply_trigger_id" { + description = "Trigger used for running TF apply" + value = module.tf_workspace.cloudbuild_apply_trigger_id +} + +output "cloudbuild_sa" { + description = "SA used by Cloud Build triggers" + value = module.tf_workspace.cloudbuild_sa +} + +output "state_bucket" { + description = "Bucket for storing TF state" + value = module.tf_workspace.state_bucket +} + +output "logs_bucket" { + description = "Bucket for storing TF logs" + value = module.tf_workspace.logs_bucket +} + +output "artifacts_bucket" { + description = "Bucket for storing TF plans" + value = module.tf_workspace.artifacts_bucket +} + +output "github_repo_id" { + description = "CSR repo for storing TF configs" + value = "TBD" +} + +output "project_id" { + value = var.project_id +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/variables.tf new file mode 100644 index 00000000..3ce9c80f --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/variables.tf @@ -0,0 +1,25 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + type = string +} diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index e67efed2..9ec4c1cd 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -148,6 +148,12 @@ variable "prefix" { default = "" } +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + type = string + default = "" +} + variable "tf_repo_uri" { description = "The URI of the repo where Terraform configs are stored." type = string @@ -188,9 +194,3 @@ variable "worker_pool_id" { type = string default = "" } - -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." - type = string - default = "" -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf new file mode 100644 index 00000000..c9883856 --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf @@ -0,0 +1,99 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "github_workspace" { + source = "../../../examples/tf_cloudbuild_workspace_simple_github" + + project_id = var.project_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id +} + + +locals { + # GitHub repo url of form "github.com/owner/name" + github_app_installation_id = "47590865" + location = "us-central1" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gh_repo_url_split = split("/", local.repoURLWithoutSuffix) + gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] + + github_secret_version_id = google_secret_manager_secret_version.github_token_secret_version.name + github_secret_id = google_secret_manager_secret.github_token_secret.id + + host_connection_name = "cb-${random_id.resources_random_id.dec}-${var.project_id}" + repo_connection_name = "cb-${random_id.resources_random_id.dec}-${local.gh_name}" +} + +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "github_token_secret" { + project = var.project_id + secret_id = "cb-github-${random_id.resources_random_id.dec}-${local.gh_name}" + + labels = { + label = "cb-${random_id.resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "github_token_secret_version" { + secret = google_secret_manager_secret.github_token_secret.id + secret_data = var.im_github_pat +} + +resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { + project = var.project_id + secret_id = local.github_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "resources_random_id" { + byte_length = 4 +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + location = local.location + + name = local.host_connection_name + + github_config { + app_installation_id = local.github_app_installation_id + authorizer_credential { + oauth_token_secret_version = local.github_secret_version_id + } + } +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + location = local.location + name = local.repo_connection_name + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf new file mode 100644 index 00000000..dd112913 --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf @@ -0,0 +1,15 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf new file mode 100644 index 00000000..56c6936b --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "im_github_pat" { + description = "GitHub personal access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} From fbb8c3b92c25c82c3c1da8c33188d13b1cdaabaf Mon Sep 17 00:00:00 2001 From: caetano-colin Date: Fri, 12 Jul 2024 12:08:09 -0300 Subject: [PATCH 06/46] update README.md adding information about github connections --- examples/tf_cloudbuild_workspace_simple_github/README.md | 9 +++++++++ modules/tf_cloudbuild_workspace/README.md | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/examples/tf_cloudbuild_workspace_simple_github/README.md b/examples/tf_cloudbuild_workspace_simple_github/README.md index 7c167808..48aa7d3e 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/README.md +++ b/examples/tf_cloudbuild_workspace_simple_github/README.md @@ -1,3 +1,12 @@ +## Github Requirements for Cloud Build Connection + +When using a Cloud Build 2nd generation repository, a Cloud Build connection to your repository provider will be needed. For Github connections you will need: + +- [Install Cloud Build App on Github](/~https://github.com/apps/google-cloud-build). +- [Create Personal Access Token on Github with `repo` and `read:user` (or if app is installed in org use `read:org`)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). + +For more information on this topic refer to the ["Connect with Github Documentation"](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) + ## Inputs diff --git a/modules/tf_cloudbuild_workspace/README.md b/modules/tf_cloudbuild_workspace/README.md index 727006fd..6749ad3a 100644 --- a/modules/tf_cloudbuild_workspace/README.md +++ b/modules/tf_cloudbuild_workspace/README.md @@ -2,6 +2,15 @@ TF Cloud Build Workspace blueprint creates an opinionated workflow for actuating Terraform on Cloud Build. A set of Cloud Build triggers manage plan and apply operations on a root configuration stored in a VCS repo. Cloud Build triggers use a per workspace Service Account which can be configured with minimal permissions required by a given Terraform configuration. Optionally dedicated GCS buckets for state and log storage are also created. +## Github Requirements for Cloud Build Connection + +When using a Cloud Build 2nd generation repository, a Cloud Build connection to your repository provider will be needed. For Github connections you will need: + +- [Install Cloud Build App on Github](/~https://github.com/apps/google-cloud-build). +- [Create Personal Access Token on Github with `repo` and `read:user` (or if app is installed in org use `read:org`)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). + +For more information on this topic refer to the ["Connect with Github Documentation"](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) + ## Usage Basic usage of this module is as follows: From e1851091adb3fc689bc5438dce12253dee81458d Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 15 Jul 2024 14:15:56 -0300 Subject: [PATCH 07/46] update github workspace example --- .../files/.gitignore | 2 + .../files/backend.tf | 22 +++++++++ .../files/main.tf | 48 +++++++++++++++++++ .../files/variables.tf | 19 ++++++++ .../main.tf | 15 ++++++ .../scripts/push-to-repo.sh | 48 +++++++++++++++++++ .../variables.tf | 23 +++++++++ .../main.tf | 19 ++++---- 8 files changed, 188 insertions(+), 8 deletions(-) create mode 100644 examples/tf_cloudbuild_workspace_simple_github/files/.gitignore create mode 100644 examples/tf_cloudbuild_workspace_simple_github/files/backend.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_github/files/main.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_github/files/variables.tf create mode 100755 examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/.gitignore b/examples/tf_cloudbuild_workspace_simple_github/files/.gitignore new file mode 100644 index 00000000..a260b99a --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/files/.gitignore @@ -0,0 +1,2 @@ +# Local .terraform directories +**/.terraform* diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf new file mode 100644 index 00000000..dc5bf3de --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + backend "gcs" { + bucket = "tf-state-prod" + prefix = "terraform/state" + } +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf new file mode 100644 index 00000000..4fbf40ef --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test-vpc-module" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + project_id = var.project_id + network_name = "my-custom-mode-network" + mtu = 1460 + + subnets = [ + { + subnet_name = "subnet-01" + subnet_ip = "10.10.10.0/24" + subnet_region = "us-west1" + }, + { + subnet_name = "subnet-02" + subnet_ip = "10.10.20.0/24" + subnet_region = "us-west1" + subnet_private_access = "true" + subnet_flow_logs = "true" + }, + { + subnet_name = "subnet-03" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1" + subnet_flow_logs = "true" + subnet_flow_logs_interval = "INTERVAL_10_MIN" + subnet_flow_logs_sampling = 0.7 + subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + subnet_flow_logs_filter = "false" + } + ] +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf new file mode 100644 index 00000000..b335edcc --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the network in" +} diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf index 67104ffd..f16d0c3d 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -18,8 +18,13 @@ module "tf_workspace" { source = "../../modules/tf_cloudbuild_workspace" project_id = module.enabled_google_apis.project_id + tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" cloudbuildv2_repository_id = var.cloudbuildv2_repository_id tf_repo_uri = "" + location = "us-central1" + trigger_location = "us-central1" + create_cloudbuild_sa = var.create_cloudbuild_sa + create_cloudbuild_sa_name = var.create_cloudbuild_sa_name # allow log/state buckets to be destroyed buckets_force_destroy = true @@ -30,4 +35,14 @@ module "tf_workspace" { } cloudbuild_env_vars = ["TF_VAR_project_id=${var.project_id}"] + depends_on = [module.enabled_google_apis] +} + +module "bootstrap_github_repo" { + source = "terraform-google-modules/gcloud/google" + version = "~> 3.1" + upgrade = false + + create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" + create_cmd_body = "${var.github_pat} ${var.repository_uri} ${path.module}/files" } diff --git a/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh new file mode 100755 index 00000000..d6ba3aa3 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -ex + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +GITHUB_TOKEN=$1 +REPO_URL=$2 +TF_CONFIG_PATH=$3 + + +# extract portion after https:// from URL +URL_PARTS=($(echo $REPO_URL | awk -F/ '{print $3, $4, $5}')) +# construct the new authenticated URL +AUTH_REPO_URL="https://${GITHUB_TOKEN}:@${URL_PARTS[0]}/${URL_PARTS[1]}/${URL_PARTS[2]}" + +tmp_dir=$(mktemp -d) +git clone "${AUTH_REPO_URL}" "${tmp_dir}" +cp -r "${TF_CONFIG_PATH}/." "${tmp_dir}" +pushd "${tmp_dir}" +git config init.defaultBranch main +git config user.email "terraform-robot@example.com" +git config user.name "TF Robot" +git checkout plan || git checkout -b plan +git add -A +git commit -m "init tf configs" +git push origin plan -f +sleep 60 +git checkout main || git checkout -b main +git push origin main -f +sleep 120 diff --git a/examples/tf_cloudbuild_workspace_simple_github/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/variables.tf index 3ce9c80f..d4581737 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/variables.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/variables.tf @@ -23,3 +23,26 @@ variable "cloudbuildv2_repository_id" { description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." type = string } + +variable "create_cloudbuild_sa" { + description = "Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified." + type = bool + default = true +} + +variable "create_cloudbuild_sa_name" { + description = "Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty" + type = string + default = "" +} + +variable "github_pat" { + description = "GitHub personal access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf index c9883856..672eafeb 100644 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf @@ -14,14 +14,6 @@ * limitations under the License. */ -module "github_workspace" { - source = "../../../examples/tf_cloudbuild_workspace_simple_github" - - project_id = var.project_id - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id -} - - locals { # GitHub repo url of form "github.com/owner/name" github_app_installation_id = "47590865" @@ -39,6 +31,17 @@ locals { repo_connection_name = "cb-${random_id.resources_random_id.dec}-${local.gh_name}" } +module "github_workspace" { + source = "../../../examples/tf_cloudbuild_workspace_simple_github" + + project_id = var.project_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id + create_cloudbuild_sa = true + create_cloudbuild_sa_name = "tf-deploy-cb-github" + github_pat = var.im_github_pat + repository_uri = var.repository_uri +} + // Create a secret containing the personal access token and grant permissions to the Service Agent. resource "google_secret_manager_secret" "github_token_secret" { project = var.project_id From 92bdda766061ae547d413e09eda96e99c75a0e58 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 15 Jul 2024 15:01:53 -0300 Subject: [PATCH 08/46] update documentation --- examples/tf_cloudbuild_workspace_simple_github/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/examples/tf_cloudbuild_workspace_simple_github/README.md b/examples/tf_cloudbuild_workspace_simple_github/README.md index 48aa7d3e..5fc2f8b8 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/README.md +++ b/examples/tf_cloudbuild_workspace_simple_github/README.md @@ -13,7 +13,11 @@ For more information on this topic refer to the ["Connect with Github Documentat | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | +| create\_cloudbuild\_sa | Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified. | `bool` | `true` | no | +| create\_cloudbuild\_sa\_name | Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty | `string` | `""` | no | +| github\_pat | GitHub personal access token. | `string` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | +| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs From 40506debdf7a8b9872cfe465bd9069f7af02322f Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 15 Jul 2024 22:08:52 -0300 Subject: [PATCH 09/46] fix module source in examples for IM workspace --- examples/im_cloudbuild_workspace_github/main.tf | 3 ++- examples/im_cloudbuild_workspace_gitlab/main.tf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/im_cloudbuild_workspace_github/main.tf b/examples/im_cloudbuild_workspace_github/main.tf index 03937f47..5679e3fe 100644 --- a/examples/im_cloudbuild_workspace_github/main.tf +++ b/examples/im_cloudbuild_workspace_github/main.tf @@ -15,7 +15,8 @@ */ module "im_workspace" { - source = "../../modules/im_cloudbuild_workspace" + source = "terraform-google-modules/bootstrap/google//modules/im_cloudbuild_workspace" + version = "~> 8.0" project_id = var.project_id deployment_id = "im-example-github-deployment" diff --git a/examples/im_cloudbuild_workspace_gitlab/main.tf b/examples/im_cloudbuild_workspace_gitlab/main.tf index f38bde4b..0d980e7e 100644 --- a/examples/im_cloudbuild_workspace_gitlab/main.tf +++ b/examples/im_cloudbuild_workspace_gitlab/main.tf @@ -15,7 +15,8 @@ */ module "im_workspace" { - source = "../../modules/im_cloudbuild_workspace" + source = "terraform-google-modules/bootstrap/google//modules/im_cloudbuild_workspace" + version = "~> 8.0" project_id = var.project_id deployment_id = "im-example-gitlab-deployment" From 5653993b56401fe7e4ca0967a9431542b02225cc Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 15 Jul 2024 22:10:21 -0300 Subject: [PATCH 10/46] fix shellcheck lint --- .../scripts/push-to-repo.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh index d6ba3aa3..21ca0345 100755 --- a/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh +++ b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh @@ -27,9 +27,9 @@ TF_CONFIG_PATH=$3 # extract portion after https:// from URL -URL_PARTS=($(echo $REPO_URL | awk -F/ '{print $3, $4, $5}')) +IFS="/"; mapfile -t -d / URL_PARTS < <(echo "$REPO_URL") # construct the new authenticated URL -AUTH_REPO_URL="https://${GITHUB_TOKEN}:@${URL_PARTS[0]}/${URL_PARTS[1]}/${URL_PARTS[2]}" +AUTH_REPO_URL="https://${GITHUB_TOKEN}:@${URL_PARTS[2]}/${URL_PARTS[3]}/${URL_PARTS[4]}" tmp_dir=$(mktemp -d) git clone "${AUTH_REPO_URL}" "${tmp_dir}" From 8b328958c7e4b0c889566b5a79612b35f4e2a6c8 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 17 Jul 2024 22:20:47 -0300 Subject: [PATCH 11/46] add integration test for cloudbuild workspace using github --- .../README.md | 3 - .../files/backend.tf | 2 +- .../files/main.tf | 2 +- .../files/variables.tf | 2 +- .../main.tf | 3 - .../outputs.tf | 5 - .../scripts/push-to-repo.sh | 3 +- .../variables.tf | 12 - modules/tf_cloudbuild_workspace/README.md | 4 +- modules/tf_cloudbuild_workspace/cb.tf | 7 +- modules/tf_cloudbuild_workspace/variables.tf | 5 +- .../main.tf | 8 +- .../outputs.tf | 34 +++ ...cloudbuild_workspace_simple_github_test.go | 212 ++++++++++++++++++ 14 files changed, 263 insertions(+), 39 deletions(-) create mode 100644 test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go diff --git a/examples/tf_cloudbuild_workspace_simple_github/README.md b/examples/tf_cloudbuild_workspace_simple_github/README.md index 5fc2f8b8..dfa8eeba 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/README.md +++ b/examples/tf_cloudbuild_workspace_simple_github/README.md @@ -13,8 +13,6 @@ For more information on this topic refer to the ["Connect with Github Documentat | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | -| create\_cloudbuild\_sa | Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified. | `bool` | `true` | no | -| create\_cloudbuild\_sa\_name | Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty | `string` | `""` | no | | github\_pat | GitHub personal access token. | `string` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | | repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | @@ -27,7 +25,6 @@ For more information on this topic refer to the ["Connect with Github Documentat | cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply | | cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan | | cloudbuild\_sa | SA used by Cloud Build triggers | -| github\_repo\_id | CSR repo for storing TF configs | | logs\_bucket | Bucket for storing TF logs | | project\_id | n/a | | state\_bucket | Bucket for storing TF state | diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf index dc5bf3de..83ae74df 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf index 4fbf40ef..81b50afd 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf index b335edcc..388f7b4a 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/variables.tf @@ -1,5 +1,5 @@ /** - * Copyright 2022 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf index f16d0c3d..52ca222d 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -20,11 +20,8 @@ module "tf_workspace" { project_id = module.enabled_google_apis.project_id tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" cloudbuildv2_repository_id = var.cloudbuildv2_repository_id - tf_repo_uri = "" location = "us-central1" trigger_location = "us-central1" - create_cloudbuild_sa = var.create_cloudbuild_sa - create_cloudbuild_sa_name = var.create_cloudbuild_sa_name # allow log/state buckets to be destroyed buckets_force_destroy = true diff --git a/examples/tf_cloudbuild_workspace_simple_github/outputs.tf b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf index 17765327..d3b9b589 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/outputs.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf @@ -44,11 +44,6 @@ output "artifacts_bucket" { value = module.tf_workspace.artifacts_bucket } -output "github_repo_id" { - description = "CSR repo for storing TF configs" - value = "TBD" -} - output "project_id" { value = var.project_id } diff --git a/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh index 21ca0345..6f7fa5c0 100755 --- a/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh +++ b/examples/tf_cloudbuild_workspace_simple_github/scripts/push-to-repo.sh @@ -27,7 +27,7 @@ TF_CONFIG_PATH=$3 # extract portion after https:// from URL -IFS="/"; mapfile -t -d / URL_PARTS < <(echo "$REPO_URL") +IFS="/"; mapfile -t -d / URL_PARTS < <(printf "%s" "$REPO_URL") # construct the new authenticated URL AUTH_REPO_URL="https://${GITHUB_TOKEN}:@${URL_PARTS[2]}/${URL_PARTS[3]}/${URL_PARTS[4]}" @@ -44,5 +44,6 @@ git commit -m "init tf configs" git push origin plan -f sleep 60 git checkout main || git checkout -b main +git merge plan git push origin main -f sleep 120 diff --git a/examples/tf_cloudbuild_workspace_simple_github/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/variables.tf index d4581737..649eec55 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/variables.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/variables.tf @@ -24,18 +24,6 @@ variable "cloudbuildv2_repository_id" { type = string } -variable "create_cloudbuild_sa" { - description = "Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified." - type = bool - default = true -} - -variable "create_cloudbuild_sa_name" { - description = "Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty" - type = string - default = "" -} - variable "github_pat" { description = "GitHub personal access token." type = string diff --git a/modules/tf_cloudbuild_workspace/README.md b/modules/tf_cloudbuild_workspace/README.md index 6749ad3a..39f8fc6e 100644 --- a/modules/tf_cloudbuild_workspace/README.md +++ b/modules/tf_cloudbuild_workspace/README.md @@ -55,7 +55,7 @@ This module creates: | cloudbuild\_plan\_filename | Optional Cloud Build YAML definition used for terraform plan. Defaults to using inline definition. | `string` | `null` | no | | cloudbuild\_sa | Custom SA id of form projects/{{project}}/serviceAccounts/{{email}} to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | cloudbuild\_sa\_roles | Optional to assign to custom CloudBuild SA. Map of project name or any static key to object with project\_id and list of roles. |
map(object({
project_id = string
roles = list(string)
}))
| `{}` | no | -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be set if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | | create\_cloudbuild\_sa | Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified. | `bool` | `true` | no | | create\_cloudbuild\_sa\_name | Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty | `string` | `""` | no | | create\_state\_bucket | Create a GCS bucket for storing state. If false `state_bucket_self_link` has to be specified. | `bool` | `true` | no | @@ -72,7 +72,7 @@ This module creates: | tf\_cloudbuilder | Name of the Cloud Builder image used for running build steps. | `string` | `"hashicorp/terraform:1.3.10"` | no | | tf\_repo\_dir | The directory inside the repo where the Terrafrom root config is located. If empty defaults to repo root. | `string` | `""` | no | | tf\_repo\_type | Type of repo. When the repo type is CLOUDBUILD\_V2\_REPOSITORY, it will use the generig Cloudbuild 2nd gen Repository API. | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | -| tf\_repo\_uri | The URI of the repo where Terraform configs are stored. | `string` | n/a | yes | +| tf\_repo\_uri | The URI of the repo where Terraform configs are stored. Must be set if repository type is not `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | | trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"global"` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | diff --git a/modules/tf_cloudbuild_workspace/cb.tf b/modules/tf_cloudbuild_workspace/cb.tf index 5619cc09..b9e1c757 100644 --- a/modules/tf_cloudbuild_workspace/cb.tf +++ b/modules/tf_cloudbuild_workspace/cb.tf @@ -52,8 +52,9 @@ locals { } # default prefix computed from repo name and dir if specified of form ${repo}-${dir?}-${plan/apply} - repo = local.is_source_repo ? local.source_repo_name : local.is_cb_v2_repo ? local.cb_v2_repo_name : local.gh_name - default_prefix = var.prefix != "" ? var.prefix : replace(var.tf_repo_dir != "" ? "${local.repo}-${var.tf_repo_dir}" : local.repo, "/", "-") + repo = local.is_source_repo ? local.source_repo_name : local.is_cb_v2_repo ? local.cb_v2_repo_name : local.gh_name + default_prefix = var.prefix != "" ? var.prefix : replace(var.tf_repo_dir != "" ? "${local.repo}-${var.tf_repo_dir}" : local.repo, "/", "-") + repo_uri_description = local.is_cb_v2_repo ? local.cb_v2_repo_name : var.tf_repo_uri # default substitutions default_subst = merge({ @@ -70,7 +71,7 @@ resource "google_cloudbuild_trigger" "triggers" { project = var.project_id location = var.trigger_location name = "${local.default_prefix}-${each.key}" - description = "${title(each.key)} Terraform configs for ${var.tf_repo_uri} ${var.tf_repo_dir}. Managed by Terraform." + description = "${title(each.key)} Terraform configs for ${local.repo_uri_description} ${var.tf_repo_dir}. Managed by Terraform." # CSR repo dynamic "trigger_template" { diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index 9ec4c1cd..efb423e7 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -149,14 +149,15 @@ variable "prefix" { } variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be set if repository type is `CLOUDBUILD_V2_REPOSITORY`." type = string default = "" } variable "tf_repo_uri" { - description = "The URI of the repo where Terraform configs are stored." + description = "The URI of the repo where Terraform configs are stored. Must be set if repository type is not `CLOUDBUILD_V2_REPOSITORY`." type = string + default = "" } variable "tf_apply_branches" { diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf index 672eafeb..a433b57f 100644 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf @@ -15,10 +15,10 @@ */ locals { - # GitHub repo url of form "github.com/owner/name" github_app_installation_id = "47590865" location = "us-central1" - repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + # GitHub repo url of form "github.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") gh_repo_url_split = split("/", local.repoURLWithoutSuffix) @@ -28,7 +28,7 @@ locals { github_secret_id = google_secret_manager_secret.github_token_secret.id host_connection_name = "cb-${random_id.resources_random_id.dec}-${var.project_id}" - repo_connection_name = "cb-${random_id.resources_random_id.dec}-${local.gh_name}" + repo_connection_name = local.gh_name } module "github_workspace" { @@ -36,8 +36,6 @@ module "github_workspace" { project_id = var.project_id cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - create_cloudbuild_sa = true - create_cloudbuild_sa_name = "tf-deploy-cb-github" github_pat = var.im_github_pat repository_uri = var.repository_uri } diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf index dd112913..0054095a 100644 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf +++ b/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf @@ -13,3 +13,37 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +output "cloudbuild_plan_trigger_id" { + description = "Trigger used for running TF plan" + value = module.github_workspace.cloudbuild_plan_trigger_id +} + +output "cloudbuild_apply_trigger_id" { + description = "Trigger used for running TF apply" + value = module.github_workspace.cloudbuild_apply_trigger_id +} + +output "cloudbuild_sa" { + description = "SA used by Cloud Build triggers" + value = module.github_workspace.cloudbuild_sa +} + +output "state_bucket" { + description = "Bucket for storing TF state" + value = module.github_workspace.state_bucket +} + +output "logs_bucket" { + description = "Bucket for storing TF logs" + value = module.github_workspace.logs_bucket +} + +output "artifacts_bucket" { + description = "Bucket for storing TF plans" + value = module.github_workspace.artifacts_bucket +} + +output "project_id" { + value = var.project_id +} diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go new file mode 100644 index 00000000..f4fae43f --- /dev/null +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -0,0 +1,212 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tf_cloudbuild_workspace_simple_github + +import ( + "context" + "fmt" + "path" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/git" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + cftutils "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/google/go-github/v62/github" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-bootstrap/test/integration/utils" +) + +type GitHubClient struct { + t *testing.T + client *github.Client + owner string + repoName string + repository *github.Repository +} + +func NewGitHubClient(t *testing.T, token, owner, repo string) *GitHubClient { + t.Helper() + client := github.NewClient(nil).WithAuthToken(token) + return &GitHubClient{ + t: t, + client: client, + owner: owner, + repoName: repo, + } +} + +func (gh *GitHubClient) GetRepository(ctx context.Context) *github.Repository { + repo, resp, err := gh.client.Repositories.Get(ctx, gh.owner, gh.repoName) + if resp.StatusCode != 404 && err != nil { + gh.t.Fatal(err.Error()) + } + gh.repository = repo + return repo +} + +func (gh *GitHubClient) CreateRepository(ctx context.Context, org, repoName string) *github.Repository { + newRepo := &github.Repository{ + Name: github.String(repoName), + AutoInit: github.Bool(true), + Private: github.Bool(true), + Visibility: github.String("private"), + } + repo, _, err := gh.client.Repositories.Create(ctx, org, newRepo) + if err != nil { + gh.t.Fatal(err.Error()) + } + gh.repository = repo + return repo +} + +func (gh *GitHubClient) DeleteRepository(ctx context.Context) { + resp, err := gh.client.Repositories.Delete(ctx, gh.owner, *gh.repository.Name) + if resp.StatusCode != 404 && err != nil { + gh.t.Fatal(err.Error()) + } +} + +func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { + ctx := context.Background() + + repoName := fmt.Sprintf("cb-bp-test-%s", utils.GetRandomStringFromSetup(t)) + githubPAT := cftutils.ValFromEnv(t, "IM_GITHUB_PAT") + owner := "im-goose" + client := NewGitHubClient(t, githubPAT, owner, repoName) + + repo := client.GetRepository(ctx) + if repo == nil { + client.CreateRepository(ctx, client.owner, client.repoName) + } + + // Testing the module's feature of appending the ".git" suffix if it's missing + repoURL := strings.TrimSuffix(client.repository.GetCloneURL(), ".git") + vars := map[string]interface{}{ + "im_github_pat": githubPAT, + "repository_uri": repoURL, + } + bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + t.Cleanup(func() { + // Delete the repository if we hit a failed state + if t.Failed() { + client.DeleteRepository(ctx) + } + }) + + projectID := bpt.GetStringOutput("project_id") + + // cloud build triggers + triggers := []string{"plan", "apply"} + for _, trigger := range triggers { + triggerOP := utils.LastElement(bpt.GetStringOutput(fmt.Sprintf("cloudbuild_%s_trigger_id", trigger)), "/") + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, "us-central1", projectID) + assert.Equal(fmt.Sprintf("%s-%s", repoName, trigger), cloudBuildOP.Get("name").String(), "should have the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-%s@%s.iam.gserviceaccount.com", projectID, repoName, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") + } + + // artifacts, state and log buckets + logsBucket := utils.LastElement(bpt.GetStringOutput("logs_bucket"), "/") + stateBucket := utils.LastElement(bpt.GetStringOutput("state_bucket"), "/") + artifactsBucket := utils.LastElement(bpt.GetStringOutput("artifacts_bucket"), "/") + buckets := []string{artifactsBucket, logsBucket, stateBucket} + for _, bucket := range buckets { + // we can't use runf since we need to override --format json with --json for alpha storage + bucketOP := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", bucket), gcloud.WithCommonArgs([]string{"--project", projectID, "--json"})).Array() + assert.Equalf(1, len(bucketOP), "%s bucket should exist", bucket) + assert.Truef(bucketOP[0].Get("metadata.iamConfiguration.uniformBucketLevelAccess.enabled").Bool(), "%s bucket uniformBucketLevelAccess should be enabled", bucket) + assert.Truef(bucketOP[0].Get("metadata.versioning.enabled").Bool(), "%s bucket versioning should be enabled", bucket) + } + + // CB SA IAM + cbSA := utils.LastElement(bpt.GetStringOutput("cloudbuild_sa"), "/") + iamOP := gcloud.Runf(t, "projects get-iam-policy %s --flatten bindings --filter bindings.members:'serviceAccount:%s'", projectID, cbSA).Array() + cftutils.GetFirstMatchResult(t, iamOP, "bindings.role", "roles/compute.networkAdmin") + + // e2e test for testing actuation through both plan/apply branches + applyTrigger := utils.LastElement(bpt.GetStringOutput("cloudbuild_apply_trigger_id"), "/") + planTrigger := utils.LastElement(bpt.GetStringOutput("cloudbuild_plan_trigger_id"), "/") + + // set up repo + tmpDir := t.TempDir() + git := git.NewCmdConfig(t, git.WithDir(tmpDir)) + gitRun := func(args ...string) { + _, err := git.RunCmdE(args...) + if err != nil { + t.Fatal(err) + } + } + + gitRun("clone", fmt.Sprintf("https://%s@github.com/%s/%s", githubPAT, owner, repoName), tmpDir) + gitRun("config", "user.email", "tf-robot@example.com") + gitRun("config", "user.name", "TF Robot") + + // push commits on plan and main branches + // plan branch should trigger plan trigger + // main branch should trigger apply trigger + branches := []string{"plan", "main"} + for _, branch := range branches { + _, err := git.RunCmdE("checkout", branch) + if err != nil { + git.RunCmdE("checkout", "-b", branch) + } + git.CommitWithMsg(fmt.Sprintf("%s commit", branch), []string{"--allow-empty"}) + gitRun("push", "--set-upstream", "origin", branch, "-f") + lastCommit := git.GetLatestCommit() + // filter builds triggered based on pushed commit sha + buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --region %s --project %s --limit 1 --sort-by ~createTime", lastCommit, "us-central1", projectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } + return true, nil + } + } + cftutils.Poll(t, pollCloudBuild(buildListCmd), 20, 10*time.Second) + build := gcloud.Runf(t, buildListCmd).Array()[0] + switch branch { + case "plan": + assert.Equal(planTrigger, build.Get("buildTriggerId").String(), "was triggered by plan trigger") + assert.Contains(build.Get("artifacts.objects.location").String(), path.Join(artifactsBucket, "plan"), "artifacts were uploaded to the correct location") + case "main": + assert.Equal(applyTrigger, build.Get("buildTriggerId").String(), "was triggered by apply trigger") + assert.Contains(build.Get("artifacts.objects.location").String(), path.Join(artifactsBucket, "apply"), "artifacts were uploaded to the correct location") + } + } + }) + + bpt.DefineTeardown(func(assert *assert.Assertions) { + // Guarantee clean up even if the normal gcloud/teardown run into errors + t.Cleanup(func() { + client.DeleteRepository(ctx) + bpt.DefaultTeardown(assert) + }) + }) + + bpt.Test() +} From 58654b98695fdae565beb14b83bba8e783da1e5d Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 17 Jul 2024 23:27:30 -0300 Subject: [PATCH 12/46] revert change in IM examples --- examples/im_cloudbuild_workspace_github/main.tf | 3 +-- examples/im_cloudbuild_workspace_gitlab/main.tf | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/examples/im_cloudbuild_workspace_github/main.tf b/examples/im_cloudbuild_workspace_github/main.tf index 5679e3fe..03937f47 100644 --- a/examples/im_cloudbuild_workspace_github/main.tf +++ b/examples/im_cloudbuild_workspace_github/main.tf @@ -15,8 +15,7 @@ */ module "im_workspace" { - source = "terraform-google-modules/bootstrap/google//modules/im_cloudbuild_workspace" - version = "~> 8.0" + source = "../../modules/im_cloudbuild_workspace" project_id = var.project_id deployment_id = "im-example-github-deployment" diff --git a/examples/im_cloudbuild_workspace_gitlab/main.tf b/examples/im_cloudbuild_workspace_gitlab/main.tf index 0d980e7e..f38bde4b 100644 --- a/examples/im_cloudbuild_workspace_gitlab/main.tf +++ b/examples/im_cloudbuild_workspace_gitlab/main.tf @@ -15,8 +15,7 @@ */ module "im_workspace" { - source = "terraform-google-modules/bootstrap/google//modules/im_cloudbuild_workspace" - version = "~> 8.0" + source = "../../modules/im_cloudbuild_workspace" project_id = var.project_id deployment_id = "im-example-gitlab-deployment" From 6390a359a0d60750695fbea76ac8a6b62c034736 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 18 Jul 2024 16:51:22 -0300 Subject: [PATCH 13/46] add build steps to test tfworkspace for github --- build/int.cloudbuild.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index b19d7376..9ba64899 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -173,6 +173,25 @@ steps: args: ['/bin/bash', '-c', 'cft test run TestIMCloudBuildWorkspaceGitLab --stage teardown --verbose'] secretEnv: ['IM_GITLAB_PAT'] +- id: apply-tfworkspace-github + waitFor: + - create-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage apply --verbose'] + secretEnv: ['IM_GITHUB_PAT'] +- id: verify-tfworkspace-github + waitFor: + - apply-tfworkspace-github + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage verify --verbose'] + secretEnv: ['IM_GITHUB_PAT'] +- id: teardown-tfworkspace-github + waitFor: + - verify-tfworkspace-github + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage teardown --verbose'] + secretEnv: ['IM_GITHUB_PAT'] + availableSecrets: secretManager: - versionName: $_IM_GITHUB_PAT_SECRET_ID/versions/latest From 022763b1c88887b8cae212d24d7c94e661aedbb6 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 18 Jul 2024 20:05:09 -0300 Subject: [PATCH 14/46] use different backend prefix --- examples/tf_cloudbuild_workspace_simple_github/files/backend.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf index 83ae74df..ff791b8b 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "tf-state-prod" - prefix = "terraform/state" + prefix = "terraform/github/state" } } From a2274972bbfca266042cbcd03fac636a4ee15261 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Fri, 19 Jul 2024 09:03:56 -0300 Subject: [PATCH 15/46] use distinct network name for the tf workspace github --- examples/tf_cloudbuild_workspace_simple_github/files/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf index 81b50afd..3fa9b64b 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf @@ -18,7 +18,7 @@ module "test-vpc-module" { source = "terraform-google-modules/network/google" version = "~> 9.0" project_id = var.project_id - network_name = "my-custom-mode-network" + network_name = "my-custom-mode-network-gh" mtu = 1460 subnets = [ From 17be087f6af1e98434576af6abda6da59b18c8aa Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 22 Jul 2024 10:01:42 -0300 Subject: [PATCH 16/46] add case for build failure in testes --- .../tf_cloudbuild_workspace_simple_test.go | 4 ++++ .../tf_cloudbuild_workspace_simple_github_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go index 1fd3a480..38bb9b5a 100644 --- a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go @@ -106,6 +106,10 @@ func TestTFCloudBuildWorkspaceSimple(t *testing.T) { if latestWorkflowRunStatus == "SUCCESS" { return false, nil } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) + } return true, nil } } diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index f4fae43f..e13da0d6 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -184,6 +184,10 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { if latestWorkflowRunStatus == "SUCCESS" { return false, nil } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) + } return true, nil } } From 0328f4c6586f7e8c9430846136816a1f091c54da Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 22 Jul 2024 20:38:23 -0300 Subject: [PATCH 17/46] use new prefix in the backend --- examples/tf_cloudbuild_workspace_simple/files/backend.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/tf_cloudbuild_workspace_simple/files/backend.tf b/examples/tf_cloudbuild_workspace_simple/files/backend.tf index dc5bf3de..69900115 100644 --- a/examples/tf_cloudbuild_workspace_simple/files/backend.tf +++ b/examples/tf_cloudbuild_workspace_simple/files/backend.tf @@ -17,6 +17,6 @@ terraform { backend "gcs" { bucket = "tf-state-prod" - prefix = "terraform/state" + prefix = "terraform/csr/state" } } From 09931b697fd1821ee1349614e9504deb0e0eed98 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 22 Jul 2024 20:49:20 -0300 Subject: [PATCH 18/46] update github-go version --- .../tf_cloudbuild_workspace_simple_github_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index e13da0d6..bce9502e 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -26,7 +26,7 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/git" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" cftutils "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" - "github.com/google/go-github/v62/github" + "github.com/google/go-github/v63/github" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-bootstrap/test/integration/utils" ) From 24dd93aa10e3789b3053182428b03c090a84de8f Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 25 Jul 2024 09:32:22 -0300 Subject: [PATCH 19/46] add example for cloud build with gitlab --- build/int.cloudbuild.yaml | 19 ++ .../README.md | 32 +++ .../apis.tf | 30 +++ .../files/.gitignore | 2 + .../files/backend.tf | 22 ++ .../files/main.tf | 48 ++++ .../files/variables.tf | 19 ++ .../main.tf | 45 ++++ .../outputs.tf | 49 ++++ .../scripts/push-to-repo.sh | 50 ++++ .../variables.tf | 36 +++ .../main.tf | 147 +++++++++++ .../outputs.tf | 49 ++++ .../variables.tf | 37 +++ ...cloudbuild_workspace_simple_gitlab_test.go | 238 ++++++++++++++++++ 15 files changed, 823 insertions(+) create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/README.md create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/apis.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/files/.gitignore create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/files/backend.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/files/variables.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/main.tf create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf create mode 100755 examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh create mode 100644 examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf create mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf create mode 100644 test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 9ba64899..34387ae3 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -192,6 +192,25 @@ steps: args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage teardown --verbose'] secretEnv: ['IM_GITHUB_PAT'] +- id: apply-tfworkspace-gitlab + waitFor: + - create-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage apply --verbose'] + secretEnv: ['IM_GITLAB_PAT'] +- id: verify-tfworkspace-gitlab + waitFor: + - apply-tfworkspace-gitlab + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage verify --verbose'] + secretEnv: ['IM_GITLAB_PAT'] +- id: teardown-tfworkspace-gitlab + waitFor: + - verify-tfworkspace-gitlab + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage teardown --verbose'] + secretEnv: ['IM_GITLAB_PAT'] + availableSecrets: secretManager: - versionName: $_IM_GITHUB_PAT_SECRET_ID/versions/latest diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/README.md b/examples/tf_cloudbuild_workspace_simple_gitlab/README.md new file mode 100644 index 00000000..28346c02 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/README.md @@ -0,0 +1,32 @@ +## Github Requirements for Cloud Build Connection + +When using a Cloud Build 2nd generation repository, a Cloud Build connection to your repository provider will be needed. +For GitLab connections you will need: + +- To create a [Personal Access Token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) on GitLab with `api` and `read_api` scopes. + +For more information on this topic refer to the [Connect to a GitLab host](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-host-gitlab) and [Connect to a GitLab repository](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-repo-gitlab) documentation + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | +| gitlab\_pat | GitLab access token. | `string` | n/a | yes | +| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | +| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| artifacts\_bucket | Bucket for storing TF plans | +| cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply | +| cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan | +| cloudbuild\_sa | SA used by Cloud Build triggers | +| logs\_bucket | Bucket for storing TF logs | +| project\_id | n/a | +| state\_bucket | Bucket for storing TF state | + + diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/apis.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/apis.tf new file mode 100644 index 00000000..38049631 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/apis.tf @@ -0,0 +1,30 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 15.0" + + project_id = var.project_id + disable_services_on_destroy = false + + activate_apis = [ + "iam.googleapis.com", + "secretmanager.googleapis.com", + "compute.googleapis.com", + "cloudbuild.googleapis.com", + ] +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/files/.gitignore b/examples/tf_cloudbuild_workspace_simple_gitlab/files/.gitignore new file mode 100644 index 00000000..a260b99a --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/files/.gitignore @@ -0,0 +1,2 @@ +# Local .terraform directories +**/.terraform* diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/files/backend.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/files/backend.tf new file mode 100644 index 00000000..feca7000 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/files/backend.tf @@ -0,0 +1,22 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + backend "gcs" { + bucket = "tf-state-prod" + prefix = "terraform/gitlab/state" + } +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf new file mode 100644 index 00000000..a0e783df --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "test-vpc-module" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + project_id = var.project_id + network_name = "my-custom-mode-network-gl" + mtu = 1460 + + subnets = [ + { + subnet_name = "subnet-01" + subnet_ip = "10.10.10.0/24" + subnet_region = "us-west1" + }, + { + subnet_name = "subnet-02" + subnet_ip = "10.10.20.0/24" + subnet_region = "us-west1" + subnet_private_access = "true" + subnet_flow_logs = "true" + }, + { + subnet_name = "subnet-03" + subnet_ip = "10.10.30.0/24" + subnet_region = "us-west1" + subnet_flow_logs = "true" + subnet_flow_logs_interval = "INTERVAL_10_MIN" + subnet_flow_logs_sampling = 0.7 + subnet_flow_logs_metadata = "INCLUDE_ALL_METADATA" + subnet_flow_logs_filter = "false" + } + ] +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/files/variables.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/files/variables.tf new file mode 100644 index 00000000..388f7b4a --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/files/variables.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the network in" +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf new file mode 100644 index 00000000..f8e6ddd7 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf @@ -0,0 +1,45 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "tf_workspace" { + source = "../../modules/tf_cloudbuild_workspace" + + project_id = module.enabled_google_apis.project_id + tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" + cloudbuildv2_repository_id = var.cloudbuildv2_repository_id + location = "us-central1" + trigger_location = "us-central1" + + # allow log/state buckets to be destroyed + buckets_force_destroy = true + cloudbuild_sa_roles = { (module.enabled_google_apis.project_id) = { + project_id = module.enabled_google_apis.project_id, + roles = ["roles/compute.networkAdmin"] + } + } + cloudbuild_env_vars = ["TF_VAR_project_id=${var.project_id}"] + + depends_on = [module.enabled_google_apis] +} + +module "bootstrap_github_repo" { + source = "terraform-google-modules/gcloud/google" + version = "~> 3.1" + upgrade = false + + create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" + create_cmd_body = "${var.gitlab_pat} ${var.repository_uri} ${path.module}/files" +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf new file mode 100644 index 00000000..d3b9b589 --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloudbuild_plan_trigger_id" { + description = "Trigger used for running TF plan" + value = module.tf_workspace.cloudbuild_plan_trigger_id +} + +output "cloudbuild_apply_trigger_id" { + description = "Trigger used for running TF apply" + value = module.tf_workspace.cloudbuild_apply_trigger_id +} + +output "cloudbuild_sa" { + description = "SA used by Cloud Build triggers" + value = module.tf_workspace.cloudbuild_sa +} + +output "state_bucket" { + description = "Bucket for storing TF state" + value = module.tf_workspace.state_bucket +} + +output "logs_bucket" { + description = "Bucket for storing TF logs" + value = module.tf_workspace.logs_bucket +} + +output "artifacts_bucket" { + description = "Bucket for storing TF plans" + value = module.tf_workspace.artifacts_bucket +} + +output "project_id" { + value = var.project_id +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh b/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh new file mode 100755 index 00000000..a0a12f7b --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -ex + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +GITLAB_TOKEN=$1 +REPO_URL=$2 +TF_CONFIG_PATH=$3 + + +# extract portion after https:// from URL +IFS="/"; mapfile -t -d / URL_PARTS < <(printf "%s" "$REPO_URL") +# construct the new authenticated URL +# AUTH_REPO_URL="https://${GITLAB_TOKEN}:@${URL_PARTS[2]}/${URL_PARTS[3]}/${URL_PARTS[4]}" +AUTH_REPO_URL="https://gitlab-bot:${GITLAB_TOKEN}@gitlab.com/${URL_PARTS[3]}/${URL_PARTS[4]}" + +tmp_dir=$(mktemp -d) +git clone "${AUTH_REPO_URL}" "${tmp_dir}" +cp -r "${TF_CONFIG_PATH}/." "${tmp_dir}" +pushd "${tmp_dir}" +git config init.defaultBranch main +git config user.email "terraform-robot@example.com" +git config user.name "TF Robot" +git checkout plan || git checkout -b plan +git add -A +git commit -m "init tf configs" +git push origin plan -f +sleep 60 +git checkout main || git checkout -b main +git merge plan +git push origin main -f +sleep 120 diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf new file mode 100644 index 00000000..2cdd2dec --- /dev/null +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + type = string +} + +variable "gitlab_pat" { + description = "GitLab access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf new file mode 100644 index 00000000..cceafe8b --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf @@ -0,0 +1,147 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + location = "us-central1" + # repo url of form "gitlab.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gh_repo_url_split = split("/", local.repoURLWithoutSuffix) + gl_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] + + gitlab_secret_version_id = google_secret_manager_secret_version.gitlab_api_secret_version.name + gitlab_secret_id = google_secret_manager_secret.gitlab_api_secret.id + + host_connection_name = "cb-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" + repo_connection_name = local.gl_name +} + +module "gitlab_workspace" { + source = "../../../examples/tf_cloudbuild_workspace_simple_gitlab" + + project_id = var.project_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id + gitlab_pat = var.gitlab_api_access_token + repository_uri = var.repository_uri +} + +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "gitlab_api_secret" { + project = var.project_id + secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" + + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { + secret = google_secret_manager_secret.gitlab_api_secret.id + secret_data = var.gitlab_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_read_api_secret" { + project = var.project_id + secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { + secret = google_secret_manager_secret.gitlab_read_api_secret.id + secret_data = var.gitlab_read_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_webhook_secret" { + project = var.project_id + secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { + secret = google_secret_manager_secret.gitlab_webhook_secret.id + secret_data = random_uuid.random_webhook_secret.result +} + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { + for_each = { + "api" = google_secret_manager_secret.gitlab_api_secret.id, + "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, + "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id + } + + project = var.project_id + secret_id = each.value + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "gitlab_resources_random_id" { + byte_length = 8 +} + +resource "random_uuid" "random_webhook_secret" { +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + location = local.location + + name = local.host_connection_name + + gitlab_config { + host_uri = null + authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name + } + read_authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name + } + webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name + } + + depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + location = local.location + name = local.repo_connection_name + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf new file mode 100644 index 00000000..fc2863be --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cloudbuild_plan_trigger_id" { + description = "Trigger used for running TF plan" + value = module.gitlab_workspace.cloudbuild_plan_trigger_id +} + +output "cloudbuild_apply_trigger_id" { + description = "Trigger used for running TF apply" + value = module.gitlab_workspace.cloudbuild_apply_trigger_id +} + +output "cloudbuild_sa" { + description = "SA used by Cloud Build triggers" + value = module.gitlab_workspace.cloudbuild_sa +} + +output "state_bucket" { + description = "Bucket for storing TF state" + value = module.gitlab_workspace.state_bucket +} + +output "logs_bucket" { + description = "Bucket for storing TF logs" + value = module.gitlab_workspace.logs_bucket +} + +output "artifacts_bucket" { + description = "Bucket for storing TF plans" + value = module.gitlab_workspace.artifacts_bucket +} + +output "project_id" { + value = var.project_id +} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf new file mode 100644 index 00000000..cfa7e530 --- /dev/null +++ b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "gitlab_api_access_token" { + description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." + type = string + sensitive = true +} + +variable "gitlab_read_api_access_token" { + description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go new file mode 100644 index 00000000..d88c274e --- /dev/null +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -0,0 +1,238 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tf_cloudbuild_workspace_simple_gitlab + +import ( + "fmt" + "path" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/git" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + cftutils "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/gruntwork-io/terratest/modules/logger" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-bootstrap/test/integration/utils" + "github.com/xanzy/go-gitlab" +) + +type GitLabClient struct { + t *testing.T + client *gitlab.Client + group string + namespace int + repo string + project *gitlab.Project +} + +func NewGitLabClient(t *testing.T, token, owner, repo string) *GitLabClient { + t.Helper() + client, err := gitlab.NewClient(token) + if err != nil { + t.Fatal(err.Error()) + } + return &GitLabClient{ + t: t, + client: client, + group: "infrastructure-manager", + namespace: 84326276, + repo: repo, + } +} + +func (gl *GitLabClient) ProjectName() string { + return fmt.Sprintf("%s/%s", gl.group, gl.repo) +} + +func (gl *GitLabClient) GetProject() *gitlab.Project { + proj, resp, err := gl.client.Projects.GetProject(gl.ProjectName(), nil) + if resp.StatusCode != 404 && err != nil { + gl.t.Fatalf("got status code %d, error %s", resp.StatusCode, err.Error()) + } + gl.project = proj + return proj +} + +func (gl *GitLabClient) CreateProject() { + opts := &gitlab.CreateProjectOptions{ + Name: gitlab.Ptr(gl.repo), + // ID of the the Infrastructure Manager group (gitlab.com/infrastructure-manager) + NamespaceID: gitlab.Ptr(gl.namespace), + // Required otherwise Cloud Build errors on creating the connection + InitializeWithReadme: gitlab.Ptr(true), + } + proj, _, err := gl.client.Projects.CreateProject(opts) + if err != nil { + gl.t.Fatal(err.Error()) + } + gl.project = proj +} + +func (gl *GitLabClient) AddFileToProject(file []byte) { + opts := &gitlab.CreateFileOptions{ + Branch: gitlab.Ptr("main"), + CommitMessage: gitlab.Ptr("Initial config commit"), + Content: gitlab.Ptr(string(file)), + } + _, _, err := gl.client.RepositoryFiles.CreateFile(gl.ProjectName(), "main.tf", opts) + if err != nil { + gl.t.Fatal(err.Error()) + } +} + +func (gl *GitLabClient) DeleteProject() { + resp, err := gl.client.Projects.DeleteProject(gl.ProjectName()) + if resp.StatusCode != 404 && err != nil { + gl.t.Errorf("error deleting project with status %s and error %s", resp.Status, err.Error()) + } + gl.project = nil +} + +func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { + repoName := fmt.Sprintf("cb-bp-test-%s", utils.GetRandomStringFromSetup(t)) + gitlabPAT := cftutils.ValFromEnv(t, "IM_GITLAB_PAT") + owner := "infrastructure-manager" + + client := NewGitLabClient(t, gitlabPAT, owner, repoName) + proj := client.GetProject() + if proj == nil { + client.CreateProject() + } + + vars := map[string]interface{}{ + "gitlab_api_access_token": gitlabPAT, + "gitlab_read_api_access_token": gitlabPAT, + "repository_uri": client.project.HTTPURLToRepo, + } + bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + t.Cleanup(func() { + // Delete the repository if we hit a failed state + if t.Failed() { + client.DeleteProject() + } + }) + + projectID := bpt.GetStringOutput("project_id") + + // cloud build triggers + triggers := []string{"plan", "apply"} + for _, trigger := range triggers { + triggerOP := utils.LastElement(bpt.GetStringOutput(fmt.Sprintf("cloudbuild_%s_trigger_id", trigger)), "/") + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, "us-central1", projectID) + assert.Equal(fmt.Sprintf("%s-%s", repoName, trigger), cloudBuildOP.Get("name").String(), "should have the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-%s@%s.iam.gserviceaccount.com", projectID, repoName, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") + } + + // artifacts, state and log buckets + logsBucket := utils.LastElement(bpt.GetStringOutput("logs_bucket"), "/") + stateBucket := utils.LastElement(bpt.GetStringOutput("state_bucket"), "/") + artifactsBucket := utils.LastElement(bpt.GetStringOutput("artifacts_bucket"), "/") + buckets := []string{artifactsBucket, logsBucket, stateBucket} + for _, bucket := range buckets { + // we can't use runf since we need to override --format json with --json for alpha storage + bucketOP := gcloud.Run(t, fmt.Sprintf("alpha storage ls --buckets gs://%s", bucket), gcloud.WithCommonArgs([]string{"--project", projectID, "--json"})).Array() + assert.Equalf(1, len(bucketOP), "%s bucket should exist", bucket) + assert.Truef(bucketOP[0].Get("metadata.iamConfiguration.uniformBucketLevelAccess.enabled").Bool(), "%s bucket uniformBucketLevelAccess should be enabled", bucket) + assert.Truef(bucketOP[0].Get("metadata.versioning.enabled").Bool(), "%s bucket versioning should be enabled", bucket) + } + + // CB SA IAM + cbSA := utils.LastElement(bpt.GetStringOutput("cloudbuild_sa"), "/") + iamOP := gcloud.Runf(t, "projects get-iam-policy %s --flatten bindings --filter bindings.members:'serviceAccount:%s'", projectID, cbSA).Array() + cftutils.GetFirstMatchResult(t, iamOP, "bindings.role", "roles/compute.networkAdmin") + + // e2e test for testing actuation through both plan/apply branches + applyTrigger := utils.LastElement(bpt.GetStringOutput("cloudbuild_apply_trigger_id"), "/") + planTrigger := utils.LastElement(bpt.GetStringOutput("cloudbuild_plan_trigger_id"), "/") + + // set up repo + tmpDir := t.TempDir() + git := git.NewCmdConfig(t, git.WithDir(tmpDir)) + gitRun := func(args ...string) { + _, err := git.RunCmdE(args...) + if err != nil { + t.Fatal(err) + } + } + + gitRun("clone", fmt.Sprintf("https://gitlab-bot:%s@gitlab.com/%s/%s", gitlabPAT, owner, repoName), tmpDir) + gitRun("config", "user.email", "tf-robot@example.com") + gitRun("config", "user.name", "TF Robot") + + // push commits on plan and main branches + // plan branch should trigger plan trigger + // main branch should trigger apply trigger + branches := []string{"plan", "main"} + for _, branch := range branches { + _, err := git.RunCmdE("checkout", branch) + if err != nil { + git.RunCmdE("checkout", "-b", branch) + } + git.CommitWithMsg(fmt.Sprintf("%s commit", branch), []string{"--allow-empty"}) + gitRun("push", "-u", fmt.Sprintf("https://gitlab-bot:%s@gitlab.com/%s/%s.git", gitlabPAT, owner, repoName), branch, "-f") + lastCommit := git.GetLatestCommit() + // filter builds triggered based on pushed commit sha + triggerLocation := "us-central1" + buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --project %s --region %s --limit 1 --sort-by ~createTime", lastCommit, projectID, triggerLocation) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Run(t, cmd, gcloud.WithLogger(logger.Discard)).Array() + if len(build) < 1 { + return true, nil + } + + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + t.Logf("%v", build) + return false, nil + } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with status %s", build[0].Get("id"), latestWorkflowRunStatus) + return false, nil + } + return true, nil + } + } + cftutils.Poll(t, pollCloudBuild(buildListCmd), 20, 15*time.Second) + build := gcloud.Runf(t, buildListCmd).Array()[0] + switch branch { + case "plan": + assert.Equal(planTrigger, build.Get("buildTriggerId").String(), "was triggered by plan trigger") + assert.Contains(build.Get("artifacts.objects.location").String(), path.Join(artifactsBucket, "plan"), "artifacts were uploaded to the correct location") + case "main": + assert.Equal(applyTrigger, build.Get("buildTriggerId").String(), "was triggered by apply trigger") + assert.Contains(build.Get("artifacts.objects.location").String(), path.Join(artifactsBucket, "apply"), "artifacts were uploaded to the correct location") + } + } + }) + + bpt.DefineTeardown(func(assert *assert.Assertions) { + // Guarantee clean up even if the normal gcloud/teardown run into errors + t.Cleanup(func() { + client.DeleteProject() + bpt.DefaultTeardown(assert) + }) + }) + + bpt.Test() +} From bdac16a875175952cccd6264b8305057a9e6c43d Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 31 Jul 2024 14:56:15 -0300 Subject: [PATCH 20/46] make enabling sourcerepo.googleapis.com conditional --- modules/cloudbuild/README.md | 3 ++- modules/cloudbuild/main.tf | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/cloudbuild/README.md b/modules/cloudbuild/README.md index febf99bd..a34a7c11 100644 --- a/modules/cloudbuild/README.md +++ b/modules/cloudbuild/README.md @@ -115,9 +115,10 @@ resources of this module: - Google Cloud Storage API `storage-api.googleapis.com` - Google Cloud Service Usage API: `serviceusage.googleapis.com` - Google Cloud Build API: `cloudbuild.googleapis.com` -- Google Cloud Source Repo API: `sourcerepo.googleapis.com` - Google Cloud KMS API: `cloudkms.googleapis.com` +If using Cloud Source Repositories, Google Cloud Source Repo API: `sourcerepo.googleapis.com` must also be enabled. + This API can be enabled in the default project created during establishing an organization. ## Contributing diff --git a/modules/cloudbuild/main.tf b/modules/cloudbuild/main.tf index a2b29b74..dd80fe69 100644 --- a/modules/cloudbuild/main.tf +++ b/modules/cloudbuild/main.tf @@ -17,12 +17,13 @@ locals { cloudbuild_project_id = var.project_id != "" ? var.project_id : format("%s-%s", var.project_prefix, "cloudbuild") gar_repo_name = var.gar_repo_name != "" ? var.gar_repo_name : format("%s-%s", var.project_prefix, "tf-runners") - cloudbuild_apis = ["cloudbuild.googleapis.com", "sourcerepo.googleapis.com", "cloudkms.googleapis.com", "artifactregistry.googleapis.com"] impersonation_enabled_count = var.sa_enable_impersonation == true ? 1 : 0 activate_apis = distinct(concat(var.activate_apis, local.cloudbuild_apis)) apply_branches_regex = "^(${join("|", var.terraform_apply_branches)})$" gar_name = split("/", google_artifact_registry_repository.tf-image-repo.name)[length(split("/", google_artifact_registry_repository.tf-image-repo.name)) - 1] impersonate_service_account = var.impersonate_service_account != "" ? "--impersonate-service-account=${var.impersonate_service_account}" : "" + basic_apis = ["cloudbuild.googleapis.com", "cloudkms.googleapis.com", "artifactregistry.googleapis.com"] + cloudbuild_apis = var.create_cloud_source_repos ? concat(["sourcerepo.googleapis.com"], local.basic_apis) : local.basic_apis } resource "random_id" "suffix" { From 3062dffa8b72d1137f237dc153e99a8e6f45bdd4 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 31 Jul 2024 15:19:11 -0300 Subject: [PATCH 21/46] use Source Repositories v2 on tf_cloudbuild_builder module --- build/int.cloudbuild.yaml | 38 +++ .../Dockerfile | 39 +++ .../README.md | 23 ++ .../apis.tf | 32 +++ .../main.tf | 40 +++ .../outputs.tf | 39 +++ .../scripts/push-to-repo.sh | 47 ++++ .../variables.tf | 35 +++ .../Dockerfile | 39 +++ .../README.md | 23 ++ .../apis.tf | 32 +++ .../main.tf | 40 +++ .../outputs.tf | 39 +++ .../scripts/push-to-repo.sh | 47 ++++ .../variables.tf | 35 +++ .../scripts/push-to-repo.sh | 1 - modules/tf_cloudbuild_builder/README.md | 3 +- modules/tf_cloudbuild_builder/cb.tf | 12 +- .../templates/workflow.yaml.tftpl | 20 +- modules/tf_cloudbuild_builder/variables.tf | 8 +- modules/tf_cloudbuild_builder/workflow.tf | 2 + modules/tf_cloudbuild_source/main.tf | 5 +- .../main.tf | 100 ++++++++ .../outputs.tf | 44 ++++ .../variables.tf | 31 +++ .../main.tf | 147 ++++++++++++ .../outputs.tf | 44 ++++ .../variables.tf | 37 +++ ...f_cloudbuild_builder_simple_github_test.go | 218 +++++++++++++++++ ...f_cloudbuild_builder_simple_gitlab_test.go | 227 ++++++++++++++++++ 30 files changed, 1429 insertions(+), 18 deletions(-) create mode 100644 examples/tf_cloudbuild_builder_simple_github/Dockerfile create mode 100644 examples/tf_cloudbuild_builder_simple_github/README.md create mode 100644 examples/tf_cloudbuild_builder_simple_github/apis.tf create mode 100644 examples/tf_cloudbuild_builder_simple_github/main.tf create mode 100644 examples/tf_cloudbuild_builder_simple_github/outputs.tf create mode 100755 examples/tf_cloudbuild_builder_simple_github/scripts/push-to-repo.sh create mode 100644 examples/tf_cloudbuild_builder_simple_github/variables.tf create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/Dockerfile create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/README.md create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/apis.tf create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/main.tf create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf create mode 100755 examples/tf_cloudbuild_builder_simple_gitlab/scripts/push-to-repo.sh create mode 100644 examples/tf_cloudbuild_builder_simple_gitlab/variables.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/main.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf create mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf create mode 100644 test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go create mode 100644 test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 34387ae3..be9663f3 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -119,6 +119,44 @@ steps: name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilder --stage teardown --verbose'] +- id: apply-tfbuilder-github + waitFor: + - create-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage apply --verbose'] + secretEnv: ['IM_GITHUB_PAT'] +- id: verify-tfbuilder-github + waitFor: + - apply-tfbuilder-github + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage verify --verbose'] + secretEnv: ['IM_GITHUB_PAT'] +- id: teardown-tfbuilder-github + waitFor: + - verify-tfbuilder-github + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitHub --stage teardown --verbose'] + secretEnv: ['IM_GITHUB_PAT'] + +- id: apply-tfbuilder-gitlab + waitFor: + - create-all + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage apply --verbose'] + secretEnv: ['IM_GITLAB_PAT'] +- id: verify-tfbuilder-gitlab + waitFor: + - apply-tfbuilder-gitlab + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage verify --verbose'] + secretEnv: ['IM_GITLAB_PAT'] +- id: teardown-tfbuilder-gitlab + waitFor: + - verify-tfbuilder-gitlab + name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' + args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildBuilderGitLab --stage teardown --verbose'] + secretEnv: ['IM_GITLAB_PAT'] + - id: apply-tfworkspace waitFor: - create-all diff --git a/examples/tf_cloudbuild_builder_simple_github/Dockerfile b/examples/tf_cloudbuild_builder_simple_github/Dockerfile new file mode 100644 index 00000000..39820b50 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-builders/gcloud-slim + +# Use ARG so that values can be overriden by user/cloudbuild +ARG TERRAFORM_VERSION=1.1.0 + +ENV ENV_TERRAFORM_VERSION=$TERRAFORM_VERSION + +RUN apt-get update && \ + /builder/google-cloud-sdk/bin/gcloud -q components install alpha beta terraform-tools && \ + apt-get -y install curl jq unzip git ca-certificates gnupg && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip --output terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS.sig --output terraform_SHA256SUMS.sig && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS --output terraform_SHA256SUMS && \ + curl https://keybase.io/hashicorp/pgp_keys.asc --output pgp_keys.asc && \ + gpg --import pgp_keys.asc && \ + gpg --verify terraform_SHA256SUMS.sig terraform_SHA256SUMS && \ + grep terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS | shasum --algorithm 256 --check && \ + unzip terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/terraform && \ + rm -f terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS && \ + apt-get --purge -y autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH=/builder/terraform/:$PATH +ENTRYPOINT ["terraform"] diff --git a/examples/tf_cloudbuild_builder_simple_github/README.md b/examples/tf_cloudbuild_builder_simple_github/README.md new file mode 100644 index 00000000..3464c925 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/README.md @@ -0,0 +1,23 @@ +## Overview + +This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Repositories V2 GitHub repo. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| artifact\_repo | GAR Repo created to store TF Cloud Builder images | +| cloudbuild\_trigger\_id | Trigger used for building new TF Builder | +| csr\_repo\_url | CSR repo for storing cloudbuilder Dockerfile | +| project\_id | n/a | +| scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | +| workflow\_id | Workflow ID for triggering new TF Builder build | + + diff --git a/examples/tf_cloudbuild_builder_simple_github/apis.tf b/examples/tf_cloudbuild_builder_simple_github/apis.tf new file mode 100644 index 00000000..9ca1a473 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/apis.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 15.0" + + project_id = var.project_id + disable_services_on_destroy = false + + activate_apis = [ + "iam.googleapis.com", + "compute.googleapis.com", + "workflows.googleapis.com", + "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", + "cloudscheduler.googleapis.com" + ] +} diff --git a/examples/tf_cloudbuild_builder_simple_github/main.tf b/examples/tf_cloudbuild_builder_simple_github/main.tf new file mode 100644 index 00000000..e9b23bea --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/main.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "cloudbuilder" { + source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder" + version = "~> 8.0" + + project_id = module.enabled_google_apis.project_id + dockerfile_repo_id = var.cloudbuildv2_repository_id + dockerfile_repo_type = "GITHUB" + use_cloudbuildv2_repository = true + trigger_location = "us-central1" + gar_repo_location = "us-central1" + + # allow logs bucket to be destroyed + cb_logs_bucket_force_destroy = true +} + +# Bootstrap GitHub with Dockerfile +module "bootstrap_github_repo" { + source = "terraform-google-modules/gcloud/google" + version = "~> 3.1" + upgrade = false + + create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" + create_cmd_body = "${var.github_pat} ${var.repository_uri} ${path.module}/Dockerfile" +} diff --git a/examples/tf_cloudbuild_builder_simple_github/outputs.tf b/examples/tf_cloudbuild_builder_simple_github/outputs.tf new file mode 100644 index 00000000..b062dcfb --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "artifact_repo" { + description = "GAR Repo created to store TF Cloud Builder images" + value = module.cloudbuilder.artifact_repo +} + +output "workflow_id" { + description = "Workflow ID for triggering new TF Builder build" + value = module.cloudbuilder.workflow_id +} + +output "scheduler_id" { + description = "Scheduler ID for periodically triggering TF Builder build Workflow" + value = module.cloudbuilder.scheduler_id +} + +output "cloudbuild_trigger_id" { + description = "Trigger used for building new TF Builder" + value = module.cloudbuilder.cloudbuild_trigger_id +} + +output "project_id" { + value = var.project_id +} diff --git a/examples/tf_cloudbuild_builder_simple_github/scripts/push-to-repo.sh b/examples/tf_cloudbuild_builder_simple_github/scripts/push-to-repo.sh new file mode 100755 index 00000000..b484f83d --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/scripts/push-to-repo.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -ex + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +GITHUB_TOKEN=$1 +REPO_URL=$2 +DOCKERFILE_PATH=$3 + +# extract portion after https:// from URL +IFS="/"; mapfile -t -d / URL_PARTS < <(printf "%s" "$REPO_URL") +# construct the new authenticated URL +AUTH_REPO_URL="https://${GITHUB_TOKEN}:@${URL_PARTS[2]}/${URL_PARTS[3]}/${URL_PARTS[4]}" + +# create temp dir, cleanup at exit +tmp_dir=$(mktemp -d) +# # shellcheck disable=SC2064 +# trap "rm -rf $tmp_dir" EXIT +git clone "${AUTH_REPO_URL}" "${tmp_dir}" +cp "${DOCKERFILE_PATH}" "${tmp_dir}" +pushd "${tmp_dir}" +git config credential.helper gcloud.sh +git config init.defaultBranch main +git config user.email "terraform-robot@example.com" +git config user.name "TF Robot" +git checkout main || git checkout -b main +git add Dockerfile +git commit -m "init tf dockerfile" +git push origin main -f diff --git a/examples/tf_cloudbuild_builder_simple_github/variables.tf b/examples/tf_cloudbuild_builder_simple_github/variables.tf new file mode 100644 index 00000000..4d571e76 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_github/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + default = "test-builder-workflow-4" +} + +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + type = string +} + +variable "github_pat" { + description = "GitHub personal access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/Dockerfile b/examples/tf_cloudbuild_builder_simple_gitlab/Dockerfile new file mode 100644 index 00000000..39820b50 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/Dockerfile @@ -0,0 +1,39 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM gcr.io/cloud-builders/gcloud-slim + +# Use ARG so that values can be overriden by user/cloudbuild +ARG TERRAFORM_VERSION=1.1.0 + +ENV ENV_TERRAFORM_VERSION=$TERRAFORM_VERSION + +RUN apt-get update && \ + /builder/google-cloud-sdk/bin/gcloud -q components install alpha beta terraform-tools && \ + apt-get -y install curl jq unzip git ca-certificates gnupg && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip --output terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS.sig --output terraform_SHA256SUMS.sig && \ + curl https://releases.hashicorp.com/terraform/${ENV_TERRAFORM_VERSION}/terraform_${ENV_TERRAFORM_VERSION}_SHA256SUMS --output terraform_SHA256SUMS && \ + curl https://keybase.io/hashicorp/pgp_keys.asc --output pgp_keys.asc && \ + gpg --import pgp_keys.asc && \ + gpg --verify terraform_SHA256SUMS.sig terraform_SHA256SUMS && \ + grep terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS | shasum --algorithm 256 --check && \ + unzip terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip -d /builder/terraform && \ + rm -f terraform_${ENV_TERRAFORM_VERSION}_linux_amd64.zip terraform_SHA256SUMS && \ + apt-get --purge -y autoremove && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +ENV PATH=/builder/terraform/:$PATH +ENTRYPOINT ["terraform"] diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/README.md b/examples/tf_cloudbuild_builder_simple_gitlab/README.md new file mode 100644 index 00000000..648aea35 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/README.md @@ -0,0 +1,23 @@ +## Overview + +This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Repositories V2 GitLab repo. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| artifact\_repo | GAR Repo created to store TF Cloud Builder images | +| cloudbuild\_trigger\_id | Trigger used for building new TF Builder | +| csr\_repo\_url | CSR repo for storing cloudbuilder Dockerfile | +| project\_id | n/a | +| scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | +| workflow\_id | Workflow ID for triggering new TF Builder build | + + diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/apis.tf b/examples/tf_cloudbuild_builder_simple_gitlab/apis.tf new file mode 100644 index 00000000..9ca1a473 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/apis.tf @@ -0,0 +1,32 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "enabled_google_apis" { + source = "terraform-google-modules/project-factory/google//modules/project_services" + version = "~> 15.0" + + project_id = var.project_id + disable_services_on_destroy = false + + activate_apis = [ + "iam.googleapis.com", + "compute.googleapis.com", + "workflows.googleapis.com", + "artifactregistry.googleapis.com", + "cloudbuild.googleapis.com", + "cloudscheduler.googleapis.com" + ] +} diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf new file mode 100644 index 00000000..9e5c79bf --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -0,0 +1,40 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "cloudbuilder" { + source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder" + version = "~> 8.0" + + project_id = module.enabled_google_apis.project_id + dockerfile_repo_id = var.cloudbuildv2_repository_id + dockerfile_repo_type = "UNKNOWN" // "GITLAB" is not one of the options available so we need to use "UNKNOWN" + use_cloudbuildv2_repository = true + trigger_location = "us-central1" + gar_repo_location = "us-central1" + + # allow logs bucket to be destroyed + cb_logs_bucket_force_destroy = true +} + +# Bootstrap GitLab with Dockerfile +module "bootstrap_gitlab_repo" { + source = "terraform-google-modules/gcloud/google" + version = "~> 3.1" + upgrade = false + + create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" + create_cmd_body = "${var.gitlab_pat} ${var.repository_uri} ${path.module}/Dockerfile" +} diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf new file mode 100644 index 00000000..b062dcfb --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf @@ -0,0 +1,39 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "artifact_repo" { + description = "GAR Repo created to store TF Cloud Builder images" + value = module.cloudbuilder.artifact_repo +} + +output "workflow_id" { + description = "Workflow ID for triggering new TF Builder build" + value = module.cloudbuilder.workflow_id +} + +output "scheduler_id" { + description = "Scheduler ID for periodically triggering TF Builder build Workflow" + value = module.cloudbuilder.scheduler_id +} + +output "cloudbuild_trigger_id" { + description = "Trigger used for building new TF Builder" + value = module.cloudbuilder.cloudbuild_trigger_id +} + +output "project_id" { + value = var.project_id +} diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/scripts/push-to-repo.sh b/examples/tf_cloudbuild_builder_simple_gitlab/scripts/push-to-repo.sh new file mode 100755 index 00000000..eeb9fd0f --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/scripts/push-to-repo.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +set -ex + +if [ "$#" -lt 3 ]; then + >&2 echo "Not all expected arguments set." + exit 1 +fi + +GITLAB_TOKEN=$1 +REPO_URL=$2 +DOCKERFILE_PATH=$3 + +# extract portion after https:// from URL +IFS="/"; mapfile -t -d / URL_PARTS < <(printf "%s" "$REPO_URL") +# construct the new authenticated URL +AUTH_REPO_URL="https://gitlab-bot:${GITLAB_TOKEN}@gitlab.com/${URL_PARTS[3]}/${URL_PARTS[4]}" + +# create temp dir, cleanup at exit +tmp_dir=$(mktemp -d) +# # shellcheck disable=SC2064 +# trap "rm -rf $tmp_dir" EXIT +git clone "${AUTH_REPO_URL}" "${tmp_dir}" +cp "${DOCKERFILE_PATH}" "${tmp_dir}" +pushd "${tmp_dir}" +git config credential.helper gcloud.sh +git config init.defaultBranch main +git config user.email "terraform-robot@example.com" +git config user.name "TF Robot" +git checkout main || git checkout -b main +git add Dockerfile +git commit -m "init tf dockerfile" +git push origin main -f diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf b/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf new file mode 100644 index 00000000..ef5a8089 --- /dev/null +++ b/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf @@ -0,0 +1,35 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + default = "test-builder-workflow-4" +} + +variable "cloudbuildv2_repository_id" { + description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + type = string +} + +variable "gitlab_pat" { + description = "GitLab personal access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh b/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh index a0a12f7b..5b7910a7 100755 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/scripts/push-to-repo.sh @@ -29,7 +29,6 @@ TF_CONFIG_PATH=$3 # extract portion after https:// from URL IFS="/"; mapfile -t -d / URL_PARTS < <(printf "%s" "$REPO_URL") # construct the new authenticated URL -# AUTH_REPO_URL="https://${GITLAB_TOKEN}:@${URL_PARTS[2]}/${URL_PARTS[3]}/${URL_PARTS[4]}" AUTH_REPO_URL="https://gitlab-bot:${GITLAB_TOKEN}@gitlab.com/${URL_PARTS[3]}/${URL_PARTS[4]}" tmp_dir=$(mktemp -d) diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 933c862e..8a91988a 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -93,13 +93,14 @@ resources of this module: ```hcl "iam.googleapis.com", "compute.googleapis.com", -"sourcerepo.googleapis.com", "workflows.googleapis.com", "artifactregistry.googleapis.com", "cloudbuild.googleapis.com", "cloudscheduler.googleapis.com" ``` +If using Cloud Source Repositories, `"sourcerepo.googleapis.com"` API must also be enabled. + ## Contributing Refer to the [contribution guidelines](../../CONTRIBUTING.md) for diff --git a/modules/tf_cloudbuild_builder/cb.tf b/modules/tf_cloudbuild_builder/cb.tf index 22006011..4ba31cd4 100644 --- a/modules/tf_cloudbuild_builder/cb.tf +++ b/modules/tf_cloudbuild_builder/cb.tf @@ -33,13 +33,11 @@ locals { img_tags_subst = [for tag in keys(local.tags_subst) : "${local.gar_uri}:v$${${tag}}"] # extract CSR project_id and repo name for granting reader access to CSR repo - is_source_repo = var.dockerfile_repo_type == "CLOUD_SOURCE_REPOSITORIES" + is_source_repo = var.dockerfile_repo_type == "CLOUD_SOURCE_REPOSITORIES" && !var.use_cloudbuildv2_repository # url of form https://source.developers.google.com/p//r/ source_repo_url_split = local.is_source_repo ? split("/", var.dockerfile_repo_uri) : [] source_repo_project = local.is_source_repo ? local.source_repo_url_split[4] : "" source_repo_name = local.is_source_repo ? local.source_repo_url_split[6] : "" - - use_repo_id = var.dockerfile_repo_id != "" } resource "google_cloudbuild_trigger" "build_trigger" { @@ -51,7 +49,7 @@ resource "google_cloudbuild_trigger" "build_trigger" { # repository accepts Generic Cloud Build 2nd Gen Repository dynamic "source_to_build" { - for_each = local.use_repo_id ? [1] : [] + for_each = var.use_cloudbuildv2_repository ? [1] : [] content { repository = var.dockerfile_repo_id ref = var.dockerfile_repo_ref @@ -60,7 +58,7 @@ resource "google_cloudbuild_trigger" "build_trigger" { } dynamic "source_to_build" { - for_each = local.use_repo_id ? [] : [1] + for_each = var.use_cloudbuildv2_repository ? [] : [1] content { uri = var.dockerfile_repo_uri ref = var.dockerfile_repo_ref @@ -98,6 +96,10 @@ resource "google_cloudbuild_trigger" "build_trigger" { substitutions = local.tags_subst service_account = local.cloudbuild_sa + lifecycle { + ignore_changes = [source_to_build.repo_type] // When using GitLab the value provided need to be "UNKNOWN" but when providing this value the API return empty. + } + depends_on = [ google_artifact_registry_repository_iam_member.push_images, google_project_iam_member.logs_writer diff --git a/modules/tf_cloudbuild_builder/templates/workflow.yaml.tftpl b/modules/tf_cloudbuild_builder/templates/workflow.yaml.tftpl index fed06115..62f4fe23 100644 --- a/modules/tf_cloudbuild_builder/templates/workflow.yaml.tftpl +++ b/modules/tf_cloudbuild_builder/templates/workflow.yaml.tftpl @@ -31,14 +31,18 @@ main: raise: $${e} - logAndExit: return: $${"Found latest runner for "+latestTFReleaseVersion+" as "+currentRunner.body.version+", skipping build."} + - triggerRunnerBuild: - call: googleapis.cloudbuild.v1.projects.triggers.run + call: http.post args: - projectId: ${project_id} - triggerId: ${trigger_id} + url: $${"https://cloudbuild.googleapis.com/v1/projects/${project_id}/locations/${location}/triggers/${trigger_hash}:run"} + auth: + type: OAuth2 body: - substitutions: - _TERRAFORM_FULL_VERSION: $${latestTFReleaseVersionNum} - _TERRAFORM_MINOR_VERSION: $${latestTFReleaseMinor} - _TERRAFORM_MAJOR_VERSION: $${latestTFReleaseMajor} - + projectId: ${project_id} + triggerId: ${trigger_id} + source: + substitutions: + _TERRAFORM_FULL_VERSION: $${latestTFReleaseVersionNum} + _TERRAFORM_MINOR_VERSION: $${latestTFReleaseMinor} + _TERRAFORM_MAJOR_VERSION: $${latestTFReleaseMajor} diff --git a/modules/tf_cloudbuild_builder/variables.tf b/modules/tf_cloudbuild_builder/variables.tf index 4250901b..fe5090a5 100644 --- a/modules/tf_cloudbuild_builder/variables.tf +++ b/modules/tf_cloudbuild_builder/variables.tf @@ -104,11 +104,17 @@ variable "dockerfile_repo_uri" { } variable "dockerfile_repo_id" { - description = "The repository id where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_uri`." + description = "The repository id where the Dockerfile for Terraform builder is stored. Use for Cloudbuild 2nd gen repository. Either specify this or the variable `dockerfile_repo_uri`." type = string default = "" } +variable "use_cloudbuildv2_repository" { + description = "Use Cloudbuild 2nd gen repository" + type = bool + default = false +} + variable "dockerfile_repo_ref" { description = "The branch or tag to use. Use refs/heads/branchname for branches or refs/tags/tagname for tags." type = string diff --git a/modules/tf_cloudbuild_builder/workflow.tf b/modules/tf_cloudbuild_builder/workflow.tf index d36cb547..e021d55e 100644 --- a/modules/tf_cloudbuild_builder/workflow.tf +++ b/modules/tf_cloudbuild_builder/workflow.tf @@ -21,6 +21,8 @@ locals { gar_repo_location = var.gar_repo_location gar_repo_name = var.gar_repo_name trigger_id = var.trigger_name + location = var.trigger_location + trigger_hash = google_cloudbuild_trigger.build_trigger.trigger_id }) workflow_sa = coalesce(var.workflow_sa, google_service_account.workflow_sa[0].email) } diff --git a/modules/tf_cloudbuild_source/main.tf b/modules/tf_cloudbuild_source/main.tf index 98d3c4db..35142b89 100644 --- a/modules/tf_cloudbuild_source/main.tf +++ b/modules/tf_cloudbuild_source/main.tf @@ -18,15 +18,16 @@ locals { cloudbuild_project_id = var.project_id != "" ? var.project_id : "tf-cloudbuild-" use_random_suffix = var.project_id == "" - cloudbuild_apis = [ + basic_apis = [ "cloudbuild.googleapis.com", - "sourcerepo.googleapis.com", "storage-api.googleapis.com", "iam.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbilling.googleapis.com" ] + cloudbuild_apis = length(var.cloud_source_repos) > 0 ? concat(["sourcerepo.googleapis.com"], local.basic_apis) : local.basic_apis + activate_apis = distinct(concat(var.activate_apis, local.cloudbuild_apis)) } diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf new file mode 100644 index 00000000..080d1e7f --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf @@ -0,0 +1,100 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + github_app_installation_id = "47590865" + location = "us-central1" + # GitHub repo url of form "github.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gh_repo_url_split = split("/", local.repoURLWithoutSuffix) + gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] + + github_secret_version_id = google_secret_manager_secret_version.github_token_secret_version.name + github_secret_id = google_secret_manager_secret.github_token_secret.id + + host_connection_name = "builder-gh-${random_id.resources_random_id.dec}-${var.project_id}" + repo_connection_name = local.gh_name +} + +module "cloudbuilder" { + source = "../../../examples/tf_cloudbuild_builder_simple_github" + + project_id = var.project_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id + github_pat = var.im_github_pat + repository_uri = var.repository_uri +} + +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "github_token_secret" { + project = var.project_id + secret_id = "builder-gh-${random_id.resources_random_id.dec}-${local.gh_name}" + + labels = { + label = "builder-gh-${random_id.resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "github_token_secret_version" { + secret = google_secret_manager_secret.github_token_secret.id + secret_data = var.im_github_pat +} + +resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { + project = var.project_id + secret_id = local.github_secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "resources_random_id" { + byte_length = 4 +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + location = local.location + + name = local.host_connection_name + + github_config { + app_installation_id = local.github_app_installation_id + authorizer_credential { + oauth_token_secret_version = local.github_secret_version_id + } + } +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + location = local.location + name = local.repo_connection_name + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf new file mode 100644 index 00000000..5932b2dc --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "artifact_repo" { + description = "GAR Repo created to store TF Cloud Builder images" + value = module.cloudbuilder.artifact_repo +} + +output "workflow_id" { + description = "Workflow ID for triggering new TF Builder build" + value = module.cloudbuilder.workflow_id +} + +output "scheduler_id" { + description = "Scheduler ID for periodically triggering TF Builder build Workflow" + value = module.cloudbuilder.scheduler_id +} + +output "cloudbuild_trigger_id" { + description = "Trigger used for building new TF Builder" + value = module.cloudbuilder.cloudbuild_trigger_id +} + +output "repository_id" { + description = "ID of the v2 repository" + value = google_cloudbuildv2_repository.repository_connection.id +} + +output "project_id" { + value = var.project_id +} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf new file mode 100644 index 00000000..56c6936b --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf @@ -0,0 +1,31 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "im_github_pat" { + description = "GitHub personal access token." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf new file mode 100644 index 00000000..8b7595a4 --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -0,0 +1,147 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + location = "us-central1" + # GitLab repo url of form "gitlab.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gl_repo_url_split = split("/", local.repoURLWithoutSuffix) + gl_name = local.gl_repo_url_split[length(local.gl_repo_url_split) - 1] + + gitlab_secret_version_id = google_secret_manager_secret_version.gitlab_api_secret_version.name + gitlab_secret_id = google_secret_manager_secret.gitlab_api_secret.id + + host_connection_name = "builder-gh-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" + repo_connection_name = local.gl_name +} + +module "cloudbuilder" { + source = "../../../examples/tf_cloudbuild_builder_simple_gitlab" + + project_id = var.project_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id + gitlab_pat = var.gitlab_api_access_token + repository_uri = var.repository_uri +} + +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "gitlab_api_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" + + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { + secret = google_secret_manager_secret.gitlab_api_secret.id + secret_data = var.gitlab_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_read_api_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { + secret = google_secret_manager_secret.gitlab_read_api_secret.id + secret_data = var.gitlab_read_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_webhook_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { + secret = google_secret_manager_secret.gitlab_webhook_secret.id + secret_data = random_uuid.random_webhook_secret.result +} + +data "google_project" "project" { + project_id = var.project_id +} + +resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { + for_each = { + "api" = google_secret_manager_secret.gitlab_api_secret.id, + "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, + "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id + } + + project = var.project_id + secret_id = each.value + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "gitlab_resources_random_id" { + byte_length = 8 +} + +resource "random_uuid" "random_webhook_secret" { +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + location = local.location + + name = local.host_connection_name + + gitlab_config { + host_uri = null + authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name + } + read_authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name + } + webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name + } + + depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + location = local.location + name = local.repo_connection_name + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf new file mode 100644 index 00000000..5932b2dc --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf @@ -0,0 +1,44 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "artifact_repo" { + description = "GAR Repo created to store TF Cloud Builder images" + value = module.cloudbuilder.artifact_repo +} + +output "workflow_id" { + description = "Workflow ID for triggering new TF Builder build" + value = module.cloudbuilder.workflow_id +} + +output "scheduler_id" { + description = "Scheduler ID for periodically triggering TF Builder build Workflow" + value = module.cloudbuilder.scheduler_id +} + +output "cloudbuild_trigger_id" { + description = "Trigger used for building new TF Builder" + value = module.cloudbuilder.cloudbuild_trigger_id +} + +output "repository_id" { + description = "ID of the v2 repository" + value = google_cloudbuildv2_repository.repository_connection.id +} + +output "project_id" { + value = var.project_id +} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf new file mode 100644 index 00000000..cfa7e530 --- /dev/null +++ b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf @@ -0,0 +1,37 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The ID of the project in which to provision resources." + type = string +} + +variable "gitlab_api_access_token" { + description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." + type = string + sensitive = true +} + +variable "gitlab_read_api_access_token" { + description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." + type = string + sensitive = true +} + +variable "repository_uri" { + description = "The URI of the repo where the Terraform configs are stored." + type = string +} diff --git a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go new file mode 100644 index 00000000..b4511ad7 --- /dev/null +++ b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go @@ -0,0 +1,218 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// define test package name +package tf_cloudbuild_builder_simple_github + +import ( + "context" + "fmt" + "log" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + cftutils "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/terraform-google-modules/terraform-google-bootstrap/test/integration/utils" + "github.com/google/go-github/v63/github" + "github.com/stretchr/testify/assert" +) + + +type GitHubClient struct { + t *testing.T + client *github.Client + owner string + repoName string + repository *github.Repository +} + +func NewGitHubClient(t *testing.T, token, owner, repo string) *GitHubClient { + t.Helper() + client := github.NewClient(nil).WithAuthToken(token) + return &GitHubClient{ + t: t, + client: client, + owner: owner, + repoName: repo, + } +} + +func (gh *GitHubClient) GetRepository(ctx context.Context) *github.Repository { + repo, resp, err := gh.client.Repositories.Get(ctx, gh.owner, gh.repoName) + if resp.StatusCode != 404 && err != nil { + gh.t.Fatal(err.Error()) + } + gh.repository = repo + return repo +} + +func (gh *GitHubClient) CreateRepository(ctx context.Context, org, repoName string) *github.Repository { + newRepo := &github.Repository{ + Name: github.String(repoName), + AutoInit: github.Bool(true), + Private: github.Bool(true), + Visibility: github.String("private"), + } + repo, _, err := gh.client.Repositories.Create(ctx, org, newRepo) + if err != nil { + gh.t.Fatal(err.Error()) + } + gh.repository = repo + return repo +} + +func (gh *GitHubClient) DeleteRepository(ctx context.Context) { + resp, err := gh.client.Repositories.Delete(ctx, gh.owner, *gh.repository.Name) + if resp.StatusCode != 404 && err != nil { + gh.t.Fatal(err.Error()) + } +} + +func TestTFCloudBuildBuilderGitHub(t *testing.T) { + ctx := context.Background() + githubPAT := cftutils.ValFromEnv(t, "IM_GITHUB_PAT") + owner := "im-goose" + repoName := fmt.Sprintf("b-gh-test-%s", utils.GetRandomStringFromSetup(t)) + client := NewGitHubClient(t, githubPAT, owner, repoName) + + repo := client.GetRepository(ctx) + if repo == nil { + client.CreateRepository(ctx, client.owner, client.repoName) + } + + // Testing the module's feature of appending the ".git" suffix if it's missing + repoURL := strings.TrimSuffix(client.repository.GetCloneURL(), ".git") + vars := map[string]interface{}{ + "im_github_pat": githubPAT, + "repository_uri": repoURL, + } + bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + t.Cleanup(func() { + // Delete the repository if we hit a failed state + if t.Failed() { + client.DeleteRepository(ctx) + } + }) + + projectID := bpt.GetStringOutput("project_id") + artifactRepo := bpt.GetStringOutput("artifact_repo") + artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + schedulerID := bpt.GetStringOutput("scheduler_id") + workflowID := bpt.GetStringOutput("workflow_id") + triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") + repositoryID := bpt.GetStringOutput("repository_id") + triggerId := strings.Split(triggerFQN, "/")[len(strings.Split(triggerFQN, "/"))-1] + + schedulerOP := gcloud.Runf(t, "scheduler jobs describe %s", schedulerID) + assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow", "has the correct name") + assert.Equal("0 8 * * *", schedulerOP.Get("schedule").String(), "has the correct schedule") + assert.Equal(fmt.Sprintf("https://workflowexecutions.googleapis.com/v1/%s/executions", workflowID), schedulerOP.Get("httpTarget.uri").String(), "has the correct target") + + workflowOP := gcloud.Runf(t, "workflows describe %s", workflowID) + assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") + + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) + log.Print(cloudBuildOP) + assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") + assert.Equal(repositoryID, cloudBuildOP.Get("sourceToBuild.repository").String(), "is connected to expected repo") + expectedSubsts := []string{"_TERRAFORM_FULL_VERSION", "_TERRAFORM_MAJOR_VERSION", "_TERRAFORM_MINOR_VERSION"} + gotSubsts := cloudBuildOP.Get("substitutions").Map() + imgs := cftutils.GetResultStrSlice(cloudBuildOP.Get("build.images").Array()) + for _, subst := range expectedSubsts { + _, found := gotSubsts[subst] + assert.Truef(found, "has %s substituion", found) + assert.Contains(imgs, fmt.Sprintf("%s:v${%s}", artifactRepoDockerRegistry, subst), "tags correct image") + } + + // e2e test + oldWorkflowRuns := gcloud.Runf(t, "workflows executions list %s", workflowID).Array() + // sometimes scheduler takes a minute to kick off workflow due to eventually consistent IAM + // continue to retrigger a few times until a new workflow is kicked off + triggerWorkflowFn := func() (bool, error) { + gcloud.Runf(t, "scheduler jobs run %s", schedulerID) + // workflow may take a few secs to trigger + time.Sleep(10 * time.Second) + newWorkflowRuns := gcloud.Runf(t, "workflows executions list %s", workflowID).Array() + // new workflow is kicked off since list has an additional workflow run + if len(newWorkflowRuns)-len(oldWorkflowRuns) > 0 { + return false, nil + } + return true, nil + } + cftutils.Poll(t, triggerWorkflowFn, 21, 10*time.Second) + + // poll until workflow complete + // workflow will poll CB LRO to completion + pollWorkflowFn := func() (bool, error) { + latestWorkflowRun := gcloud.Runf(t, "workflows executions list %s --sort-by=startTime --limit=1", workflowID).Array() + latestWorkflowRunStatus := latestWorkflowRun[0].Get("state").String() + if latestWorkflowRunStatus == "SUCCEEDED" { + return false, nil + } + // if failed it maybe due to eventually consistent IAM, retry trigger + if latestWorkflowRunStatus == "FAILED" { + triggerWorkflowFn() + } + return true, nil + } + cftutils.Poll(t, pollWorkflowFn, 100, 20*time.Second) + + // Poll the build to wait for it to run + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) + } + return true, nil + } + } + cftutils.Poll(t, pollCloudBuild(buildListCmd), 100, 10*time.Second) + + // Check if the images where created + images := gcloud.Runf(t, "artifacts docker images list %s --include-tags", artifactRepoDockerRegistry).Array() + assert.Equal(1, len(images), "only one image is in registry") + imageTags := strings.Split(images[0].Get("tags").String(), ",") + assert.Equal(3, len(imageTags), "image has three tags") + }) + + bpt.DefineTeardown(func(assert *assert.Assertions) { + // Guarantee clean up even if the normal gcloud/teardown run into errors + t.Cleanup(func() { + client.DeleteRepository(ctx) + bpt.DefaultTeardown(assert) + }) + }) + + bpt.Test() +} diff --git a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go new file mode 100644 index 00000000..11423715 --- /dev/null +++ b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go @@ -0,0 +1,227 @@ +// Copyright 2022 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// define test package name +package tf_cloudbuild_builder_simple_github + +import ( + "fmt" + "log" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + cftutils "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-bootstrap/test/integration/utils" + "github.com/xanzy/go-gitlab" +) + +type GitLabClient struct { + t *testing.T + client *gitlab.Client + group string + namespace int + repo string + project *gitlab.Project +} + +func NewGitLabClient(t *testing.T, token, owner, repo string) *GitLabClient { + t.Helper() + client, err := gitlab.NewClient(token) + if err != nil { + t.Fatal(err.Error()) + } + return &GitLabClient{ + t: t, + client: client, + group: "infrastructure-manager", + namespace: 84326276, + repo: repo, + } +} + +func (gl *GitLabClient) ProjectName() string { + return fmt.Sprintf("%s/%s", gl.group, gl.repo) +} + +func (gl *GitLabClient) GetProject() *gitlab.Project { + proj, resp, err := gl.client.Projects.GetProject(gl.ProjectName(), nil) + if resp.StatusCode != 404 && err != nil { + gl.t.Fatalf("got status code %d, error %s", resp.StatusCode, err.Error()) + } + gl.project = proj + return proj +} + +func (gl *GitLabClient) CreateProject() { + opts := &gitlab.CreateProjectOptions{ + Name: gitlab.Ptr(gl.repo), + // ID of the the Infrastructure Manager group (gitlab.com/infrastructure-manager) + NamespaceID: gitlab.Ptr(gl.namespace), + // Required otherwise Cloud Build errors on creating the connection + InitializeWithReadme: gitlab.Ptr(true), + } + proj, _, err := gl.client.Projects.CreateProject(opts) + if err != nil { + gl.t.Fatal(err.Error()) + } + gl.project = proj +} + +func (gl *GitLabClient) DeleteProject() { + resp, err := gl.client.Projects.DeleteProject(gl.ProjectName()) + if resp.StatusCode != 404 && err != nil { + gl.t.Errorf("error deleting project with status %s and error %s", resp.Status, err.Error()) + } + gl.project = nil +} + +func TestTFCloudBuildBuilderGitLab(t *testing.T) { + gitlabPAT := cftutils.ValFromEnv(t, "IM_GITLAB_PAT") + owner := "im-goose" + repoName := fmt.Sprintf("b-gl-test-%s", utils.GetRandomStringFromSetup(t)) + + client := NewGitLabClient(t, gitlabPAT, owner, repoName) + + proj := client.GetProject() + if proj == nil { + client.CreateProject() + } + + // Testing the module's feature of appending the ".git" suffix if it's missing + // repoURL := strings.TrimSuffix(client.repository.GetCloneURL(), ".git") + vars := map[string]interface{}{ + "gitlab_api_access_token": gitlabPAT, + "gitlab_read_api_access_token": gitlabPAT, + "repository_uri": client.project.HTTPURLToRepo, + } + bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) + + bpt.DefineVerify(func(assert *assert.Assertions) { + bpt.DefaultVerify(assert) + + t.Cleanup(func() { + // Delete the repository if we hit a failed state + if t.Failed() { + client.DeleteProject() + } + }) + + projectID := bpt.GetStringOutput("project_id") + artifactRepo := bpt.GetStringOutput("artifact_repo") + artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + schedulerID := bpt.GetStringOutput("scheduler_id") + workflowID := bpt.GetStringOutput("workflow_id") + triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") + repositoryID := bpt.GetStringOutput("repository_id") + triggerId := strings.Split(triggerFQN, "/")[len(strings.Split(triggerFQN, "/"))-1] + + schedulerOP := gcloud.Runf(t, "scheduler jobs describe %s", schedulerID) + assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow", "has the correct name") + assert.Equal("0 8 * * *", schedulerOP.Get("schedule").String(), "has the correct schedule") + assert.Equal(fmt.Sprintf("https://workflowexecutions.googleapis.com/v1/%s/executions", workflowID), schedulerOP.Get("httpTarget.uri").String(), "has the correct target") + + workflowOP := gcloud.Runf(t, "workflows describe %s", workflowID) + assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") + + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) + log.Print(cloudBuildOP) + assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") + assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") + assert.Equal(repositoryID, cloudBuildOP.Get("sourceToBuild.repository").String(), "is connected to expected repo") + expectedSubsts := []string{"_TERRAFORM_FULL_VERSION", "_TERRAFORM_MAJOR_VERSION", "_TERRAFORM_MINOR_VERSION"} + gotSubsts := cloudBuildOP.Get("substitutions").Map() + imgs := cftutils.GetResultStrSlice(cloudBuildOP.Get("build.images").Array()) + for _, subst := range expectedSubsts { + _, found := gotSubsts[subst] + assert.Truef(found, "has %s substituion", found) + assert.Contains(imgs, fmt.Sprintf("%s:v${%s}", artifactRepoDockerRegistry, subst), "tags correct image") + } + + // e2e test + oldWorkflowRuns := gcloud.Runf(t, "workflows executions list %s", workflowID).Array() + // sometimes scheduler takes a minute to kick off workflow due to eventually consistent IAM + // continue to retrigger a few times until a new workflow is kicked off + triggerWorkflowFn := func() (bool, error) { + gcloud.Runf(t, "scheduler jobs run %s", schedulerID) + // workflow may take a few secs to trigger + time.Sleep(10 * time.Second) + newWorkflowRuns := gcloud.Runf(t, "workflows executions list %s", workflowID).Array() + // new workflow is kicked off since list has an additional workflow run + if len(newWorkflowRuns)-len(oldWorkflowRuns) > 0 { + return false, nil + } + return true, nil + } + cftutils.Poll(t, triggerWorkflowFn, 21, 10*time.Second) + + // poll until workflow complete + // workflow will poll CB LRO to completion + pollWorkflowFn := func() (bool, error) { + latestWorkflowRun := gcloud.Runf(t, "workflows executions list %s --sort-by=startTime --limit=1", workflowID).Array() + latestWorkflowRunStatus := latestWorkflowRun[0].Get("state").String() + if latestWorkflowRunStatus == "SUCCEEDED" { + return false, nil + } + // if failed it maybe due to eventually consistent IAM, retry trigger + if latestWorkflowRunStatus == "FAILED" { + triggerWorkflowFn() + } + return true, nil + } + cftutils.Poll(t, pollWorkflowFn, 100, 20*time.Second) + + // Poll the build to wait for it to run + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) + } + return true, nil + } + } + cftutils.Poll(t, pollCloudBuild(buildListCmd), 100, 10*time.Second) + + // Check if the images where created + images := gcloud.Runf(t, "artifacts docker images list %s --include-tags", artifactRepoDockerRegistry).Array() + assert.Equal(1, len(images), "only one image is in registry") + imageTags := strings.Split(images[0].Get("tags").String(), ",") + assert.Equal(3, len(imageTags), "image has three tags") + }) + + bpt.DefineTeardown(func(assert *assert.Assertions) { + // Guarantee clean up even if the normal gcloud/teardown run into errors + t.Cleanup(func() { + client.DeleteProject() + bpt.DefaultTeardown(assert) + }) + }) + + bpt.Test() +} From 9dd944a1a619182bcfe45df544d1a4e00d0b251e Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 31 Jul 2024 15:22:12 -0300 Subject: [PATCH 22/46] fix lint issues --- examples/tf_cloudbuild_builder_simple_github/README.md | 4 +++- examples/tf_cloudbuild_builder_simple_gitlab/README.md | 4 +++- modules/tf_cloudbuild_builder/README.md | 3 ++- test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf | 2 +- test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf | 2 +- test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf | 2 +- 6 files changed, 11 insertions(+), 6 deletions(-) diff --git a/examples/tf_cloudbuild_builder_simple_github/README.md b/examples/tf_cloudbuild_builder_simple_github/README.md index 3464c925..5cdc26cd 100644 --- a/examples/tf_cloudbuild_builder_simple_github/README.md +++ b/examples/tf_cloudbuild_builder_simple_github/README.md @@ -7,7 +7,10 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | +| github\_pat | GitHub personal access token. | `string` | n/a | yes | | project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | +| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -15,7 +18,6 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. |------|-------------| | artifact\_repo | GAR Repo created to store TF Cloud Builder images | | cloudbuild\_trigger\_id | Trigger used for building new TF Builder | -| csr\_repo\_url | CSR repo for storing cloudbuilder Dockerfile | | project\_id | n/a | | scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | | workflow\_id | Workflow ID for triggering new TF Builder build | diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/README.md b/examples/tf_cloudbuild_builder_simple_gitlab/README.md index 648aea35..5c28d4c0 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/README.md +++ b/examples/tf_cloudbuild_builder_simple_gitlab/README.md @@ -7,7 +7,10 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | +| gitlab\_pat | GitLab personal access token. | `string` | n/a | yes | | project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | +| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -15,7 +18,6 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. |------|-------------| | artifact\_repo | GAR Repo created to store TF Cloud Builder images | | cloudbuild\_trigger\_id | Trigger used for building new TF Builder | -| csr\_repo\_url | CSR repo for storing cloudbuilder Dockerfile | | project\_id | n/a | | scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | | workflow\_id | Workflow ID for triggering new TF Builder build | diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 8a91988a..553d0682 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -38,7 +38,7 @@ This module creates: | cb\_logs\_bucket\_force\_destroy | When deleting the bucket for storing CloudBuild logs, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | | cloudbuild\_sa | Custom SA email to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | dockerfile\_repo\_dir | The directory inside the repo where the Dockerfile is located. If empty defaults to repo root. | `string` | `""` | no | -| dockerfile\_repo\_id | The repository id where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_uri`. | `string` | `""` | no | +| dockerfile\_repo\_id | The repository id where the Dockerfile for Terraform builder is stored. Use for Cloudbuild 2nd gen repository. Either specify this or the variable `dockerfile_repo_uri`. | `string` | `""` | no | | dockerfile\_repo\_ref | The branch or tag to use. Use refs/heads/branchname for branches or refs/tags/tagname for tags. | `string` | `"refs/heads/main"` | no | | dockerfile\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | | dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories. | `string` | `""` | no | @@ -50,6 +50,7 @@ This module creates: | terraform\_version | The initial terraform version in semantic version format. | `string` | `"1.1.0"` | no | | trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | `"global"` | no | | trigger\_name | Name of the Cloud Build trigger building the Terraform builder. | `string` | `"tf-cloud-builder-build"` | no | +| use\_cloudbuildv2\_repository | Use Cloudbuild 2nd gen repository | `bool` | `false` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | | workflow\_name | Name of the workflow managing builds. | `string` | `"terraform-runner-workflow"` | no | | workflow\_region | The region of the workflow. | `string` | `"us-central1"` | no | diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf index 5932b2dc..80528381 100644 --- a/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf +++ b/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf @@ -36,7 +36,7 @@ output "cloudbuild_trigger_id" { output "repository_id" { description = "ID of the v2 repository" - value = google_cloudbuildv2_repository.repository_connection.id + value = google_cloudbuildv2_repository.repository_connection.id } output "project_id" { diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf index 8b7595a4..06af3d96 100644 --- a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf +++ b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -15,7 +15,7 @@ */ locals { - location = "us-central1" + location = "us-central1" # GitLab repo url of form "gitlab.com/owner/name" repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf index 5932b2dc..80528381 100644 --- a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf +++ b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf @@ -36,7 +36,7 @@ output "cloudbuild_trigger_id" { output "repository_id" { description = "ID of the v2 repository" - value = google_cloudbuildv2_repository.repository_connection.id + value = google_cloudbuildv2_repository.repository_connection.id } output "project_id" { From fb07686d2df189a17c4498d66ac911791b968899 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 31 Jul 2024 15:33:49 -0300 Subject: [PATCH 23/46] fix lint issue --- modules/tf_cloudbuild_builder/cb.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/tf_cloudbuild_builder/cb.tf b/modules/tf_cloudbuild_builder/cb.tf index 4ba31cd4..541fa51a 100644 --- a/modules/tf_cloudbuild_builder/cb.tf +++ b/modules/tf_cloudbuild_builder/cb.tf @@ -97,7 +97,7 @@ resource "google_cloudbuild_trigger" "build_trigger" { service_account = local.cloudbuild_sa lifecycle { - ignore_changes = [source_to_build.repo_type] // When using GitLab the value provided need to be "UNKNOWN" but when providing this value the API return empty. + ignore_changes = [source_to_build[0].repo_type] // When using GitLab the value provided need to be "UNKNOWN" but when providing this value the API return empty. } depends_on = [ From 8a2cbf51713322ddf26480627ee1bedbe0bbda84 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 5 Aug 2024 16:23:39 -0300 Subject: [PATCH 24/46] use distinct names for resources in the examples --- examples/tf_cloudbuild_builder_simple_github/main.tf | 3 +++ examples/tf_cloudbuild_builder_simple_gitlab/main.tf | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/tf_cloudbuild_builder_simple_github/main.tf b/examples/tf_cloudbuild_builder_simple_github/main.tf index e9b23bea..25e4d6b3 100644 --- a/examples/tf_cloudbuild_builder_simple_github/main.tf +++ b/examples/tf_cloudbuild_builder_simple_github/main.tf @@ -24,6 +24,9 @@ module "cloudbuilder" { use_cloudbuildv2_repository = true trigger_location = "us-central1" gar_repo_location = "us-central1" + bucket_name = "tf-cloudbuilder-build-logs-${var.project_id}-gh" + gar_repo_name = "tf-runners-gh" + workflow_name = "terraform-runner-workflow-gh" # allow logs bucket to be destroyed cb_logs_bucket_force_destroy = true diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf index 9e5c79bf..05ca919b 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -24,6 +24,9 @@ module "cloudbuilder" { use_cloudbuildv2_repository = true trigger_location = "us-central1" gar_repo_location = "us-central1" + bucket_name = "tf-cloudbuilder-build-logs-${var.project_id}-gl" + gar_repo_name = "tf-runners-gl" + workflow_name = "terraform-runner-workflow-gl" # allow logs bucket to be destroyed cb_logs_bucket_force_destroy = true From c11095f0748c670159874017a1e5181e63c59e85 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 5 Aug 2024 18:11:39 -0300 Subject: [PATCH 25/46] use distinct names for trigger name in the examples --- examples/tf_cloudbuild_builder_simple_github/main.tf | 1 + examples/tf_cloudbuild_builder_simple_gitlab/main.tf | 1 + 2 files changed, 2 insertions(+) diff --git a/examples/tf_cloudbuild_builder_simple_github/main.tf b/examples/tf_cloudbuild_builder_simple_github/main.tf index 25e4d6b3..fd269a9b 100644 --- a/examples/tf_cloudbuild_builder_simple_github/main.tf +++ b/examples/tf_cloudbuild_builder_simple_github/main.tf @@ -27,6 +27,7 @@ module "cloudbuilder" { bucket_name = "tf-cloudbuilder-build-logs-${var.project_id}-gh" gar_repo_name = "tf-runners-gh" workflow_name = "terraform-runner-workflow-gh" + trigger_name = "tf-cloud-builder-build-gh" # allow logs bucket to be destroyed cb_logs_bucket_force_destroy = true diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf index 05ca919b..ec7368e0 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -27,6 +27,7 @@ module "cloudbuilder" { bucket_name = "tf-cloudbuilder-build-logs-${var.project_id}-gl" gar_repo_name = "tf-runners-gl" workflow_name = "terraform-runner-workflow-gl" + trigger_name = "tf-cloud-builder-build-gl" # allow logs bucket to be destroyed cb_logs_bucket_force_destroy = true From 77bfd2f6b2c17657d833d563126827c03e3b0633 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 5 Aug 2024 22:53:35 -0300 Subject: [PATCH 26/46] use distinct names for resources in the workspace examples --- examples/tf_cloudbuild_workspace_simple_github/main.tf | 3 +++ examples/tf_cloudbuild_workspace_simple_gitlab/main.tf | 3 +++ 2 files changed, 6 insertions(+) diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf index 52ca222d..cf829573 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -22,6 +22,9 @@ module "tf_workspace" { cloudbuildv2_repository_id = var.cloudbuildv2_repository_id location = "us-central1" trigger_location = "us-central1" + artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gh" + log_bucket_name = "tf-configs-build-logs-${var.project_id}-gh" + create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gh" # allow log/state buckets to be destroyed buckets_force_destroy = true diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf index f8e6ddd7..28c0101b 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf @@ -22,6 +22,9 @@ module "tf_workspace" { cloudbuildv2_repository_id = var.cloudbuildv2_repository_id location = "us-central1" trigger_location = "us-central1" + artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gl" + log_bucket_name = "tf-configs-build-logs-${var.project_id}-gl" + create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gl" # allow log/state buckets to be destroyed buckets_force_destroy = true From 4d5ae9e16df59cb19048aec4e8bd5e1f3ffe3668 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Mon, 5 Aug 2024 23:37:38 -0300 Subject: [PATCH 27/46] use distinct repository names in the workspace examples --- .../tf_cloudbuild_workspace_simple_github_test.go | 2 +- .../tf_cloudbuild_workspace_simple_gitlab_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index bce9502e..fd943cdd 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -84,7 +84,7 @@ func (gh *GitHubClient) DeleteRepository(ctx context.Context) { func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { ctx := context.Background() - repoName := fmt.Sprintf("cb-bp-test-%s", utils.GetRandomStringFromSetup(t)) + repoName := fmt.Sprintf("cb-bp-gh-%s", utils.GetRandomStringFromSetup(t)) githubPAT := cftutils.ValFromEnv(t, "IM_GITHUB_PAT") owner := "im-goose" client := NewGitHubClient(t, githubPAT, owner, repoName) diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go index d88c274e..ea3aeb83 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -103,7 +103,7 @@ func (gl *GitLabClient) DeleteProject() { } func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { - repoName := fmt.Sprintf("cb-bp-test-%s", utils.GetRandomStringFromSetup(t)) + repoName := fmt.Sprintf("cb-bp-gl-%s", utils.GetRandomStringFromSetup(t)) gitlabPAT := cftutils.ValFromEnv(t, "IM_GITLAB_PAT") owner := "infrastructure-manager" From 9e66641c25b9fe7f6ecca1c144632c1217ef8c93 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 01:33:25 -0300 Subject: [PATCH 28/46] fix tests --- .../tf_cloudbuild_builder_simple_test.go | 22 +++++++++++++++++++ ...f_cloudbuild_builder_simple_github_test.go | 6 ++--- ...f_cloudbuild_builder_simple_gitlab_test.go | 6 ++--- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go index eaaf4e05..8cd3381c 100644 --- a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go +++ b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go @@ -99,6 +99,28 @@ func TestTFCloudBuildBuilder(t *testing.T) { } utils.Poll(t, pollWorkflowFn, 100, 20*time.Second) + // Poll the build to wait for it to run + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + // poll build until complete + pollCloudBuild := func(cmd string) func() (bool, error) { + return func() (bool, error) { + build := gcloud.Runf(t, cmd).Array() + if len(build) < 1 { + return true, nil + } + latestWorkflowRunStatus := build[0].Get("status").String() + if latestWorkflowRunStatus == "SUCCESS" { + return false, nil + } + if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { + t.Logf("%v", build[0]) + t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) + } + return true, nil + } + } + utils.Poll(t, pollCloudBuild(buildListCmd), 100, 10*time.Second) + images := gcloud.Runf(t, "artifacts docker images list %s --include-tags", artifactRepoDockerRegistry).Array() assert.Equal(1, len(images), "only one image is in registry") imageTags := strings.Split(images[0].Get("tags").String(), ",") diff --git a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go index b4511ad7..8af83e83 100644 --- a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go @@ -122,17 +122,17 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { triggerId := strings.Split(triggerFQN, "/")[len(strings.Split(triggerFQN, "/"))-1] schedulerOP := gcloud.Runf(t, "scheduler jobs describe %s", schedulerID) - assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow", "has the correct name") + assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow-gh", "has the correct name") assert.Equal("0 8 * * *", schedulerOP.Get("schedule").String(), "has the correct schedule") assert.Equal(fmt.Sprintf("https://workflowexecutions.googleapis.com/v1/%s/executions", workflowID), schedulerOP.Get("httpTarget.uri").String(), "has the correct target") workflowOP := gcloud.Runf(t, "workflows describe %s", workflowID) - assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") + assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow-gh", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) log.Print(cloudBuildOP) - assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") + assert.Equal("tf-cloud-builder-build-gh", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") assert.Equal(repositoryID, cloudBuildOP.Get("sourceToBuild.repository").String(), "is connected to expected repo") expectedSubsts := []string{"_TERRAFORM_FULL_VERSION", "_TERRAFORM_MAJOR_VERSION", "_TERRAFORM_MINOR_VERSION"} diff --git a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go index 11423715..00e183b3 100644 --- a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go @@ -131,17 +131,17 @@ func TestTFCloudBuildBuilderGitLab(t *testing.T) { triggerId := strings.Split(triggerFQN, "/")[len(strings.Split(triggerFQN, "/"))-1] schedulerOP := gcloud.Runf(t, "scheduler jobs describe %s", schedulerID) - assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow", "has the correct name") + assert.Contains(schedulerOP.Get("name").String(), "trigger-terraform-runner-workflow-gl", "has the correct name") assert.Equal("0 8 * * *", schedulerOP.Get("schedule").String(), "has the correct schedule") assert.Equal(fmt.Sprintf("https://workflowexecutions.googleapis.com/v1/%s/executions", workflowID), schedulerOP.Get("httpTarget.uri").String(), "has the correct target") workflowOP := gcloud.Runf(t, "workflows describe %s", workflowID) - assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") + assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow-gl", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) log.Print(cloudBuildOP) - assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") + assert.Equal("tf-cloud-builder-build-gl", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") assert.Equal(repositoryID, cloudBuildOP.Get("sourceToBuild.repository").String(), "is connected to expected repo") expectedSubsts := []string{"_TERRAFORM_FULL_VERSION", "_TERRAFORM_MAJOR_VERSION", "_TERRAFORM_MINOR_VERSION"} From 25bc7fa102fac9be6bf36571e7190b8c88a49ac5 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 10:52:02 -0300 Subject: [PATCH 29/46] log build logs in case of failure --- .../tf_cloudbuild_workspace_simple_test.go | 3 +++ .../tf_cloudbuild_workspace_simple_github_test.go | 3 +++ .../tf_cloudbuild_workspace_simple_gitlab_test.go | 3 +++ 3 files changed, 9 insertions(+) diff --git a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go index 38bb9b5a..d8f86fe2 100644 --- a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go @@ -108,6 +108,9 @@ func TestTFCloudBuildWorkspaceSimple(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) + err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + t.Logf("err %v", err) + t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) } return true, nil diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index fd943cdd..32b3d8a6 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -186,6 +186,9 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) + err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + t.Logf("err %v", err) + t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) } return true, nil diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go index ea3aeb83..3ee97698 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -207,6 +207,9 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) + err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + t.Logf("err %v", err) + t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with status %s", build[0].Get("id"), latestWorkflowRunStatus) return false, nil } From c706c73441966cc653a09f061d46c2c537ffb20c Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 12:07:37 -0300 Subject: [PATCH 30/46] fix builds log command in tests --- .../tf_cloudbuild_workspace_simple_test.go | 2 +- .../tf_cloudbuild_workspace_simple_github_test.go | 2 +- .../tf_cloudbuild_workspace_simple_gitlab_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go index d8f86fe2..9034f81c 100644 --- a/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple/tf_cloudbuild_workspace_simple_test.go @@ -108,7 +108,7 @@ func TestTFCloudBuildWorkspaceSimple(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) - err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) t.Logf("err %v", err) t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index 32b3d8a6..6ace5857 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -186,7 +186,7 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) - err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), "us-central1")) t.Logf("err %v", err) t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go index 3ee97698..3d967184 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -207,7 +207,7 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) - err, logs := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s", build[0].Get("id"))) + logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), "us-central1")) t.Logf("err %v", err) t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with status %s", build[0].Get("id"), latestWorkflowRunStatus) From 0f341f66f72f45549ed8d3673da785b4330faea9 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 14:02:52 -0300 Subject: [PATCH 31/46] serialize execution of workspace integration tests --- build/int.cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index be9663f3..0fd8d42c 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -213,7 +213,7 @@ steps: - id: apply-tfworkspace-github waitFor: - - create-all + - teardown-tfworkspace name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage apply --verbose'] secretEnv: ['IM_GITHUB_PAT'] @@ -232,7 +232,7 @@ steps: - id: apply-tfworkspace-gitlab waitFor: - - create-all + - teardown-tfworkspace-github name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage apply --verbose'] secretEnv: ['IM_GITLAB_PAT'] From aa3ed4124b0ca12fea1edfd663c5740624cf5672 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 17:21:47 -0300 Subject: [PATCH 32/46] make subnetworks distinct --- .../tf_cloudbuild_workspace_simple_github/files/main.tf | 6 +++--- .../tf_cloudbuild_workspace_simple_gitlab/files/main.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf index 3fa9b64b..8c7cc7c6 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/files/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/files/main.tf @@ -23,19 +23,19 @@ module "test-vpc-module" { subnets = [ { - subnet_name = "subnet-01" + subnet_name = "subnet-gh-01" subnet_ip = "10.10.10.0/24" subnet_region = "us-west1" }, { - subnet_name = "subnet-02" + subnet_name = "subnet-gh-02" subnet_ip = "10.10.20.0/24" subnet_region = "us-west1" subnet_private_access = "true" subnet_flow_logs = "true" }, { - subnet_name = "subnet-03" + subnet_name = "subnet-gh-03" subnet_ip = "10.10.30.0/24" subnet_region = "us-west1" subnet_flow_logs = "true" diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf index a0e783df..06d63bfb 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/files/main.tf @@ -23,19 +23,19 @@ module "test-vpc-module" { subnets = [ { - subnet_name = "subnet-01" + subnet_name = "subnet-gl-01" subnet_ip = "10.10.10.0/24" subnet_region = "us-west1" }, { - subnet_name = "subnet-02" + subnet_name = "subnet-gl-02" subnet_ip = "10.10.20.0/24" subnet_region = "us-west1" subnet_private_access = "true" subnet_flow_logs = "true" }, { - subnet_name = "subnet-03" + subnet_name = "subnet-gl-03" subnet_ip = "10.10.30.0/24" subnet_region = "us-west1" subnet_flow_logs = "true" From cbbb5890281608aade35f506b26bb46e2e6c1ef1 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 22:01:26 -0300 Subject: [PATCH 33/46] create resources in a single region in the builder example --- examples/tf_cloudbuild_builder_simple/main.tf | 2 ++ .../tf_cloudbuild_builder_simple_test.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/tf_cloudbuild_builder_simple/main.tf b/examples/tf_cloudbuild_builder_simple/main.tf index 179b9939..e4bd9ef5 100644 --- a/examples/tf_cloudbuild_builder_simple/main.tf +++ b/examples/tf_cloudbuild_builder_simple/main.tf @@ -20,6 +20,8 @@ module "cloudbuilder" { project_id = module.enabled_google_apis.project_id dockerfile_repo_uri = google_sourcerepo_repository.builder_dockerfile_repo.url + trigger_location = "us-central1" + gar_repo_location = "us-central1" # allow logs bucket to be destroyed cb_logs_bucket_force_destroy = true } diff --git a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go index 8cd3381c..3eed8391 100644 --- a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go +++ b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go @@ -36,7 +36,7 @@ func TestTFCloudBuildBuilder(t *testing.T) { projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") - artifactRepoDockerRegistry := fmt.Sprintf("us-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) schedulerID := bpt.GetStringOutput("scheduler_id") workflowID := bpt.GetStringOutput("workflow_id") triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") @@ -52,7 +52,7 @@ func TestTFCloudBuildBuilder(t *testing.T) { assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s", triggerId, projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) log.Print(cloudBuildOP) assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") From 0cf9a631e488f050b206b2abdf103eaaf0c5e91a Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 22:05:38 -0300 Subject: [PATCH 34/46] make location required --- modules/tf_cloudbuild_builder/README.md | 4 ++-- modules/tf_cloudbuild_builder/variables.tf | 2 -- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 553d0682..82ad1697 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -43,12 +43,12 @@ This module creates: | dockerfile\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | | dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories. | `string` | `""` | no | | enable\_worker\_pool | Set to true to use a private worker pool in the Cloud Build Trigger. | `bool` | `false` | no | -| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | `"us"` | no | +| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | n/a | yes | | gar\_repo\_name | Name of the Google Artifact Repository where the Terraform builder images are stored. | `string` | `"tf-runners"` | no | | image\_name | Name of the image for the Terraform builder. | `string` | `"terraform"` | no | | project\_id | GCP project for Cloud Build trigger,workflow and scheduler. | `string` | n/a | yes | | terraform\_version | The initial terraform version in semantic version format. | `string` | `"1.1.0"` | no | -| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | `"global"` | no | +| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | n/a | yes | | trigger\_name | Name of the Cloud Build trigger building the Terraform builder. | `string` | `"tf-cloud-builder-build"` | no | | use\_cloudbuildv2\_repository | Use Cloudbuild 2nd gen repository | `bool` | `false` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | diff --git a/modules/tf_cloudbuild_builder/variables.tf b/modules/tf_cloudbuild_builder/variables.tf index fe5090a5..3ad2d5a7 100644 --- a/modules/tf_cloudbuild_builder/variables.tf +++ b/modules/tf_cloudbuild_builder/variables.tf @@ -70,7 +70,6 @@ variable "gar_repo_name" { variable "gar_repo_location" { description = "Name of the location for the Google Artifact Repository." type = string - default = "us" } variable "terraform_version" { @@ -94,7 +93,6 @@ variable "trigger_name" { variable "trigger_location" { description = "Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool." type = string - default = "global" } variable "dockerfile_repo_uri" { From 08688b83c0374a6548127ab8fcb8dae739792489 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 6 Aug 2024 22:40:28 -0300 Subject: [PATCH 35/46] run tests simultaneously --- build/int.cloudbuild.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 0fd8d42c..be9663f3 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -213,7 +213,7 @@ steps: - id: apply-tfworkspace-github waitFor: - - teardown-tfworkspace + - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitHub --stage apply --verbose'] secretEnv: ['IM_GITHUB_PAT'] @@ -232,7 +232,7 @@ steps: - id: apply-tfworkspace-gitlab waitFor: - - teardown-tfworkspace-github + - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestCloudBuildWorkspaceSimpleGitLab --stage apply --verbose'] secretEnv: ['IM_GITLAB_PAT'] From ae539accf31e292a4796e80c7e72544d561adc2a Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 7 Aug 2024 01:02:35 -0300 Subject: [PATCH 36/46] change test order to prevent collision --- build/int.cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index be9663f3..3eaed011 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -89,7 +89,7 @@ steps: - id: apply-tfsource waitFor: - - create-all + - destroy-cloudbuild-enabled name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage apply --verbose'] - id: verify-tfsource From 77845b3addf18d02542b7d0d07e8555dc7efcc7f Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 7 Aug 2024 02:53:01 -0300 Subject: [PATCH 37/46] wait build to reinstate ci-integration account as project creator --- build/int.cloudbuild.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 3eaed011..fa5e6d6f 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -89,7 +89,7 @@ steps: - id: apply-tfsource waitFor: - - destroy-cloudbuild-enabled + - destroy-simple-folder name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' args: ['/bin/bash', '-c', 'cft test run TestTFCloudBuildSourceSimple --stage apply --verbose'] - id: verify-tfsource From e668ca07800758249af752e8406c4f6ef47829b5 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 7 Aug 2024 19:17:54 -0300 Subject: [PATCH 38/46] move location to a variable in the tests --- .../tf_cloudbuild_builder_simple_test.go | 7 ++++--- .../tf_cloudbuild_builder_simple_github_test.go | 7 ++++--- .../tf_cloudbuild_builder_simple_gitlab_test.go | 7 ++++--- .../tf_cloudbuild_workspace_simple_github_test.go | 7 ++++--- .../tf_cloudbuild_workspace_simple_gitlab_test.go | 8 ++++---- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go index 3eed8391..1b836703 100644 --- a/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go +++ b/test/integration/tf_cloudbuild_builder_simple/tf_cloudbuild_builder_simple_test.go @@ -34,9 +34,10 @@ func TestTFCloudBuildBuilder(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { bpt.DefaultVerify(assert) + location := "us-central1" projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") - artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + artifactRepoDockerRegistry := fmt.Sprintf("%s-docker.pkg.dev/%s/%s/terraform", location, projectID, artifactRepo) schedulerID := bpt.GetStringOutput("scheduler_id") workflowID := bpt.GetStringOutput("workflow_id") triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") @@ -52,7 +53,7 @@ func TestTFCloudBuildBuilder(t *testing.T) { assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region %s", triggerId, projectID, location) log.Print(cloudBuildOP) assert.Equal("tf-cloud-builder-build", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") @@ -100,7 +101,7 @@ func TestTFCloudBuildBuilder(t *testing.T) { utils.Poll(t, pollWorkflowFn, 100, 20*time.Second) // Poll the build to wait for it to run - buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, location, projectID) // poll build until complete pollCloudBuild := func(cmd string) func() (bool, error) { return func() (bool, error) { diff --git a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go index 8af83e83..173d2f24 100644 --- a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go @@ -112,9 +112,10 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { } }) + location := "us-central1" projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") - artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + artifactRepoDockerRegistry := fmt.Sprintf("%s-docker.pkg.dev/%s/%s/terraform", location, projectID, artifactRepo) schedulerID := bpt.GetStringOutput("scheduler_id") workflowID := bpt.GetStringOutput("workflow_id") triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") @@ -130,7 +131,7 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow-gh", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region %s", triggerId, projectID, location) log.Print(cloudBuildOP) assert.Equal("tf-cloud-builder-build-gh", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") @@ -178,7 +179,7 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { cftutils.Poll(t, pollWorkflowFn, 100, 20*time.Second) // Poll the build to wait for it to run - buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, location, projectID) // poll build until complete pollCloudBuild := func(cmd string) func() (bool, error) { return func() (bool, error) { diff --git a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go index 00e183b3..18146542 100644 --- a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go @@ -121,9 +121,10 @@ func TestTFCloudBuildBuilderGitLab(t *testing.T) { } }) + location := "us-central1" projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") - artifactRepoDockerRegistry := fmt.Sprintf("us-central1-docker.pkg.dev/%s/%s/terraform", projectID, artifactRepo) + artifactRepoDockerRegistry := fmt.Sprintf("%s-docker.pkg.dev/%s/%s/terraform", location, projectID, artifactRepo) schedulerID := bpt.GetStringOutput("scheduler_id") workflowID := bpt.GetStringOutput("workflow_id") triggerFQN := bpt.GetStringOutput("cloudbuild_trigger_id") @@ -139,7 +140,7 @@ func TestTFCloudBuildBuilderGitLab(t *testing.T) { assert.Contains(workflowOP.Get("name").String(), "terraform-runner-workflow-gl", "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/terraform-runner-workflow-sa@%s.iam.gserviceaccount.com", projectID, projectID), workflowOP.Get("serviceAccount").String(), "uses expected SA") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region us-central1", triggerId, projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --project %s --region %s", triggerId, projectID, location) log.Print(cloudBuildOP) assert.Equal("tf-cloud-builder-build-gl", cloudBuildOP.Get("name").String(), "has the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-builder-sa@%s.iam.gserviceaccount.com", projectID, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") @@ -187,7 +188,7 @@ func TestTFCloudBuildBuilderGitLab(t *testing.T) { cftutils.Poll(t, pollWorkflowFn, 100, 20*time.Second) // Poll the build to wait for it to run - buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, "us-central1", projectID) + buildListCmd := fmt.Sprintf("builds list --filter buildTriggerId='%s' --region %s --project %s --limit 1 --sort-by ~createTime", triggerId, location, projectID) // poll build until complete pollCloudBuild := func(cmd string) func() (bool, error) { return func() (bool, error) { diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index 6ace5857..b88fc3c5 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -112,13 +112,14 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { } }) + location := "us-central1" projectID := bpt.GetStringOutput("project_id") // cloud build triggers triggers := []string{"plan", "apply"} for _, trigger := range triggers { triggerOP := utils.LastElement(bpt.GetStringOutput(fmt.Sprintf("cloudbuild_%s_trigger_id", trigger)), "/") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, "us-central1", projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, location, projectID) assert.Equal(fmt.Sprintf("%s-%s", repoName, trigger), cloudBuildOP.Get("name").String(), "should have the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-%s@%s.iam.gserviceaccount.com", projectID, repoName, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") } @@ -172,7 +173,7 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { gitRun("push", "--set-upstream", "origin", branch, "-f") lastCommit := git.GetLatestCommit() // filter builds triggered based on pushed commit sha - buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --region %s --project %s --limit 1 --sort-by ~createTime", lastCommit, "us-central1", projectID) + buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --region %s --project %s --limit 1 --sort-by ~createTime", lastCommit, location, projectID) // poll build until complete pollCloudBuild := func(cmd string) func() (bool, error) { return func() (bool, error) { @@ -186,7 +187,7 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) - logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), "us-central1")) + logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), location)) t.Logf("err %v", err) t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with failureInfo %s", build[0].Get("id"), build[0].Get("failureInfo")) diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go index 3d967184..bbc4ac2b 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -130,13 +130,14 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { } }) + location := "us-central1" projectID := bpt.GetStringOutput("project_id") // cloud build triggers triggers := []string{"plan", "apply"} for _, trigger := range triggers { triggerOP := utils.LastElement(bpt.GetStringOutput(fmt.Sprintf("cloudbuild_%s_trigger_id", trigger)), "/") - cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, "us-central1", projectID) + cloudBuildOP := gcloud.Runf(t, "beta builds triggers describe %s --region %s --project %s", triggerOP, location, projectID) assert.Equal(fmt.Sprintf("%s-%s", repoName, trigger), cloudBuildOP.Get("name").String(), "should have the correct name") assert.Equal(fmt.Sprintf("projects/%s/serviceAccounts/tf-cb-%s@%s.iam.gserviceaccount.com", projectID, repoName, projectID), cloudBuildOP.Get("serviceAccount").String(), "uses expected SA") } @@ -190,8 +191,7 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { gitRun("push", "-u", fmt.Sprintf("https://gitlab-bot:%s@gitlab.com/%s/%s.git", gitlabPAT, owner, repoName), branch, "-f") lastCommit := git.GetLatestCommit() // filter builds triggered based on pushed commit sha - triggerLocation := "us-central1" - buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --project %s --region %s --limit 1 --sort-by ~createTime", lastCommit, projectID, triggerLocation) + buildListCmd := fmt.Sprintf("builds list --filter substitutions.COMMIT_SHA='%s' --project %s --region %s --limit 1 --sort-by ~createTime", lastCommit, projectID, location) // poll build until complete pollCloudBuild := func(cmd string) func() (bool, error) { return func() (bool, error) { @@ -207,7 +207,7 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { } if latestWorkflowRunStatus == "TIMEOUT" || latestWorkflowRunStatus == "FAILURE" { t.Logf("%v", build[0]) - logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), "us-central1")) + logs, err := gcloud.RunCmdE(t, fmt.Sprintf("builds log %s --region %s", build[0].Get("id"), location)) t.Logf("err %v", err) t.Logf("logs %s", logs) t.Fatalf("workflow %s failed with status %s", build[0].Get("id"), latestWorkflowRunStatus) From 4c627ee7c7cf6d11ac87e5cdc406e70e35b36306 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 8 Aug 2024 20:02:27 -0300 Subject: [PATCH 39/46] merge fixture and example for the usage of Cloud Build repositories 2nd gen --- .../README.md | 19 ++- .../main.tf | 80 +++++++++- .../outputs.tf | 13 +- .../variables.tf | 8 +- .../README.md | 17 +- .../main.tf | 120 +++++++++++++- .../outputs.tf | 8 +- .../variables.tf | 14 +- .../README.md | 17 +- .../main.tf | 77 ++++++++- .../outputs.tf | 8 +- .../variables.tf | 7 +- .../README.md | 18 +-- .../main.tf | 120 +++++++++++++- .../outputs.tf | 8 +- .../variables.tf | 11 +- .../main.tf | 100 ------------ .../outputs.tf | 44 ------ .../variables.tf | 31 ---- .../main.tf | 147 ------------------ .../outputs.tf | 44 ------ .../variables.tf | 37 ----- .../main.tf | 100 ------------ .../outputs.tf | 49 ------ .../variables.tf | 31 ---- .../main.tf | 147 ------------------ .../outputs.tf | 49 ------ .../variables.tf | 37 ----- ...f_cloudbuild_builder_simple_github_test.go | 4 +- ...f_cloudbuild_builder_simple_gitlab_test.go | 2 +- ...cloudbuild_workspace_simple_github_test.go | 4 +- ...cloudbuild_workspace_simple_gitlab_test.go | 2 +- 32 files changed, 489 insertions(+), 884 deletions(-) delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/main.tf delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf delete mode 100644 test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf delete mode 100644 test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf diff --git a/examples/tf_cloudbuild_builder_simple_github/README.md b/examples/tf_cloudbuild_builder_simple_github/README.md index 5cdc26cd..4fafa298 100644 --- a/examples/tf_cloudbuild_builder_simple_github/README.md +++ b/examples/tf_cloudbuild_builder_simple_github/README.md @@ -1,16 +1,23 @@ ## Overview -This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Repositories V2 GitHub repo. +This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Cloud Build repositories (2nd gen) GitHub repository. + +For GitHub connections you will need: + +- Install the [Cloud Build App](/~https://github.com/apps/google-cloud-build) on Github. +- Create a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) on Github with [scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes) `repo` and `read:user` (or if app is installed in a organization use `read:org`). + +For more information on this topic refer to the Cloud Build repositories (2nd gen) documentation for +[Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen). ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | | github\_pat | GitHub personal access token. | `string` | n/a | yes | -| project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | -| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | +| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | +| repository\_uri | The URI of the GitHub repository where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -18,7 +25,9 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. |------|-------------| | artifact\_repo | GAR Repo created to store TF Cloud Builder images | | cloudbuild\_trigger\_id | Trigger used for building new TF Builder | -| project\_id | n/a | +| location | The location in which the resources were provisioned | +| project\_id | The ID of the project in which the resources were provisioned | +| repository\_id | ID of the Cloud Build repositories (2nd gen) repository | | scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | | workflow\_id | Workflow ID for triggering new TF Builder build | diff --git a/examples/tf_cloudbuild_builder_simple_github/main.tf b/examples/tf_cloudbuild_builder_simple_github/main.tf index fd269a9b..5b09c681 100644 --- a/examples/tf_cloudbuild_builder_simple_github/main.tf +++ b/examples/tf_cloudbuild_builder_simple_github/main.tf @@ -14,16 +14,39 @@ * limitations under the License. */ +locals { + // Found in the URL of your Cloud Build GitHub app configuration settings + // https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen#connecting_a_github_host_programmatically + github_app_installation_id = "47590865" + + # GitHub repo url of form "github.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gh_repo_url_split = split("/", local.repoURLWithoutSuffix) + gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] + + location = "us-central1" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "resources_random_id" { + byte_length = 4 +} + module "cloudbuilder" { source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder" version = "~> 8.0" project_id = module.enabled_google_apis.project_id - dockerfile_repo_id = var.cloudbuildv2_repository_id + dockerfile_repo_id = google_cloudbuildv2_repository.repository_connection.id dockerfile_repo_type = "GITHUB" use_cloudbuildv2_repository = true - trigger_location = "us-central1" - gar_repo_location = "us-central1" + trigger_location = local.location + gar_repo_location = local.location bucket_name = "tf-cloudbuilder-build-logs-${var.project_id}-gh" gar_repo_name = "tf-runners-gh" workflow_name = "terraform-runner-workflow-gh" @@ -33,6 +56,57 @@ module "cloudbuilder" { cb_logs_bucket_force_destroy = true } +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "github_token_secret" { + project = var.project_id + secret_id = "builder-gh-${random_id.resources_random_id.dec}-${local.gh_name}" + + labels = { + label = "builder-gh-${random_id.resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "github_token_secret_version" { + secret = google_secret_manager_secret.github_token_secret.id + secret_data = var.github_pat +} + +resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { + project = var.project_id + secret_id = google_secret_manager_secret.github_token_secret.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +// See https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + name = "builder-gh-${random_id.resources_random_id.dec}-${var.project_id}" + location = local.location + + github_config { + app_installation_id = local.github_app_installation_id + authorizer_credential { + oauth_token_secret_version = google_secret_manager_secret_version.github_token_secret_version.name + } + } +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + name = local.gh_name + location = local.location + + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} + # Bootstrap GitHub with Dockerfile module "bootstrap_github_repo" { source = "terraform-google-modules/gcloud/google" diff --git a/examples/tf_cloudbuild_builder_simple_github/outputs.tf b/examples/tf_cloudbuild_builder_simple_github/outputs.tf index b062dcfb..3cbb32f9 100644 --- a/examples/tf_cloudbuild_builder_simple_github/outputs.tf +++ b/examples/tf_cloudbuild_builder_simple_github/outputs.tf @@ -34,6 +34,17 @@ output "cloudbuild_trigger_id" { value = module.cloudbuilder.cloudbuild_trigger_id } +output "repository_id" { + description = "ID of the Cloud Build repositories (2nd gen) repository" + value = google_cloudbuildv2_repository.repository_connection.id +} + output "project_id" { - value = var.project_id + description = "The ID of the project in which the resources were provisioned" + value = var.project_id +} + +output "location" { + description = "The location in which the resources were provisioned" + value = local.location } diff --git a/examples/tf_cloudbuild_builder_simple_github/variables.tf b/examples/tf_cloudbuild_builder_simple_github/variables.tf index 4d571e76..517c347a 100644 --- a/examples/tf_cloudbuild_builder_simple_github/variables.tf +++ b/examples/tf_cloudbuild_builder_simple_github/variables.tf @@ -15,11 +15,7 @@ */ variable "project_id" { - default = "test-builder-workflow-4" -} - -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." + description = "The ID of the project in which to provision resources." type = string } @@ -30,6 +26,6 @@ variable "github_pat" { } variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." + description = "The URI of the GitHub repository where the Terraform configs are stored." type = string } diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/README.md b/examples/tf_cloudbuild_builder_simple_gitlab/README.md index 5c28d4c0..da88c01e 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/README.md +++ b/examples/tf_cloudbuild_builder_simple_gitlab/README.md @@ -1,16 +1,20 @@ ## Overview -This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Repositories V2 GitLab repo. +This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../../modules/tf_cloudbuild_builder/) module with a Cloud Build repositories (2nd gen) GitLab repository. + +For more information on this topic refer to the Cloud Build repositories (2nd gen) documentation: +- [Connect to a GitLab host](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-host-gitlab) +- [Connect to a GitLab repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | -| gitlab\_pat | GitLab personal access token. | `string` | n/a | yes | -| project\_id | n/a | `string` | `"test-builder-workflow-4"` | no | -| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | +| gitlab\_api\_access\_token | GitLab personal access token with api scope. If provided, creates a secret within Secret Manager. | `string` | n/a | yes | +| gitlab\_read\_api\_access\_token | GitLab personal access token with read\_api scope. If provided, creates a secret within Secret Manager. | `string` | n/a | yes | +| project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | +| repository\_uri | The URI of the GitLab repository where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -18,7 +22,8 @@ This example demonstrates the simplest usage of the [tf_cloudbuild_builder](../. |------|-------------| | artifact\_repo | GAR Repo created to store TF Cloud Builder images | | cloudbuild\_trigger\_id | Trigger used for building new TF Builder | -| project\_id | n/a | +| location | The location in which the resources were provisioned | +| project\_id | The ID of the project in which the resources were provisioned | | scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | | workflow\_id | Workflow ID for triggering new TF Builder build | diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf index ec7368e0..186d49f7 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -14,12 +14,34 @@ * limitations under the License. */ +locals { + # GitLab repo url of form "gitlab.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gl_repo_url_split = split("/", local.repoURLWithoutSuffix) + gl_name = local.gl_repo_url_split[length(local.gl_repo_url_split) - 1] + + location = "us-central1" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "gitlab_resources_random_id" { + byte_length = 8 +} + +resource "random_uuid" "random_webhook_secret" { +} + module "cloudbuilder" { source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder" version = "~> 8.0" project_id = module.enabled_google_apis.project_id - dockerfile_repo_id = var.cloudbuildv2_repository_id + dockerfile_repo_id = google_cloudbuildv2_repository.repository_connection.id dockerfile_repo_type = "UNKNOWN" // "GITLAB" is not one of the options available so we need to use "UNKNOWN" use_cloudbuildv2_repository = true trigger_location = "us-central1" @@ -33,6 +55,100 @@ module "cloudbuilder" { cb_logs_bucket_force_destroy = true } +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "gitlab_api_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" + + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { + secret = google_secret_manager_secret.gitlab_api_secret.id + secret_data = var.gitlab_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_read_api_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { + secret = google_secret_manager_secret.gitlab_read_api_secret.id + secret_data = var.gitlab_read_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_webhook_secret" { + project = var.project_id + secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" + labels = { + label = "b-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { + secret = google_secret_manager_secret.gitlab_webhook_secret.id + secret_data = random_uuid.random_webhook_secret.result +} + +resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { + for_each = { + "api" = google_secret_manager_secret.gitlab_api_secret.id, + "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, + "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id + } + + project = var.project_id + secret_id = each.value + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + name = "builder-gl-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" + location = local.location + + gitlab_config { + host_uri = null + authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name + } + read_authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name + } + webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name + } + + depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + name = local.gl_name + location = local.location + + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} + # Bootstrap GitLab with Dockerfile module "bootstrap_gitlab_repo" { source = "terraform-google-modules/gcloud/google" @@ -40,5 +156,5 @@ module "bootstrap_gitlab_repo" { upgrade = false create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" - create_cmd_body = "${var.gitlab_pat} ${var.repository_uri} ${path.module}/Dockerfile" + create_cmd_body = "${var.gitlab_api_access_token} ${var.repository_uri} ${path.module}/Dockerfile" } diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf index b062dcfb..38dbd34a 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf @@ -35,5 +35,11 @@ output "cloudbuild_trigger_id" { } output "project_id" { - value = var.project_id + description = "The ID of the project in which the resources were provisioned" + value = var.project_id +} + +output "location" { + description = "The location in which the resources were provisioned" + value = local.location } diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf b/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf index ef5a8089..d3516954 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/variables.tf @@ -15,21 +15,23 @@ */ variable "project_id" { - default = "test-builder-workflow-4" + description = "The ID of the project in which to provision resources." + type = string } -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." +variable "gitlab_api_access_token" { + description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." type = string + sensitive = true } -variable "gitlab_pat" { - description = "GitLab personal access token." +variable "gitlab_read_api_access_token" { + description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." type = string sensitive = true } variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." + description = "The URI of the GitLab repository where the Terraform configs are stored." type = string } diff --git a/examples/tf_cloudbuild_workspace_simple_github/README.md b/examples/tf_cloudbuild_workspace_simple_github/README.md index dfa8eeba..d53c1846 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/README.md +++ b/examples/tf_cloudbuild_workspace_simple_github/README.md @@ -1,21 +1,23 @@ ## Github Requirements for Cloud Build Connection -When using a Cloud Build 2nd generation repository, a Cloud Build connection to your repository provider will be needed. For Github connections you will need: +When using a Cloud Build repositories (2nd gen) GitHub repository, a Cloud Build connection to your repository provider will be created. -- [Install Cloud Build App on Github](/~https://github.com/apps/google-cloud-build). -- [Create Personal Access Token on Github with `repo` and `read:user` (or if app is installed in org use `read:org`)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token). +For GitHub connections you will need: -For more information on this topic refer to the ["Connect with Github Documentation"](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) +- Install the [Cloud Build App](/~https://github.com/apps/google-cloud-build) on Github. +- Create a [Personal Access Token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) on Github with [scopes](https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/scopes-for-oauth-apps#available-scopes) `repo` and `read:user` (or if app is installed in a organization use `read:org`). + +For more information on this topic refer to the Cloud Build repositories (2nd gen) documentation for +[Connect to a GitHub repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen). ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | | github\_pat | GitHub personal access token. | `string` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | -| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | +| repository\_uri | The URI of the GitHub repository where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -25,8 +27,9 @@ For more information on this topic refer to the ["Connect with Github Documentat | cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply | | cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan | | cloudbuild\_sa | SA used by Cloud Build triggers | +| location | The location in which the resources were provisioned | | logs\_bucket | Bucket for storing TF logs | -| project\_id | n/a | +| project\_id | The ID of the project in which the resources were provisioned | | state\_bucket | Bucket for storing TF state | diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf index cf829573..ac3f36b8 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -14,12 +14,35 @@ * limitations under the License. */ +locals { + // Found in the URL of your Cloud Build GitHub app configuration settings + // https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen#connecting_a_github_host_programmatically + github_app_installation_id = "47590865" + + # GitHub repo url of form "github.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gh_repo_url_split = split("/", local.repoURLWithoutSuffix) + gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] + + location = "us-central1" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "resources_random_id" { + byte_length = 4 +} + module "tf_workspace" { source = "../../modules/tf_cloudbuild_workspace" project_id = module.enabled_google_apis.project_id tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" - cloudbuildv2_repository_id = var.cloudbuildv2_repository_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id location = "us-central1" trigger_location = "us-central1" artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gh" @@ -38,6 +61,58 @@ module "tf_workspace" { depends_on = [module.enabled_google_apis] } + +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "github_token_secret" { + project = var.project_id + secret_id = "cb-github-${random_id.resources_random_id.dec}-${local.gh_name}" + + labels = { + label = "cb-${random_id.resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "github_token_secret_version" { + secret = google_secret_manager_secret.github_token_secret.id + secret_data = var.github_pat +} + +resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { + project = var.project_id + secret_id = google_secret_manager_secret.github_token_secret.id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +// See https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + name = "cb-${random_id.resources_random_id.dec}-${var.project_id}" + location = local.location + + github_config { + app_installation_id = local.github_app_installation_id + authorizer_credential { + oauth_token_secret_version = google_secret_manager_secret_version.github_token_secret_version.name + } + } +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + name = local.gh_name + location = local.location + + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} + module "bootstrap_github_repo" { source = "terraform-google-modules/gcloud/google" version = "~> 3.1" diff --git a/examples/tf_cloudbuild_workspace_simple_github/outputs.tf b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf index d3b9b589..70238553 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/outputs.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/outputs.tf @@ -45,5 +45,11 @@ output "artifacts_bucket" { } output "project_id" { - value = var.project_id + description = "The ID of the project in which the resources were provisioned" + value = var.project_id +} + +output "location" { + description = "The location in which the resources were provisioned" + value = local.location } diff --git a/examples/tf_cloudbuild_workspace_simple_github/variables.tf b/examples/tf_cloudbuild_workspace_simple_github/variables.tf index 649eec55..517c347a 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/variables.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/variables.tf @@ -19,11 +19,6 @@ variable "project_id" { type = string } -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." - type = string -} - variable "github_pat" { description = "GitHub personal access token." type = string @@ -31,6 +26,6 @@ variable "github_pat" { } variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." + description = "The URI of the GitHub repository where the Terraform configs are stored." type = string } diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/README.md b/examples/tf_cloudbuild_workspace_simple_gitlab/README.md index 28346c02..b63182ea 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/README.md +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/README.md @@ -1,21 +1,20 @@ ## Github Requirements for Cloud Build Connection -When using a Cloud Build 2nd generation repository, a Cloud Build connection to your repository provider will be needed. -For GitLab connections you will need: +When using a Cloud Build repositories (2nd gen) GitLab repository, a Cloud Build connection to your repository provider will be needed. -- To create a [Personal Access Token](https://docs.gitlab.com/ee/user/profile/personal_access_tokens.html) on GitLab with `api` and `read_api` scopes. - -For more information on this topic refer to the [Connect to a GitLab host](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-host-gitlab) and [Connect to a GitLab repository](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-repo-gitlab) documentation +For more information on this topic refer to the Cloud Build repositories (2nd gen) documentation: +- [Connect to a GitLab host](https://cloud.google.com/build/docs/automating-builds/gitlab/connect-host-gitlab) +- [Connect to a GitLab repository](https://cloud.google.com/build/docs/automating-builds/github/connect-repo-github?generation=2nd-gen) ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | n/a | yes | -| gitlab\_pat | GitLab access token. | `string` | n/a | yes | +| gitlab\_api\_access\_token | GitLab personal access token with api scope. If provided, creates a secret within Secret Manager. | `string` | n/a | yes | +| gitlab\_read\_api\_access\_token | GitLab personal access token with read\_api scope. If provided, creates a secret within Secret Manager. | `string` | n/a | yes | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | -| repository\_uri | The URI of the repo where the Terraform configs are stored. | `string` | n/a | yes | +| repository\_uri | The URI of the GitLab repository where the Terraform configs are stored. | `string` | n/a | yes | ## Outputs @@ -25,8 +24,9 @@ For more information on this topic refer to the [Connect to a GitLab host](https | cloudbuild\_apply\_trigger\_id | Trigger used for running TF apply | | cloudbuild\_plan\_trigger\_id | Trigger used for running TF plan | | cloudbuild\_sa | SA used by Cloud Build triggers | +| location | The location in which the resources were provisioned | | logs\_bucket | Bucket for storing TF logs | -| project\_id | n/a | +| project\_id | The ID of the project in which the resources were provisioned | | state\_bucket | Bucket for storing TF state | diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf index 28c0101b..3221dfd1 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf @@ -14,12 +14,34 @@ * limitations under the License. */ +locals { + # GitLab repo url of form "gitlab.com/owner/name" + repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" + repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") + gl_repo_url_split = split("/", local.repoURLWithoutSuffix) + gl_name = local.gl_repo_url_split[length(local.gl_repo_url_split) - 1] + + location = "us-central1" +} + +data "google_project" "project" { + project_id = var.project_id +} + +// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. +resource "random_id" "gitlab_resources_random_id" { + byte_length = 8 +} + +resource "random_uuid" "random_webhook_secret" { +} + module "tf_workspace" { source = "../../modules/tf_cloudbuild_workspace" project_id = module.enabled_google_apis.project_id tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" - cloudbuildv2_repository_id = var.cloudbuildv2_repository_id + cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id location = "us-central1" trigger_location = "us-central1" artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gl" @@ -38,11 +60,105 @@ module "tf_workspace" { depends_on = [module.enabled_google_apis] } +// Create a secret containing the personal access token and grant permissions to the Service Agent. +resource "google_secret_manager_secret" "gitlab_api_secret" { + project = var.project_id + secret_id = "cb-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" + + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + + replication { + auto {} + } +} + +// Personal access token from VCS. +resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { + secret = google_secret_manager_secret.gitlab_api_secret.id + secret_data = var.gitlab_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_read_api_secret" { + project = var.project_id + secret_id = "cb-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { + secret = google_secret_manager_secret.gitlab_read_api_secret.id + secret_data = var.gitlab_read_api_access_token +} + +resource "google_secret_manager_secret" "gitlab_webhook_secret" { + project = var.project_id + secret_id = "cb-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" + labels = { + label = "cb-${random_id.gitlab_resources_random_id.dec}" + } + replication { + auto {} + } +} + +resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { + secret = google_secret_manager_secret.gitlab_webhook_secret.id + secret_data = random_uuid.random_webhook_secret.result +} + +resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { + for_each = { + "api" = google_secret_manager_secret.gitlab_api_secret.id, + "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, + "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id + } + + project = var.project_id + secret_id = each.value + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" +} + +resource "google_cloudbuildv2_connection" "vcs_connection" { + project = var.project_id + name = "cb-gl-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" + location = local.location + + gitlab_config { + host_uri = null + authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name + } + read_authorizer_credential { + user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name + } + webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name + } + + depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] +} + +// Create the repository connection. +resource "google_cloudbuildv2_repository" "repository_connection" { + project = var.project_id + name = local.gl_name + location = local.location + + parent_connection = google_cloudbuildv2_connection.vcs_connection.name + remote_uri = local.repoURL +} + module "bootstrap_github_repo" { source = "terraform-google-modules/gcloud/google" version = "~> 3.1" upgrade = false create_cmd_entrypoint = "${path.module}/scripts/push-to-repo.sh" - create_cmd_body = "${var.gitlab_pat} ${var.repository_uri} ${path.module}/files" + create_cmd_body = "${var.gitlab_api_access_token} ${var.repository_uri} ${path.module}/files" } diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf index d3b9b589..70238553 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/outputs.tf @@ -45,5 +45,11 @@ output "artifacts_bucket" { } output "project_id" { - value = var.project_id + description = "The ID of the project in which the resources were provisioned" + value = var.project_id +} + +output "location" { + description = "The location in which the resources were provisioned" + value = local.location } diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf index 2cdd2dec..d3516954 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/variables.tf @@ -19,18 +19,19 @@ variable "project_id" { type = string } -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be defined if repository type is `CLOUDBUILD_V2_REPOSITORY`." +variable "gitlab_api_access_token" { + description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." type = string + sensitive = true } -variable "gitlab_pat" { - description = "GitLab access token." +variable "gitlab_read_api_access_token" { + description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." type = string sensitive = true } variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." + description = "The URI of the GitLab repository where the Terraform configs are stored." type = string } diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf deleted file mode 100644 index 080d1e7f..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_github/main.tf +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - github_app_installation_id = "47590865" - location = "us-central1" - # GitHub repo url of form "github.com/owner/name" - repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" - - repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") - gh_repo_url_split = split("/", local.repoURLWithoutSuffix) - gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] - - github_secret_version_id = google_secret_manager_secret_version.github_token_secret_version.name - github_secret_id = google_secret_manager_secret.github_token_secret.id - - host_connection_name = "builder-gh-${random_id.resources_random_id.dec}-${var.project_id}" - repo_connection_name = local.gh_name -} - -module "cloudbuilder" { - source = "../../../examples/tf_cloudbuild_builder_simple_github" - - project_id = var.project_id - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - github_pat = var.im_github_pat - repository_uri = var.repository_uri -} - -// Create a secret containing the personal access token and grant permissions to the Service Agent. -resource "google_secret_manager_secret" "github_token_secret" { - project = var.project_id - secret_id = "builder-gh-${random_id.resources_random_id.dec}-${local.gh_name}" - - labels = { - label = "builder-gh-${random_id.resources_random_id.dec}" - } - - replication { - auto {} - } -} - -// Personal access token from VCS. -resource "google_secret_manager_secret_version" "github_token_secret_version" { - secret = google_secret_manager_secret.github_token_secret.id - secret_data = var.im_github_pat -} - -resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { - project = var.project_id - secret_id = local.github_secret_id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" -} - -data "google_project" "project" { - project_id = var.project_id -} - -// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. -resource "random_id" "resources_random_id" { - byte_length = 4 -} - -resource "google_cloudbuildv2_connection" "vcs_connection" { - project = var.project_id - location = local.location - - name = local.host_connection_name - - github_config { - app_installation_id = local.github_app_installation_id - authorizer_credential { - oauth_token_secret_version = local.github_secret_version_id - } - } -} - -// Create the repository connection. -resource "google_cloudbuildv2_repository" "repository_connection" { - project = var.project_id - location = local.location - name = local.repo_connection_name - parent_connection = google_cloudbuildv2_connection.vcs_connection.name - remote_uri = local.repoURL -} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf deleted file mode 100644 index 80528381..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_github/outputs.tf +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "artifact_repo" { - description = "GAR Repo created to store TF Cloud Builder images" - value = module.cloudbuilder.artifact_repo -} - -output "workflow_id" { - description = "Workflow ID for triggering new TF Builder build" - value = module.cloudbuilder.workflow_id -} - -output "scheduler_id" { - description = "Scheduler ID for periodically triggering TF Builder build Workflow" - value = module.cloudbuilder.scheduler_id -} - -output "cloudbuild_trigger_id" { - description = "Trigger used for building new TF Builder" - value = module.cloudbuilder.cloudbuild_trigger_id -} - -output "repository_id" { - description = "ID of the v2 repository" - value = google_cloudbuildv2_repository.repository_connection.id -} - -output "project_id" { - value = var.project_id -} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf b/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf deleted file mode 100644 index 56c6936b..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_github/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The ID of the project in which to provision resources." - type = string -} - -variable "im_github_pat" { - description = "GitHub personal access token." - type = string - sensitive = true -} - -variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." - type = string -} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf deleted file mode 100644 index 06af3d96..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/main.tf +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - location = "us-central1" - # GitLab repo url of form "gitlab.com/owner/name" - repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" - - repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") - gl_repo_url_split = split("/", local.repoURLWithoutSuffix) - gl_name = local.gl_repo_url_split[length(local.gl_repo_url_split) - 1] - - gitlab_secret_version_id = google_secret_manager_secret_version.gitlab_api_secret_version.name - gitlab_secret_id = google_secret_manager_secret.gitlab_api_secret.id - - host_connection_name = "builder-gh-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" - repo_connection_name = local.gl_name -} - -module "cloudbuilder" { - source = "../../../examples/tf_cloudbuild_builder_simple_gitlab" - - project_id = var.project_id - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - gitlab_pat = var.gitlab_api_access_token - repository_uri = var.repository_uri -} - -// Create a secret containing the personal access token and grant permissions to the Service Agent. -resource "google_secret_manager_secret" "gitlab_api_secret" { - project = var.project_id - secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" - - labels = { - label = "b-${random_id.gitlab_resources_random_id.dec}" - } - - replication { - auto {} - } -} - -// Personal access token from VCS. -resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { - secret = google_secret_manager_secret.gitlab_api_secret.id - secret_data = var.gitlab_api_access_token -} - -resource "google_secret_manager_secret" "gitlab_read_api_secret" { - project = var.project_id - secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" - labels = { - label = "b-${random_id.gitlab_resources_random_id.dec}" - } - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { - secret = google_secret_manager_secret.gitlab_read_api_secret.id - secret_data = var.gitlab_read_api_access_token -} - -resource "google_secret_manager_secret" "gitlab_webhook_secret" { - project = var.project_id - secret_id = "builder-gl-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" - labels = { - label = "b-${random_id.gitlab_resources_random_id.dec}" - } - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { - secret = google_secret_manager_secret.gitlab_webhook_secret.id - secret_data = random_uuid.random_webhook_secret.result -} - -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { - for_each = { - "api" = google_secret_manager_secret.gitlab_api_secret.id, - "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, - "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id - } - - project = var.project_id - secret_id = each.value - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" -} - - -// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. -resource "random_id" "gitlab_resources_random_id" { - byte_length = 8 -} - -resource "random_uuid" "random_webhook_secret" { -} - -resource "google_cloudbuildv2_connection" "vcs_connection" { - project = var.project_id - location = local.location - - name = local.host_connection_name - - gitlab_config { - host_uri = null - authorizer_credential { - user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name - } - read_authorizer_credential { - user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name - } - webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name - } - - depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] -} - -// Create the repository connection. -resource "google_cloudbuildv2_repository" "repository_connection" { - project = var.project_id - location = local.location - name = local.repo_connection_name - parent_connection = google_cloudbuildv2_connection.vcs_connection.name - remote_uri = local.repoURL -} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf deleted file mode 100644 index 80528381..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/outputs.tf +++ /dev/null @@ -1,44 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "artifact_repo" { - description = "GAR Repo created to store TF Cloud Builder images" - value = module.cloudbuilder.artifact_repo -} - -output "workflow_id" { - description = "Workflow ID for triggering new TF Builder build" - value = module.cloudbuilder.workflow_id -} - -output "scheduler_id" { - description = "Scheduler ID for periodically triggering TF Builder build Workflow" - value = module.cloudbuilder.scheduler_id -} - -output "cloudbuild_trigger_id" { - description = "Trigger used for building new TF Builder" - value = module.cloudbuilder.cloudbuild_trigger_id -} - -output "repository_id" { - description = "ID of the v2 repository" - value = google_cloudbuildv2_repository.repository_connection.id -} - -output "project_id" { - value = var.project_id -} diff --git a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf b/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf deleted file mode 100644 index cfa7e530..00000000 --- a/test/fixtures/tf_cloudbuild_builder_simple_gitlab/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The ID of the project in which to provision resources." - type = string -} - -variable "gitlab_api_access_token" { - description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." - type = string - sensitive = true -} - -variable "gitlab_read_api_access_token" { - description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." - type = string - sensitive = true -} - -variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." - type = string -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf deleted file mode 100644 index a433b57f..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/main.tf +++ /dev/null @@ -1,100 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - github_app_installation_id = "47590865" - location = "us-central1" - # GitHub repo url of form "github.com/owner/name" - repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" - - repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") - gh_repo_url_split = split("/", local.repoURLWithoutSuffix) - gh_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] - - github_secret_version_id = google_secret_manager_secret_version.github_token_secret_version.name - github_secret_id = google_secret_manager_secret.github_token_secret.id - - host_connection_name = "cb-${random_id.resources_random_id.dec}-${var.project_id}" - repo_connection_name = local.gh_name -} - -module "github_workspace" { - source = "../../../examples/tf_cloudbuild_workspace_simple_github" - - project_id = var.project_id - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - github_pat = var.im_github_pat - repository_uri = var.repository_uri -} - -// Create a secret containing the personal access token and grant permissions to the Service Agent. -resource "google_secret_manager_secret" "github_token_secret" { - project = var.project_id - secret_id = "cb-github-${random_id.resources_random_id.dec}-${local.gh_name}" - - labels = { - label = "cb-${random_id.resources_random_id.dec}" - } - - replication { - auto {} - } -} - -// Personal access token from VCS. -resource "google_secret_manager_secret_version" "github_token_secret_version" { - secret = google_secret_manager_secret.github_token_secret.id - secret_data = var.im_github_pat -} - -resource "google_secret_manager_secret_iam_member" "github_token_iam_member" { - project = var.project_id - secret_id = local.github_secret_id - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" -} - -data "google_project" "project" { - project_id = var.project_id -} - -// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. -resource "random_id" "resources_random_id" { - byte_length = 4 -} - -resource "google_cloudbuildv2_connection" "vcs_connection" { - project = var.project_id - location = local.location - - name = local.host_connection_name - - github_config { - app_installation_id = local.github_app_installation_id - authorizer_credential { - oauth_token_secret_version = local.github_secret_version_id - } - } -} - -// Create the repository connection. -resource "google_cloudbuildv2_repository" "repository_connection" { - project = var.project_id - location = local.location - name = local.repo_connection_name - parent_connection = google_cloudbuildv2_connection.vcs_connection.name - remote_uri = local.repoURL -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf deleted file mode 100644 index 0054095a..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/outputs.tf +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cloudbuild_plan_trigger_id" { - description = "Trigger used for running TF plan" - value = module.github_workspace.cloudbuild_plan_trigger_id -} - -output "cloudbuild_apply_trigger_id" { - description = "Trigger used for running TF apply" - value = module.github_workspace.cloudbuild_apply_trigger_id -} - -output "cloudbuild_sa" { - description = "SA used by Cloud Build triggers" - value = module.github_workspace.cloudbuild_sa -} - -output "state_bucket" { - description = "Bucket for storing TF state" - value = module.github_workspace.state_bucket -} - -output "logs_bucket" { - description = "Bucket for storing TF logs" - value = module.github_workspace.logs_bucket -} - -output "artifacts_bucket" { - description = "Bucket for storing TF plans" - value = module.github_workspace.artifacts_bucket -} - -output "project_id" { - value = var.project_id -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf b/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf deleted file mode 100644 index 56c6936b..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_github/variables.tf +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The ID of the project in which to provision resources." - type = string -} - -variable "im_github_pat" { - description = "GitHub personal access token." - type = string - sensitive = true -} - -variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." - type = string -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf deleted file mode 100644 index cceafe8b..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/main.tf +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -locals { - location = "us-central1" - # repo url of form "gitlab.com/owner/name" - repoURL = endswith(var.repository_uri, ".git") ? var.repository_uri : "${var.repository_uri}.git" - - repoURLWithoutSuffix = trimsuffix(local.repoURL, ".git") - gh_repo_url_split = split("/", local.repoURLWithoutSuffix) - gl_name = local.gh_repo_url_split[length(local.gh_repo_url_split) - 1] - - gitlab_secret_version_id = google_secret_manager_secret_version.gitlab_api_secret_version.name - gitlab_secret_id = google_secret_manager_secret.gitlab_api_secret.id - - host_connection_name = "cb-${random_id.gitlab_resources_random_id.dec}-${var.project_id}" - repo_connection_name = local.gl_name -} - -module "gitlab_workspace" { - source = "../../../examples/tf_cloudbuild_workspace_simple_gitlab" - - project_id = var.project_id - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - gitlab_pat = var.gitlab_api_access_token - repository_uri = var.repository_uri -} - -// Create a secret containing the personal access token and grant permissions to the Service Agent. -resource "google_secret_manager_secret" "gitlab_api_secret" { - project = var.project_id - secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-api-access-token" - - labels = { - label = "cb-${random_id.gitlab_resources_random_id.dec}" - } - - replication { - auto {} - } -} - -// Personal access token from VCS. -resource "google_secret_manager_secret_version" "gitlab_api_secret_version" { - secret = google_secret_manager_secret.gitlab_api_secret.id - secret_data = var.gitlab_api_access_token -} - -resource "google_secret_manager_secret" "gitlab_read_api_secret" { - project = var.project_id - secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-read-api-access-token" - labels = { - label = "cb-${random_id.gitlab_resources_random_id.dec}" - } - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "gitlab_read_api_secret_version" { - secret = google_secret_manager_secret.gitlab_read_api_secret.id - secret_data = var.gitlab_read_api_access_token -} - -resource "google_secret_manager_secret" "gitlab_webhook_secret" { - project = var.project_id - secret_id = "cb-gitlab-${local.gl_name}-${random_id.gitlab_resources_random_id.dec}-webhook-secret" - labels = { - label = "cb-${random_id.gitlab_resources_random_id.dec}" - } - replication { - auto {} - } -} - -resource "google_secret_manager_secret_version" "gitlab_webhook_secret_version" { - secret = google_secret_manager_secret.gitlab_webhook_secret.id - secret_data = random_uuid.random_webhook_secret.result -} - -data "google_project" "project" { - project_id = var.project_id -} - -resource "google_secret_manager_secret_iam_member" "gitlab_token_iam_member" { - for_each = { - "api" = google_secret_manager_secret.gitlab_api_secret.id, - "read_api" = google_secret_manager_secret.gitlab_read_api_secret.id, - "webhook" = google_secret_manager_secret.gitlab_webhook_secret.id - } - - project = var.project_id - secret_id = each.value - role = "roles/secretmanager.secretAccessor" - member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-cloudbuild.iam.gserviceaccount.com" -} - - -// Added to various IDs to prevent potential conflicts for deployments targeting the same repository. -resource "random_id" "gitlab_resources_random_id" { - byte_length = 8 -} - -resource "random_uuid" "random_webhook_secret" { -} - -resource "google_cloudbuildv2_connection" "vcs_connection" { - project = var.project_id - location = local.location - - name = local.host_connection_name - - gitlab_config { - host_uri = null - authorizer_credential { - user_token_secret_version = google_secret_manager_secret_version.gitlab_api_secret_version.name - } - read_authorizer_credential { - user_token_secret_version = google_secret_manager_secret_version.gitlab_read_api_secret_version.name - } - webhook_secret_secret_version = google_secret_manager_secret_version.gitlab_webhook_secret_version.name - } - - depends_on = [google_secret_manager_secret_iam_member.gitlab_token_iam_member] -} - -// Create the repository connection. -resource "google_cloudbuildv2_repository" "repository_connection" { - project = var.project_id - location = local.location - name = local.repo_connection_name - parent_connection = google_cloudbuildv2_connection.vcs_connection.name - remote_uri = local.repoURL -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf deleted file mode 100644 index fc2863be..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/outputs.tf +++ /dev/null @@ -1,49 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cloudbuild_plan_trigger_id" { - description = "Trigger used for running TF plan" - value = module.gitlab_workspace.cloudbuild_plan_trigger_id -} - -output "cloudbuild_apply_trigger_id" { - description = "Trigger used for running TF apply" - value = module.gitlab_workspace.cloudbuild_apply_trigger_id -} - -output "cloudbuild_sa" { - description = "SA used by Cloud Build triggers" - value = module.gitlab_workspace.cloudbuild_sa -} - -output "state_bucket" { - description = "Bucket for storing TF state" - value = module.gitlab_workspace.state_bucket -} - -output "logs_bucket" { - description = "Bucket for storing TF logs" - value = module.gitlab_workspace.logs_bucket -} - -output "artifacts_bucket" { - description = "Bucket for storing TF plans" - value = module.gitlab_workspace.artifacts_bucket -} - -output "project_id" { - value = var.project_id -} diff --git a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf b/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf deleted file mode 100644 index cfa7e530..00000000 --- a/test/fixtures/tf_cloudbuild_workspace_simple_gitlab/variables.tf +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - description = "The ID of the project in which to provision resources." - type = string -} - -variable "gitlab_api_access_token" { - description = "GitLab personal access token with api scope. If provided, creates a secret within Secret Manager." - type = string - sensitive = true -} - -variable "gitlab_read_api_access_token" { - description = "GitLab personal access token with read_api scope. If provided, creates a secret within Secret Manager." - type = string - sensitive = true -} - -variable "repository_uri" { - description = "The URI of the repo where the Terraform configs are stored." - type = string -} diff --git a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go index 173d2f24..60a78ae2 100644 --- a/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_github/tf_cloudbuild_builder_simple_github_test.go @@ -97,7 +97,7 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { // Testing the module's feature of appending the ".git" suffix if it's missing repoURL := strings.TrimSuffix(client.repository.GetCloneURL(), ".git") vars := map[string]interface{}{ - "im_github_pat": githubPAT, + "github_pat": githubPAT, "repository_uri": repoURL, } bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) @@ -112,7 +112,7 @@ func TestTFCloudBuildBuilderGitHub(t *testing.T) { } }) - location := "us-central1" + location := bpt.GetStringOutput("location") projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") artifactRepoDockerRegistry := fmt.Sprintf("%s-docker.pkg.dev/%s/%s/terraform", location, projectID, artifactRepo) diff --git a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go index 18146542..05632c92 100644 --- a/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_builder_simple_gitlab/tf_cloudbuild_builder_simple_gitlab_test.go @@ -121,7 +121,7 @@ func TestTFCloudBuildBuilderGitLab(t *testing.T) { } }) - location := "us-central1" + location := bpt.GetStringOutput("location") projectID := bpt.GetStringOutput("project_id") artifactRepo := bpt.GetStringOutput("artifact_repo") artifactRepoDockerRegistry := fmt.Sprintf("%s-docker.pkg.dev/%s/%s/terraform", location, projectID, artifactRepo) diff --git a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go index b88fc3c5..220cb368 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_github/tf_cloudbuild_workspace_simple_github_test.go @@ -97,7 +97,7 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { // Testing the module's feature of appending the ".git" suffix if it's missing repoURL := strings.TrimSuffix(client.repository.GetCloneURL(), ".git") vars := map[string]interface{}{ - "im_github_pat": githubPAT, + "github_pat": githubPAT, "repository_uri": repoURL, } bpt := tft.NewTFBlueprintTest(t, tft.WithVars(vars)) @@ -112,7 +112,7 @@ func TestCloudBuildWorkspaceSimpleGitHub(t *testing.T) { } }) - location := "us-central1" + location := bpt.GetStringOutput("location") projectID := bpt.GetStringOutput("project_id") // cloud build triggers diff --git a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go index bbc4ac2b..b0bccac2 100644 --- a/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go +++ b/test/integration/tf_cloudbuild_workspace_simple_gitlab/tf_cloudbuild_workspace_simple_gitlab_test.go @@ -130,7 +130,7 @@ func TestCloudBuildWorkspaceSimpleGitLab(t *testing.T) { } }) - location := "us-central1" + location := bpt.GetStringOutput("location") projectID := bpt.GetStringOutput("project_id") // cloud build triggers From a2de7b54519f07e4b1f237eb26aab21dc8a30866 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 8 Aug 2024 20:41:36 -0300 Subject: [PATCH 40/46] add missing output --- examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf index 38dbd34a..3cbb32f9 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/outputs.tf @@ -34,6 +34,11 @@ output "cloudbuild_trigger_id" { value = module.cloudbuilder.cloudbuild_trigger_id } +output "repository_id" { + description = "ID of the Cloud Build repositories (2nd gen) repository" + value = google_cloudbuildv2_repository.repository_connection.id +} + output "project_id" { description = "The ID of the project in which the resources were provisioned" value = var.project_id From cfb6fa87b49c809fe703c91da7fdea569b816e8c Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 8 Aug 2024 20:52:55 -0300 Subject: [PATCH 41/46] fix lint issue --- examples/tf_cloudbuild_builder_simple_gitlab/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/README.md b/examples/tf_cloudbuild_builder_simple_gitlab/README.md index da88c01e..48f207d2 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/README.md +++ b/examples/tf_cloudbuild_builder_simple_gitlab/README.md @@ -24,6 +24,7 @@ For more information on this topic refer to the Cloud Build repositories (2nd ge | cloudbuild\_trigger\_id | Trigger used for building new TF Builder | | location | The location in which the resources were provisioned | | project\_id | The ID of the project in which the resources were provisioned | +| repository\_id | ID of the Cloud Build repositories (2nd gen) repository | | scheduler\_id | Scheduler ID for periodically triggering TF Builder build Workflow | | workflow\_id | Workflow ID for triggering new TF Builder build | From c07457c9e14b3088f602caae40cf7af27512afa7 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 13 Aug 2024 22:13:11 -0300 Subject: [PATCH 42/46] add upgrading to v9.0 documentation --- docs/upgrading_to_v9.0.md | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 docs/upgrading_to_v9.0.md diff --git a/docs/upgrading_to_v9.0.md b/docs/upgrading_to_v9.0.md new file mode 100644 index 00000000..217600ec --- /dev/null +++ b/docs/upgrading_to_v9.0.md @@ -0,0 +1,42 @@ +# Upgrading to v9.0 + +The v9.0 release of *bootstrap* is a backwards incompatible release. + +Some variables default values were replaced to align with the restriction that Cloud Build Repositories (2nd Gen) cannot be created in multi-regions or in the `global` region. + +You need to update your configurations if you used the default values to prevent resources to be recreated. + +## Default value for variable `trigger_location` in module `tf_cloudbuild_workspace` has changed + +To preserve the resources created before. include the input `trigger_location` with the previous default value in the module call + +```diff +module "tf_workspace" { + source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace" +- version = "~> 8.0" ++ version = "~> 9.0" + ++ trigger_location = "global" +``` + +## Default value for variables `trigger_location` and `gar_repo_location` in module `tf_cloudbuild_builde` have changed + +To preserve the resources created before, include the inputs `trigger_location` and `gar_repo_location` with the previous default values in the module call + +```diff +module "cloudbuilder" { + source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_builder" +- version = "~> 8.0" ++ version = "~> 9.0" + ++ trigger_location = "global" ++ gar_repo_location = "us" +``` + +An apply after adding the two inputs will still have an *in-place update* in the `google_workflows_workflow` created by the module. + +The endpoint that is used to trigger a build was replaced with a new one that allows a location to be provided. + +``` + # module.cloudbuilder.google_workflows_workflow.builder will be updated in-place +``` From a4094d48f5d93144214313f4d88d9311af269630 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 13 Aug 2024 22:23:00 -0300 Subject: [PATCH 43/46] fix variable usage for Cloud Build repositories (2nd Gen) --- .../tf_cloudbuild_builder_simple_github/main.tf | 2 +- .../tf_cloudbuild_builder_simple_gitlab/main.tf | 2 +- modules/tf_cloudbuild_builder/README.md | 9 ++++----- modules/tf_cloudbuild_builder/cb.tf | 2 +- modules/tf_cloudbuild_builder/variables.tf | 12 ++++-------- modules/tf_cloudbuild_workspace/README.md | 7 +++---- modules/tf_cloudbuild_workspace/cb.tf | 4 ++-- modules/tf_cloudbuild_workspace/variables.tf | 14 ++++---------- 8 files changed, 20 insertions(+), 32 deletions(-) diff --git a/examples/tf_cloudbuild_builder_simple_github/main.tf b/examples/tf_cloudbuild_builder_simple_github/main.tf index 5b09c681..efc37c78 100644 --- a/examples/tf_cloudbuild_builder_simple_github/main.tf +++ b/examples/tf_cloudbuild_builder_simple_github/main.tf @@ -42,7 +42,7 @@ module "cloudbuilder" { version = "~> 8.0" project_id = module.enabled_google_apis.project_id - dockerfile_repo_id = google_cloudbuildv2_repository.repository_connection.id + dockerfile_repo_uri = google_cloudbuildv2_repository.repository_connection.id dockerfile_repo_type = "GITHUB" use_cloudbuildv2_repository = true trigger_location = local.location diff --git a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf index 186d49f7..27c63b91 100644 --- a/examples/tf_cloudbuild_builder_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_builder_simple_gitlab/main.tf @@ -41,7 +41,7 @@ module "cloudbuilder" { version = "~> 8.0" project_id = module.enabled_google_apis.project_id - dockerfile_repo_id = google_cloudbuildv2_repository.repository_connection.id + dockerfile_repo_uri = google_cloudbuildv2_repository.repository_connection.id dockerfile_repo_type = "UNKNOWN" // "GITLAB" is not one of the options available so we need to use "UNKNOWN" use_cloudbuildv2_repository = true trigger_location = "us-central1" diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 82ad1697..0ce8d91c 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -38,19 +38,18 @@ This module creates: | cb\_logs\_bucket\_force\_destroy | When deleting the bucket for storing CloudBuild logs, this boolean option will delete all contained objects. If false, Terraform will fail to delete buckets which contain objects. | `bool` | `false` | no | | cloudbuild\_sa | Custom SA email to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | dockerfile\_repo\_dir | The directory inside the repo where the Dockerfile is located. If empty defaults to repo root. | `string` | `""` | no | -| dockerfile\_repo\_id | The repository id where the Dockerfile for Terraform builder is stored. Use for Cloudbuild 2nd gen repository. Either specify this or the variable `dockerfile_repo_uri`. | `string` | `""` | no | | dockerfile\_repo\_ref | The branch or tag to use. Use refs/heads/branchname for branches or refs/tags/tagname for tags. | `string` | `"refs/heads/main"` | no | | dockerfile\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | -| dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories. | `string` | `""` | no | +| dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}' | `string` | `""` | no | | enable\_worker\_pool | Set to true to use a private worker pool in the Cloud Build Trigger. | `bool` | `false` | no | -| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | n/a | yes | +| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | `"us-central1"` | no | | gar\_repo\_name | Name of the Google Artifact Repository where the Terraform builder images are stored. | `string` | `"tf-runners"` | no | | image\_name | Name of the image for the Terraform builder. | `string` | `"terraform"` | no | | project\_id | GCP project for Cloud Build trigger,workflow and scheduler. | `string` | n/a | yes | | terraform\_version | The initial terraform version in semantic version format. | `string` | `"1.1.0"` | no | -| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | n/a | yes | +| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | `"us-central1"` | no | | trigger\_name | Name of the Cloud Build trigger building the Terraform builder. | `string` | `"tf-cloud-builder-build"` | no | -| use\_cloudbuildv2\_repository | Use Cloudbuild 2nd gen repository | `bool` | `false` | no | +| use\_cloudbuildv2\_repository | Use Cloud Build repository (2nd gen) | `bool` | `false` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | | workflow\_name | Name of the workflow managing builds. | `string` | `"terraform-runner-workflow"` | no | | workflow\_region | The region of the workflow. | `string` | `"us-central1"` | no | diff --git a/modules/tf_cloudbuild_builder/cb.tf b/modules/tf_cloudbuild_builder/cb.tf index 541fa51a..420e355f 100644 --- a/modules/tf_cloudbuild_builder/cb.tf +++ b/modules/tf_cloudbuild_builder/cb.tf @@ -51,7 +51,7 @@ resource "google_cloudbuild_trigger" "build_trigger" { dynamic "source_to_build" { for_each = var.use_cloudbuildv2_repository ? [1] : [] content { - repository = var.dockerfile_repo_id + repository = var.dockerfile_repo_uri ref = var.dockerfile_repo_ref repo_type = var.dockerfile_repo_type } diff --git a/modules/tf_cloudbuild_builder/variables.tf b/modules/tf_cloudbuild_builder/variables.tf index 3ad2d5a7..fa561dc8 100644 --- a/modules/tf_cloudbuild_builder/variables.tf +++ b/modules/tf_cloudbuild_builder/variables.tf @@ -70,6 +70,7 @@ variable "gar_repo_name" { variable "gar_repo_location" { description = "Name of the location for the Google Artifact Repository." type = string + default = "us-central1" } variable "terraform_version" { @@ -93,22 +94,17 @@ variable "trigger_name" { variable "trigger_location" { description = "Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool." type = string + default = "us-central1" } variable "dockerfile_repo_uri" { - description = "The URI of the repo where the Dockerfile for Terraform builder is stored. Either specify this or the variable `dockerfile_repo_id` for cloudbuildv2 repositories." - type = string - default = "" -} - -variable "dockerfile_repo_id" { - description = "The repository id where the Dockerfile for Terraform builder is stored. Use for Cloudbuild 2nd gen repository. Either specify this or the variable `dockerfile_repo_uri`." + description = "The URI of the repo where the Dockerfile for Terraform builder is stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'" type = string default = "" } variable "use_cloudbuildv2_repository" { - description = "Use Cloudbuild 2nd gen repository" + description = "Use Cloud Build repository (2nd gen)" type = bool default = false } diff --git a/modules/tf_cloudbuild_workspace/README.md b/modules/tf_cloudbuild_workspace/README.md index 39f8fc6e..b9e698b0 100644 --- a/modules/tf_cloudbuild_workspace/README.md +++ b/modules/tf_cloudbuild_workspace/README.md @@ -55,7 +55,6 @@ This module creates: | cloudbuild\_plan\_filename | Optional Cloud Build YAML definition used for terraform plan. Defaults to using inline definition. | `string` | `null` | no | | cloudbuild\_sa | Custom SA id of form projects/{{project}}/serviceAccounts/{{email}} to be used by the CloudBuild trigger. Defaults to being created if empty. | `string` | `""` | no | | cloudbuild\_sa\_roles | Optional to assign to custom CloudBuild SA. Map of project name or any static key to object with project\_id and list of roles. |
map(object({
project_id = string
roles = list(string)
}))
| `{}` | no | -| cloudbuildv2\_repository\_id | Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. Must be set if repository type is `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | | create\_cloudbuild\_sa | Create a Service Account for use in Cloud Build. If false `cloudbuild_sa` has to be specified. | `bool` | `true` | no | | create\_cloudbuild\_sa\_name | Custom name to be used in the creation of the Cloud Build service account if `create_cloudbuild_sa` is true. Defaults to generated name if empty | `string` | `""` | no | | create\_state\_bucket | Create a GCS bucket for storing state. If false `state_bucket_self_link` has to be specified. | `bool` | `true` | no | @@ -71,9 +70,9 @@ This module creates: | tf\_apply\_branches | List of git branches configured to run terraform apply Cloud Build trigger. All other branches will run plan by default. | `list(string)` |
[
"main"
]
| no | | tf\_cloudbuilder | Name of the Cloud Builder image used for running build steps. | `string` | `"hashicorp/terraform:1.3.10"` | no | | tf\_repo\_dir | The directory inside the repo where the Terrafrom root config is located. If empty defaults to repo root. | `string` | `""` | no | -| tf\_repo\_type | Type of repo. When the repo type is CLOUDBUILD\_V2\_REPOSITORY, it will use the generig Cloudbuild 2nd gen Repository API. | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | -| tf\_repo\_uri | The URI of the repo where Terraform configs are stored. Must be set if repository type is not `CLOUDBUILD_V2_REPOSITORY`. | `string` | `""` | no | -| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"global"` | no | +| tf\_repo\_type | Type of repo. When the repo type is CLOUDBUILD\_V2\_REPOSITORY, it will use the generic Cloudbuild 2nd gen Repository API. | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | +| tf\_repo\_uri | The URI of the repo where Terraform configs are stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. | `string` | `""` | no | +| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"us-central1"` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | ## Outputs diff --git a/modules/tf_cloudbuild_workspace/cb.tf b/modules/tf_cloudbuild_workspace/cb.tf index b9e1c757..fa5a4e8a 100644 --- a/modules/tf_cloudbuild_workspace/cb.tf +++ b/modules/tf_cloudbuild_workspace/cb.tf @@ -33,7 +33,7 @@ locals { is_cb_v2_repo = var.tf_repo_type == "CLOUDBUILD_V2_REPOSITORY" # Generic repo name extracted from format projects/{{project}}/locations/{{location}}/connections/{{name}} - cb_v2_repo_name = local.is_cb_v2_repo ? element(split("/", var.cloudbuildv2_repository_id), length(split("/", var.cloudbuildv2_repository_id)) - 1) : "" + cb_v2_repo_name = local.is_cb_v2_repo ? element(split("/", var.tf_repo_uri), length(split("/", var.tf_repo_uri)) - 1) : "" # default build steps default_entrypoint = "terraform" @@ -87,7 +87,7 @@ resource "google_cloudbuild_trigger" "triggers" { dynamic "repository_event_config" { for_each = local.is_cb_v2_repo ? [1] : [] content { - repository = var.cloudbuildv2_repository_id + repository = var.tf_repo_uri # plan for non apply branch dynamic "push" { for_each = each.key != "apply" ? [1] : [] diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index efb423e7..8bc71f83 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -28,7 +28,7 @@ variable "location" { variable "trigger_location" { description = "Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool." type = string - default = "global" + default = "us-central1" } variable "create_cloudbuild_sa" { @@ -148,14 +148,8 @@ variable "prefix" { default = "" } -variable "cloudbuildv2_repository_id" { - description = "Cloudbuild 2nd gen repository ID. Format: 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'. Must be set if repository type is `CLOUDBUILD_V2_REPOSITORY`." - type = string - default = "" -} - variable "tf_repo_uri" { - description = "The URI of the repo where Terraform configs are stored. Must be set if repository type is not `CLOUDBUILD_V2_REPOSITORY`." + description = "The URI of the repo where Terraform configs are stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent_connection}}/repositories/{{name}}'." type = string default = "" } @@ -175,12 +169,12 @@ variable "tf_repo_dir" { } variable "tf_repo_type" { - description = "Type of repo. When the repo type is CLOUDBUILD_V2_REPOSITORY, it will use the generig Cloudbuild 2nd gen Repository API." + description = "Type of repo. When the repo type is CLOUDBUILD_V2_REPOSITORY, it will use the generic Cloudbuild 2nd gen Repository API." type = string default = "CLOUD_SOURCE_REPOSITORIES" validation { condition = contains(["CLOUD_SOURCE_REPOSITORIES", "GITHUB", "CLOUDBUILD_V2_REPOSITORY"], var.tf_repo_type) - error_message = "Must be one of CLOUD_SOURCE_REPOSITORIES, GITHUB, CLOUDBUILD_V2_REPOSITORY" + error_message = "Must be one of CLOUD_SOURCE_REPOSITORIES, GITHUB, or CLOUDBUILD_V2_REPOSITORY" } } From 695345c21332b48d15e0238bca1c04ee4d52eeb8 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Tue, 13 Aug 2024 22:31:41 -0300 Subject: [PATCH 44/46] fix lint issue --- .../main.tf | 16 ++++++++-------- .../main.tf | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/examples/tf_cloudbuild_workspace_simple_github/main.tf b/examples/tf_cloudbuild_workspace_simple_github/main.tf index ac3f36b8..f7587cf9 100644 --- a/examples/tf_cloudbuild_workspace_simple_github/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_github/main.tf @@ -40,14 +40,14 @@ resource "random_id" "resources_random_id" { module "tf_workspace" { source = "../../modules/tf_cloudbuild_workspace" - project_id = module.enabled_google_apis.project_id - tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - location = "us-central1" - trigger_location = "us-central1" - artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gh" - log_bucket_name = "tf-configs-build-logs-${var.project_id}-gh" - create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gh" + project_id = module.enabled_google_apis.project_id + tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" + tf_repo_uri = google_cloudbuildv2_repository.repository_connection.id + location = "us-central1" + trigger_location = "us-central1" + artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gh" + log_bucket_name = "tf-configs-build-logs-${var.project_id}-gh" + create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gh" # allow log/state buckets to be destroyed buckets_force_destroy = true diff --git a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf index 3221dfd1..8d8e0947 100644 --- a/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf +++ b/examples/tf_cloudbuild_workspace_simple_gitlab/main.tf @@ -39,14 +39,14 @@ resource "random_uuid" "random_webhook_secret" { module "tf_workspace" { source = "../../modules/tf_cloudbuild_workspace" - project_id = module.enabled_google_apis.project_id - tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" - cloudbuildv2_repository_id = google_cloudbuildv2_repository.repository_connection.id - location = "us-central1" - trigger_location = "us-central1" - artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gl" - log_bucket_name = "tf-configs-build-logs-${var.project_id}-gl" - create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gl" + project_id = module.enabled_google_apis.project_id + tf_repo_type = "CLOUDBUILD_V2_REPOSITORY" + tf_repo_uri = google_cloudbuildv2_repository.repository_connection.id + location = "us-central1" + trigger_location = "us-central1" + artifacts_bucket_name = "tf-configs-build-artifacts-${var.project_id}-gl" + log_bucket_name = "tf-configs-build-logs-${var.project_id}-gl" + create_state_bucket_name = "tf-configs-build-state-${var.project_id}-gl" # allow log/state buckets to be destroyed buckets_force_destroy = true From 51c2c848703d42d96819c765d0e934bef3106c4c Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Wed, 14 Aug 2024 17:36:42 -0300 Subject: [PATCH 45/46] set trigger_location on tf_cloudbuild_workspace_simple example --- examples/tf_cloudbuild_workspace_simple/main.tf | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/tf_cloudbuild_workspace_simple/main.tf b/examples/tf_cloudbuild_workspace_simple/main.tf index 0bd541bf..4f1b1186 100644 --- a/examples/tf_cloudbuild_workspace_simple/main.tf +++ b/examples/tf_cloudbuild_workspace_simple/main.tf @@ -18,8 +18,9 @@ module "tf_workspace" { source = "terraform-google-modules/bootstrap/google//modules/tf_cloudbuild_workspace" version = "~> 8.0" - project_id = module.enabled_google_apis.project_id - tf_repo_uri = google_sourcerepo_repository.tf_config_repo.url + project_id = module.enabled_google_apis.project_id + tf_repo_uri = google_sourcerepo_repository.tf_config_repo.url + trigger_location = "global" # allow log/state buckets to be destroyed buckets_force_destroy = true cloudbuild_sa_roles = { (module.enabled_google_apis.project_id) = { From 01f78c5e22e69bc3dfee1f663382b38bf73d5a68 Mon Sep 17 00:00:00 2001 From: Daniel Andrade Date: Thu, 15 Aug 2024 17:40:09 -0300 Subject: [PATCH 46/46] remove default value for trigger_location and gar_repo_location --- docs/upgrading_to_v9.0.md | 6 +++--- modules/tf_cloudbuild_builder/README.md | 4 ++-- modules/tf_cloudbuild_builder/variables.tf | 2 -- modules/tf_cloudbuild_workspace/README.md | 2 +- modules/tf_cloudbuild_workspace/variables.tf | 1 - 5 files changed, 6 insertions(+), 9 deletions(-) diff --git a/docs/upgrading_to_v9.0.md b/docs/upgrading_to_v9.0.md index 217600ec..55139076 100644 --- a/docs/upgrading_to_v9.0.md +++ b/docs/upgrading_to_v9.0.md @@ -2,11 +2,11 @@ The v9.0 release of *bootstrap* is a backwards incompatible release. -Some variables default values were replaced to align with the restriction that Cloud Build Repositories (2nd Gen) cannot be created in multi-regions or in the `global` region. +Some variables default values were removed to align with the restriction that Cloud Build Repositories (2nd Gen) cannot be created in multi-regions or in the `global` region. You need to update your configurations if you used the default values to prevent resources to be recreated. -## Default value for variable `trigger_location` in module `tf_cloudbuild_workspace` has changed +## Default value for variable `trigger_location` in module `tf_cloudbuild_workspace` was removed To preserve the resources created before. include the input `trigger_location` with the previous default value in the module call @@ -19,7 +19,7 @@ module "tf_workspace" { + trigger_location = "global" ``` -## Default value for variables `trigger_location` and `gar_repo_location` in module `tf_cloudbuild_builde` have changed +## Default value for variables `trigger_location` and `gar_repo_location` in module `tf_cloudbuild_builde` were removed To preserve the resources created before, include the inputs `trigger_location` and `gar_repo_location` with the previous default values in the module call diff --git a/modules/tf_cloudbuild_builder/README.md b/modules/tf_cloudbuild_builder/README.md index 0ce8d91c..28457ef6 100644 --- a/modules/tf_cloudbuild_builder/README.md +++ b/modules/tf_cloudbuild_builder/README.md @@ -42,12 +42,12 @@ This module creates: | dockerfile\_repo\_type | Type of repo | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | | dockerfile\_repo\_uri | The URI of the repo where the Dockerfile for Terraform builder is stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}' | `string` | `""` | no | | enable\_worker\_pool | Set to true to use a private worker pool in the Cloud Build Trigger. | `bool` | `false` | no | -| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | `"us-central1"` | no | +| gar\_repo\_location | Name of the location for the Google Artifact Repository. | `string` | n/a | yes | | gar\_repo\_name | Name of the Google Artifact Repository where the Terraform builder images are stored. | `string` | `"tf-runners"` | no | | image\_name | Name of the image for the Terraform builder. | `string` | `"terraform"` | no | | project\_id | GCP project for Cloud Build trigger,workflow and scheduler. | `string` | n/a | yes | | terraform\_version | The initial terraform version in semantic version format. | `string` | `"1.1.0"` | no | -| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | `"us-central1"` | no | +| trigger\_location | Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool. | `string` | n/a | yes | | trigger\_name | Name of the Cloud Build trigger building the Terraform builder. | `string` | `"tf-cloud-builder-build"` | no | | use\_cloudbuildv2\_repository | Use Cloud Build repository (2nd gen) | `bool` | `false` | no | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | diff --git a/modules/tf_cloudbuild_builder/variables.tf b/modules/tf_cloudbuild_builder/variables.tf index fa561dc8..d858815d 100644 --- a/modules/tf_cloudbuild_builder/variables.tf +++ b/modules/tf_cloudbuild_builder/variables.tf @@ -70,7 +70,6 @@ variable "gar_repo_name" { variable "gar_repo_location" { description = "Name of the location for the Google Artifact Repository." type = string - default = "us-central1" } variable "terraform_version" { @@ -94,7 +93,6 @@ variable "trigger_name" { variable "trigger_location" { description = "Location of the Cloud Build trigger building the Terraform builder. If using private pools should be the same location as the pool." type = string - default = "us-central1" } variable "dockerfile_repo_uri" { diff --git a/modules/tf_cloudbuild_workspace/README.md b/modules/tf_cloudbuild_workspace/README.md index b9e698b0..02e16239 100644 --- a/modules/tf_cloudbuild_workspace/README.md +++ b/modules/tf_cloudbuild_workspace/README.md @@ -72,7 +72,7 @@ This module creates: | tf\_repo\_dir | The directory inside the repo where the Terrafrom root config is located. If empty defaults to repo root. | `string` | `""` | no | | tf\_repo\_type | Type of repo. When the repo type is CLOUDBUILD\_V2\_REPOSITORY, it will use the generic Cloudbuild 2nd gen Repository API. | `string` | `"CLOUD_SOURCE_REPOSITORIES"` | no | | tf\_repo\_uri | The URI of the repo where Terraform configs are stored. If using Cloud Build Repositories (2nd Gen) this is the repository ID where the Dockerfile is stored. Repository ID Format is 'projects/{{project}}/locations/{{location}}/connections/{{parent\_connection}}/repositories/{{name}}'. | `string` | `""` | no | -| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | `"us-central1"` | no | +| trigger\_location | Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool. | `string` | n/a | yes | | worker\_pool\_id | Custom private worker pool ID. Format: 'projects/PROJECT\_ID/locations/REGION/workerPools/PRIVATE\_POOL\_ID'. | `string` | `""` | no | ## Outputs diff --git a/modules/tf_cloudbuild_workspace/variables.tf b/modules/tf_cloudbuild_workspace/variables.tf index 8bc71f83..d3b711b8 100644 --- a/modules/tf_cloudbuild_workspace/variables.tf +++ b/modules/tf_cloudbuild_workspace/variables.tf @@ -28,7 +28,6 @@ variable "location" { variable "trigger_location" { description = "Location of for Cloud Build triggers created in the workspace. If using private pools should be the same location as the pool." type = string - default = "us-central1" } variable "create_cloudbuild_sa" {