Merge branch 'GoogleCloudPlatform:main' into main

.ci/README.md

@@ -102,6 +102,9 @@ The best approach is
 * Build the `downstream-generator` container locally, with the new Gemfile and Gemfile.lock.  This will involve hand-modifying the Dockerfile to use the local Gemfile/Gemfile.lock instead of wget from this repo's `main` branch.  You don't need to check in those changes.
 * When that container is built, and while nothing else is running in GCB (wait, if you need to), push the container to GCR, and as soon as possible afterwards, merge the dependency-changing PR.
 
+## Changes to cloud build yaml:
+If changes are made to `gcb-contributor-membership-checker.yml` or `gcb-community-checker.yml` they will not be reflected in presubmit runs for existing PRs without a rebase. This is because these build triggers are linked to pull request creation and not pushes to the PR branch. If changes are needed to these build files they will need to be made in a backwards-compatible manner. Note that changes to other files used by these triggers will be immediately reflected in all PRs, leading to a possible disconnect between the yaml files and the rest of the CI code.
+
 ## Historical Note: Design choices & tradeoffs
 * The downstream push doesn't wait for checks on its PRs against downstreams.  This may inconvenience some existing workflows which rely on the downstream PR checks.  This ensures that merge conflicts never come into play, since the downstreams never have dangling PRs, but it requires some up-front work to get those checks into the differ.  If a new check is introduced into the downstream Travis, we will need to introduce it into the terraform-tester container.
 * The downstream push is disconnected from the output of the differ (but runs the same code).  This means that the diff which is approved isn't guaranteed to be applied *exactly*, if for instance magic modules' behavior changes on main between diff generation and downstream push.  This is also intended to avoid merge conflicts by, effectively, rebasing each commit on top of main before final generation is done.

.ci/gcb-community-checker.yml

@@ -72,6 +72,7 @@ steps:
         - $_HEAD_BRANCH
         - $_BASE_BRANCH
 
+logsBucket: 'gs://cloudbuild-community-checker-logs'
 availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest

.ci/gcb-contributor-membership-checker.yml

@@ -69,6 +69,7 @@ steps:
       - $_PR_NUMBER
       - $COMMIT_SHA
 
+logsBucket: 'gs://cloudbuild-membership-checker-logs'
 availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest

.ci/gcb-generate-diffs-new.yml

@@ -283,6 +283,7 @@ timeout: 20000s
 options:
     machineType: 'N1_HIGHCPU_32'
 
+logsBucket: 'gs://cloudbuild-generate-diffs-logs'
 availableSecrets:
   secretManager:
     - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-downstreams/versions/latest

.ci/infra/terraform/main.tf

@@ -135,6 +135,12 @@ resource "google_organization_iam_member" "sa_storage_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "apphub_admin" {
+  org_id = data.google_organization.org.org_id
+  role               = "roles/apphub.admin"
+  member             = google_service_account.sa.member
+}
+
 resource "google_billing_account_iam_member" "sa_master_billing_admin" {
   billing_account_id = data.google_billing_account.master_acct.id
   role               = "roles/billing.admin"

.ci/magician/cmd/generate_comment.go

@@ -262,12 +262,6 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 		for _, serviceLabel := range serviceLabels {
 			uniqueServiceLabels[serviceLabel] = struct{}{}
 		}
-
-		err = cleanDiffProcessor(diffProcessorPath, rnr)
-		if err != nil {
-			fmt.Println("cleaning up diff processor: ", err)
-			errors[repo.Title] = append(errors[repo.Title], "The diff processor failed to clean up properly.")
-		}
 	}
 	breakingChangesSlice := maps.Keys(uniqueBreakingChanges)
 	sort.Strings(breakingChangesSlice)
@@ -376,6 +370,11 @@ func computeDiff(repo *source.Repo, oldBranch string, ctlr *source.Controller) (
 
 // Build the diff processor for tpg or tpgb
 func buildDiffProcessor(diffProcessorPath, providerLocalPath string, env map[string]string, rnr ExecRunner) error {
+	for _, path := range []string{"old", "new", "bin"} {
+		if err := rnr.RemoveAll(filepath.Join(diffProcessorPath, path)); err != nil {
+			return err
+		}
+	}
 	if err := rnr.PushDir(diffProcessorPath); err != nil {
 		return err
 	}
@@ -442,15 +441,6 @@ func changedSchemaLabels(prNumber int, currentLabels []string, diffProcessorPath
 	return labels, nil
 }
 
-func cleanDiffProcessor(diffProcessorPath string, rnr ExecRunner) error {
-	for _, path := range []string{"old", "new", "bin"} {
-		if err := rnr.RemoveAll(filepath.Join(diffProcessorPath, path)); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 // Run the missing test detector and return the results.
 // Returns an empty string unless there are missing tests.
 // Error will be nil unless an error occurs during setup.

.github/workflows/build-downstream.yml

@@ -58,7 +58,7 @@ jobs:
           restore-keys: |
             ${{ runner.os }}-go-
 
-      - run: go install golang.org/x/tools/cmd/goimports@latest
+      - run: go install golang.org/x/tools/cmd/goimports@7656c4c657688cae30795365d2a5f30d6f18be7f # v0.19.0
 
       - name: Build ${{ inputs.repo  }}
         run: |

.github/workflows/codeql.yml

@@ -0,0 +1,62 @@
+name: "CodeQL"
+permissions: read-all
+
+on:
+  push:
+    branches: [ "main", "FEATURE-BRANCH-*", "FEATURE-BRANCH-major-release-*" ]
+  # TODO: enable pull_request once behavior on main is confirmed
+  # pull_request:
+  #   branches: [ "main", "FEATURE-BRANCH-*", "FEATURE-BRANCH-major-release-*" ]
+  schedule:
+    - cron: '26 13 * * 3'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: 'ubuntu-latest'
+    timeout-minutes: 360
+    permissions:
+      # required for all workflows
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go', 'ruby' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@05963f47d870e2cb19a537396c1f668a348c7d8f # v3.24.8
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/mmv1-lint-product-yaml.yml

@@ -30,7 +30,7 @@ jobs:
           fi
       - name: Install yamllint
         if: ${{ !failure() && steps.pull_request.outputs.yamlfiles != '' }}
-        run: pip install yamllint
+        run: pip install yamllint==1.32.0 pyyaml==6.0.1 --no-deps
       - name: Lint YAML files
         if: ${{ !failure() && steps.pull_request.outputs.yamlfiles != '' }}
         run: yamllint -c repo/.yamllint ${{steps.pull_request.outputs.yamlfiles}}

mmv1/api/product.go

@@ -211,6 +211,13 @@ func (p *Product) SetPropertiesBasedOnVersion(version *product.Version) {
 	p.BaseUrl = version.BaseUrl
 }
 
+func (p *Product) TerraformName() string {
+	if p.LegacyName != "" {
+		return google.Underscore(p.LegacyName)
+	}
+	return google.Underscore(p.Name)
+}
+
 // ====================
 // Debugging Methods
 // ====================

mmv1/api/resource.go

@@ -764,3 +764,10 @@ func (r Resource) HasZone() bool {
 func (r Resource) Lineage() string {
 	return r.Name
 }
+
+func (r Resource) TerraformName() string {
+	if r.LegacyName != "" {
+		return r.LegacyName
+	}
+	return fmt.Sprintf("google_%s_%s", r.ProductMetadata.TerraformName(), google.Underscore(r.Name))
+}