Add plural data source google_kms_crypto_keys (#11053) (#18605)

[upstream:f514cc2e5b57129f3136b17ce602aa724fa390c4]

Signed-off-by: Modular Magician <magic-modules@google.com>

.changelog/11053.txt

@@ -0,0 +1,3 @@
+```release-note:new-datasource
+`google_kms_crypto_keys`
+```

google/provider/provider_mmv1_resources.go

@@ -232,6 +232,7 @@ var handwrittenDatasources = map[string]*schema.Resource{
 	"google_iam_testable_permissions":                     resourcemanager.DataSourceGoogleIamTestablePermissions(),
 	"google_iap_client":                                   iap.DataSourceGoogleIapClient(),
 	"google_kms_crypto_key":                               kms.DataSourceGoogleKmsCryptoKey(),
+	"google_kms_crypto_keys":                              kms.DataSourceGoogleKmsCryptoKeys(),
 	"google_kms_crypto_key_version":                       kms.DataSourceGoogleKmsCryptoKeyVersion(),
 	"google_kms_key_ring":                                 kms.DataSourceGoogleKmsKeyRing(),
 	"google_kms_secret":                                   kms.DataSourceGoogleKmsSecret(),

google/services/kms/data_source_google_kms_crypto_keys.go

@@ -0,0 +1,207 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package kms
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"regexp"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-provider-google/google/tpgresource"
+	transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
+)
+
+func DataSourceGoogleKmsCryptoKeys() *schema.Resource {
+	dsSchema := tpgresource.DatasourceSchemaFromResourceSchema(ResourceKMSCryptoKey().Schema)
+	tpgresource.AddOptionalFieldsToSchema(dsSchema, "name")
+	tpgresource.AddOptionalFieldsToSchema(dsSchema, "key_ring")
+
+	// We need to explicitly add the id field to the schema used for individual keys
+	// Currently the id field in the google_kms_crypto_key resource is implied/added by the SDK
+	dsSchema["id"] = &schema.Schema{
+		Type:     schema.TypeString,
+		Computed: true,
+	}
+
+	return &schema.Resource{
+		Read: dataSourceGoogleKmsCryptoKeysRead,
+		Schema: map[string]*schema.Schema{
+			"key_ring": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: `The key ring that the keys belongs to. Format: 'projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}'.`,
+			},
+			"filter": {
+				Type:     schema.TypeString,
+				Optional: true,
+				Description: `
+					The filter argument is used to add a filter query parameter that limits which keys are retrieved by the data source: ?filter={{filter}}.
+					Example values:
+					
+					* "name:my-key-" will retrieve keys that contain "my-key-" anywhere in their name. Note: names take the form projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}/cryptoKeys/{{cryptoKey}}.
+					* "name=projects/my-project/locations/global/keyRings/my-key-ring/cryptoKeys/my-key-1" will only retrieve a key with that exact name.
+					
+					[See the documentation about using filters](https://cloud.google.com/kms/docs/sorting-and-filtering)
+				`,
+			},
+			"keys": {
+				Type:        schema.TypeList,
+				Computed:    true,
+				Description: "A list of all the retrieved keys from the provided key ring",
+				Elem: &schema.Resource{
+					Schema: dsSchema,
+				},
+			},
+		},
+	}
+}
+
+func dataSourceGoogleKmsCryptoKeysRead(d *schema.ResourceData, meta interface{}) error {
+	config := meta.(*transport_tpg.Config)
+
+	keyRingId, err := parseKmsKeyRingId(d.Get("key_ring").(string), config)
+	if err != nil {
+		return err
+	}
+
+	id := fmt.Sprintf("%s/cryptoKeys", keyRingId.KeyRingId())
+	if filter, ok := d.GetOk("filter"); ok {
+		id += "/filter=" + filter.(string)
+	}
+	d.SetId(id)
+
+	log.Printf("[DEBUG] Searching for keys in key ring %s", keyRingId.KeyRingId())
+	keys, err := dataSourceKMSCryptoKeysList(d, meta, keyRingId.KeyRingId())
+	if err != nil {
+		return err
+	}
+
+	if len(keys) > 0 {
+		log.Printf("[DEBUG] Found %d keys in key ring %s", len(keys), keyRingId.KeyRingId())
+		value, err := flattenKMSKeysList(d, config, keys, keyRingId.KeyRingId())
+		if err != nil {
+			return fmt.Errorf("error flattening keys list: %s", err)
+		}
+		if err := d.Set("keys", value); err != nil {
+			return fmt.Errorf("error setting keys: %s", err)
+		}
+	} else {
+		log.Printf("[DEBUG] Found 0 keys in key ring %s", keyRingId.KeyRingId())
+	}
+
+	return nil
+}
+
+// dataSourceKMSCryptoKeysList calls the list endpoint for Crypto Key resources and collects all keys in a slice.
+// This function handles pagination by collecting the resources returned by multiple calls to the list endpoint.
+// This function also handles server-side filtering by setting the filter query parameter on each API call.
+func dataSourceKMSCryptoKeysList(d *schema.ResourceData, meta interface{}, keyRingId string) ([]interface{}, error) {
+	config := meta.(*transport_tpg.Config)
+	userAgent, err := tpgresource.GenerateUserAgentString(d, config.UserAgent)
+	if err != nil {
+		return nil, err
+	}
+
+	url, err := tpgresource.ReplaceVars(d, config, "{{KMSBasePath}}{{key_ring}}/cryptoKeys")
+	if err != nil {
+		return nil, err
+	}
+
+	billingProject := ""
+
+	if parts := regexp.MustCompile(`projects\/([^\/]+)\/`).FindStringSubmatch(url); parts != nil {
+		billingProject = parts[1]
+	}
+
+	// err == nil indicates that the billing_project value was found
+	if bp, err := tpgresource.GetBillingProject(d, config); err == nil {
+		billingProject = bp
+	}
+
+	// Always include the filter param, and optionally include the pageToken parameter for subsequent requests
+	var params = make(map[string]string, 0)
+	if filter, ok := d.GetOk("filter"); ok {
+		log.Printf("[DEBUG] Search for keys in key ring %s is using filter ?filter=%s", keyRingId, filter.(string))
+		params["filter"] = filter.(string)
+	}
+
+	cryptoKeys := make([]interface{}, 0)
+	for {
+		// Depending on previous iterations, params might contain a pageToken param
+		url, err = transport_tpg.AddQueryParams(url, params)
+		if err != nil {
+			return nil, err
+		}
+
+		headers := make(http.Header)
+		res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
+			Config:    config,
+			Method:    "GET",
+			Project:   billingProject,
+			RawURL:    url,
+			UserAgent: userAgent,
+			Headers:   headers,
+			// ErrorRetryPredicates used to allow retrying if rate limits are hit when requesting multiple pages in a row
+			ErrorRetryPredicates: []transport_tpg.RetryErrorPredicateFunc{transport_tpg.Is429RetryableQuotaError},
+		})
+		if err != nil {
+			return nil, transport_tpg.HandleNotFoundError(err, d, fmt.Sprintf("KMSCryptoKeys %q", d.Id()))
+		}
+
+		if res == nil {
+			// Decoding the object has resulted in it being gone. It may be marked deleted
+			log.Printf("[DEBUG] Removing KMSCryptoKey because it no longer exists.")
+			d.SetId("")
+			return nil, nil
+		}
+
+		// Store info from this page
+		if v, ok := res["cryptoKeys"].([]interface{}); ok {
+			cryptoKeys = append(cryptoKeys, v...)
+		}
+
+		// Handle pagination for next loop, or break loop
+		v, ok := res["nextPageToken"]
+		if ok {
+			params["pageToken"] = v.(string)
+		}
+		if !ok {
+			break
+		}
+	}
+	return cryptoKeys, nil
+}
+
+// flattenKMSKeysList flattens a list of crypto keys from a given crypto key ring
+func flattenKMSKeysList(d *schema.ResourceData, config *transport_tpg.Config, keysList []interface{}, keyRingId string) ([]interface{}, error) {
+	var keys []interface{}
+	for _, k := range keysList {
+		key := k.(map[string]interface{})
+		parsedId, err := ParseKmsCryptoKeyId(key["name"].(string), config)
+		if err != nil {
+			return nil, err
+		}
+
+		data := map[string]interface{}{}
+		// The google_kms_crypto_key resource and dataset set
+		// id as the value of name (projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}/cryptoKeys/{{name}})
+		// and set name is set as just {{name}}.
+		data["id"] = key["name"]
+		data["name"] = parsedId.Name
+		data["key_ring"] = keyRingId
+
+		data["labels"] = flattenKMSCryptoKeyLabels(key["labels"], d, config)
+		data["primary"] = flattenKMSCryptoKeyPrimary(key["primary"], d, config)
+		data["purpose"] = flattenKMSCryptoKeyPurpose(key["purpose"], d, config)
+		data["rotation_period"] = flattenKMSCryptoKeyRotationPeriod(key["rotationPeriod"], d, config)
+		data["version_template"] = flattenKMSCryptoKeyVersionTemplate(key["versionTemplate"], d, config)
+		data["destroy_scheduled_duration"] = flattenKMSCryptoKeyDestroyScheduledDuration(key["destroyScheduledDuration"], d, config)
+		data["import_only"] = flattenKMSCryptoKeyImportOnly(key["importOnly"], d, config)
+		data["crypto_key_backend"] = flattenKMSCryptoKeyCryptoKeyBackend(key["cryptoKeyBackend"], d, config)
+		keys = append(keys, data)
+	}
+
+	return keys, nil
+}

google/services/kms/data_source_google_kms_crypto_keys_test.go

@@ -0,0 +1,74 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: MPL-2.0
+package kms_test
+
+import (
+	"fmt"
+	"regexp"
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+	"github.com/hashicorp/terraform-provider-google/google/acctest"
+)
+
+func TestAccDataSourceGoogleKmsCryptoKeys_basic(t *testing.T) {
+	kms := acctest.BootstrapKMSKey(t)
+
+	id := kms.KeyRing.Name + "/cryptoKeys"
+
+	randomString := acctest.RandString(t, 10)
+	filterNameFindSharedKeys := "name:tftest-shared-"
+	filterNameFindsNoKeys := fmt.Sprintf("name:%s", randomString)
+
+	findSharedKeysId := fmt.Sprintf("%s/filter=%s", id, filterNameFindSharedKeys)
+	findsNoKeysId := fmt.Sprintf("%s/filter=%s", id, filterNameFindsNoKeys)
+
+	context := map[string]interface{}{
+		"key_ring": kms.KeyRing.Name,
+		"filter":   "", // Can be overridden using 2nd argument to config funcs
+	}
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccDataSourceGoogleKmsCryptoKeys_basic(context, ""),
+				Check: resource.ComposeTestCheckFunc(
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "id", id),
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "key_ring", kms.KeyRing.Name),
+					resource.TestMatchResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "keys.#", regexp.MustCompile("[1-9]+[0-9]*")),
+				),
+			},
+			{
+				Config: testAccDataSourceGoogleKmsCryptoKeys_basic(context, fmt.Sprintf("filter = \"%s\"", filterNameFindSharedKeys)),
+				Check: resource.ComposeTestCheckFunc(
+					// This filter should retrieve keys in the bootstrapped KMS key ring used by the test
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "id", findSharedKeysId),
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "key_ring", kms.KeyRing.Name),
+					resource.TestMatchResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "keys.#", regexp.MustCompile("[1-9]+[0-9]*")),
+				),
+			},
+			{
+				Config: testAccDataSourceGoogleKmsCryptoKeys_basic(context, fmt.Sprintf("filter = \"%s\"", filterNameFindsNoKeys)),
+				Check: resource.ComposeTestCheckFunc(
+					// This filter should retrieve no keys
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "id", findsNoKeysId),
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "key_ring", kms.KeyRing.Name),
+					resource.TestCheckResourceAttr("data.google_kms_crypto_keys.all_keys_in_ring", "keys.#", "0"),
+				),
+			},
+		},
+	})
+}
+
+func testAccDataSourceGoogleKmsCryptoKeys_basic(context map[string]interface{}, filter string) string {
+	context["filter"] = filter
+
+	return acctest.Nprintf(`
+data "google_kms_crypto_keys" "all_keys_in_ring" {
+  key_ring = "%{key_ring}"
+  %{filter}
+}
+`, context)
+}

google/transport/error_retry_predicates.go

@@ -455,6 +455,31 @@ func Is429QuotaError(err error) (bool, string) {
 	return false, ""
 }
 
+// Do retry if operation returns a 429 and the reason is RATE_LIMIT_EXCEEDED
+func Is429RetryableQuotaError(err error) (bool, string) {
+	if gerr, ok := err.(*googleapi.Error); ok {
+		if gerr.Code == 429 {
+			// Quota error isn't necessarily retryable if it's a resource instance limit; check details
+			isRateLimitExceeded := false
+			for _, d := range gerr.Details {
+				data := d.(map[string]interface{})
+				dType, ok := data["@type"]
+				// Find google.rpc.ErrorInfo in Details
+				if ok && strings.Contains(dType.(string), "ErrorInfo") {
+					if v, ok := data["reason"]; ok {
+						if v.(string) == "RATE_LIMIT_EXCEEDED" {
+							isRateLimitExceeded = true
+							break
+						}
+					}
+				}
+			}
+			return isRateLimitExceeded, "429s are retryable for this resource, but only if the reason is RATE_LIMIT_EXCEEDED"
+		}
+	}
+	return false, ""
+}
+
 // Retry if App Engine operation returns a 409 with a specific message for
 // concurrent operations, or a 404 indicating p4sa has not yet propagated.
 func IsAppEngineRetryableError(err error) (bool, string) {

website/docs/d/kms_crypto_keys.html.markdown

@@ -0,0 +1,56 @@
+---
+subcategory: "Cloud Key Management Service"
+description: |-
+ Provides access to data about all KMS keys within a key ring with Google Cloud KMS.
+---
+
+# google_kms_crypto_keys
+
+Provides access to all Google Cloud Platform KMS CryptoKeys in a given KeyRing. For more information see
+[the official documentation](https://cloud.google.com/kms/docs/object-hierarchy#key)
+and
+[API](https://cloud.google.com/kms/docs/reference/rest/v1/projects.locations.keyRings.cryptoKeys).
+
+A CryptoKey is an interface to key material which can be used to encrypt and decrypt data. A CryptoKey belongs to a
+Google Cloud KMS KeyRing.
+
+## Example Usage
+
+```hcl
+// Get all keys in the key ring
+data "google_kms_crypto_keys" "all_crypto_keys" {
+  key_ring = data.google_kms_key_ring.my_key_ring.id
+}
+
+// Get keys in the key ring that have "foobar" in their name
+data "google_kms_crypto_keys" "all_crypto_keys" {
+  key_ring = data.google_kms_key_ring.my_key_ring.id
+  filter   = "name:foobar"
+}
+```
+
+## Argument Reference
+
+The following arguments are supported:
+
+* `key_ring` - (Required) The key ring that the keys belongs to. Format: 'projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}'.,
+
+* `filter` - (Optional) The filter argument is used to add a filter query parameter that limits which keys are retrieved by the data source: ?filter={{filter}}. When no value is provided there is no filtering.
+
+Example filter values if filtering on name. Note: names take the form projects/{{project}}/locations/{{location}}/keyRings/{{keyRing}}/cryptoKeys/{{cryptoKey}}.
+
+* `"name:my-key-"` will retrieve keys that contain "my-key-" anywhere in their name.
+* `"name=projects/my-project/locations/global/keyRings/my-key-ring/cryptoKeys/my-key-1"` will only retrieve a key with that exact name.
+
+[See the documentation about using filters](https://cloud.google.com/kms/docs/sorting-and-filtering)
+
+
+
+## Attributes Reference
+
+In addition to the arguments listed above, the following computed attributes are exported:
+
+* `keys` - A list of all the retrieved keys from the provided key ring. This list is influenced by the provided filter argument.
+
+See [google_kms_crypto_key](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/kms_crypto_key) resource for details of the available attributes on each key.
+