From 5a91a9bfbec61a676d61f32c9082def76d45e929 Mon Sep 17 00:00:00 2001 From: Modular Magician Date: Tue, 31 Oct 2023 22:39:08 +0000 Subject: [PATCH] add support for instance_flexibility_policy in google_dataproc_cluster (#9269) Co-authored-by: Cameron Thornton Co-authored-by: sunchengxuanivy [upstream:f17360368c7bb48288f69b09634908a39d010581] Signed-off-by: Modular Magician --- .changelog/9269.txt | 3 + .../dataproc/resource_dataproc_cluster.go | 136 ++++++++++++++++++ .../resource_dataproc_cluster_migrate.go | 68 +++++++++ .../resource_dataproc_cluster_test.go | 64 +++++++++ website/docs/r/dataproc_cluster.html.markdown | 19 ++- 5 files changed, 289 insertions(+), 1 deletion(-) create mode 100644 .changelog/9269.txt diff --git a/.changelog/9269.txt b/.changelog/9269.txt new file mode 100644 index 0000000000..d2232ba557 --- /dev/null +++ b/.changelog/9269.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +dataproc: added support for `instance_flexibility_policy` in `google_dataproc_cluster` +``` diff --git a/google-beta/services/dataproc/resource_dataproc_cluster.go b/google-beta/services/dataproc/resource_dataproc_cluster.go index b91a4899c8..a3fb57f765 100644 --- a/google-beta/services/dataproc/resource_dataproc_cluster.go +++ b/google-beta/services/dataproc/resource_dataproc_cluster.go @@ -1141,6 +1141,74 @@ func ResourceDataprocCluster() *schema.Resource { Elem: &schema.Schema{Type: schema.TypeString}, Description: `List of preemptible instance names which have been assigned to the cluster.`, }, + "instance_flexibility_policy": { + Type: schema.TypeList, + Optional: true, + Computed: true, + Description: `Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.`, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", + "cluster_config.0.preemptible_worker_config.0.disk_config", + "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy", + }, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_selection_list": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list", + }, + Description: `List of instance selection options that the group will use when creating new VMs.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_types": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: `Full machine-type names, e.g. "n1-standard-16".`, + }, + "rank": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeInt}, + Description: `Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.`, + }, + }, + }, + }, + "instance_selection_results": { + Type: schema.TypeList, + Computed: true, + Description: `A list of instance selection results in the group.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_type": { + Type: schema.TypeString, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: `Full machine-type names, e.g. "n1-standard-16".`, + }, + "vm_count": { + Type: schema.TypeInt, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeInt}, + Description: `Number of VM provisioned with the machine_type.`, + }, + }, + }, + }, + }, + }, + }, }, }, }, @@ -2097,12 +2165,46 @@ func expandPreemptibleInstanceGroupConfig(cfg map[string]interface{}) *dataproc. } } } + + if ifpc, ok := cfg["instance_flexibility_policy"]; ok { + ifps := ifpc.([]interface{}) + if len(ifps) > 0 { + flexibilityPolicy := ifps[0].(map[string]interface{}) + icg.InstanceFlexibilityPolicy = &dataproc.InstanceFlexibilityPolicy{} + if v, ok := flexibilityPolicy["instance_selection_list"]; ok { + icg.InstanceFlexibilityPolicy.InstanceSelectionList = expandInstanceSelectionList(v) + } + } + + } if p, ok := cfg["preemptibility"]; ok { icg.Preemptibility = p.(string) } return icg } +func expandInstanceSelectionList(v interface{}) []*dataproc.InstanceSelection { + instanceSelectionList := v.([]interface{}) + + instanceSelections := []*dataproc.InstanceSelection{} + for _, v1 := range instanceSelectionList { + instanceSelectionItem := v1.(map[string]interface{}) + machineTypes := []string{} + for _, machineType := range instanceSelectionItem["machine_types"].([]interface{}) { + machineTypes = append(machineTypes, machineType.(string)) + } + instanceSelection := &dataproc.InstanceSelection{ + MachineTypes: machineTypes, + } + if x, ok := instanceSelectionItem["rank"]; ok { + instanceSelection.Rank = int64(x.(int)) + } + instanceSelections = append(instanceSelections, instanceSelection) + } + + return instanceSelections +} + func expandMasterInstanceGroupConfig(cfg map[string]interface{}) *dataproc.InstanceGroupConfig { icg := &dataproc.InstanceGroupConfig{} @@ -2754,6 +2856,7 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc } disk := map[string]interface{}{} + instanceFlexibilityPolicy := map[string]interface{}{} data := map[string]interface{}{} if icg != nil { @@ -2765,12 +2868,45 @@ func flattenPreemptibleInstanceGroupConfig(d *schema.ResourceData, icg *dataproc disk["num_local_ssds"] = icg.DiskConfig.NumLocalSsds disk["boot_disk_type"] = icg.DiskConfig.BootDiskType } + if icg.InstanceFlexibilityPolicy != nil { + instanceFlexibilityPolicy["instance_selection_list"] = flattenInstanceSelectionList(icg.InstanceFlexibilityPolicy.InstanceSelectionList) + instanceFlexibilityPolicy["instance_selection_results"] = flattenInstanceSelectionResults(icg.InstanceFlexibilityPolicy.InstanceSelectionResults) + } } data["disk_config"] = []map[string]interface{}{disk} + data["instance_flexibility_policy"] = []map[string]interface{}{instanceFlexibilityPolicy} return []map[string]interface{}{data} } +func flattenInstanceSelectionList(is []*dataproc.InstanceSelection) []map[string]interface{} { + + instanceSelections := []map[string]interface{}{} + for _, v := range is { + instanceSelection := map[string]interface{}{} + if len(v.MachineTypes) > 0 { + instanceSelection["machine_types"] = v.MachineTypes + } + instanceSelection["rank"] = v.Rank + instanceSelections = append(instanceSelections, instanceSelection) + } + return instanceSelections + +} + +func flattenInstanceSelectionResults(isr []*dataproc.InstanceSelectionResult) []map[string]interface{} { + + instanceSelectionResults := []map[string]interface{}{} + for _, v := range isr { + instanceSelection := map[string]interface{}{} + instanceSelection["machine_type"] = v.MachineType + instanceSelection["vm_count"] = v.VmCount + instanceSelectionResults = append(instanceSelectionResults, instanceSelection) + } + return instanceSelectionResults + +} + func flattenMasterInstanceGroupConfig(d *schema.ResourceData, icg *dataproc.InstanceGroupConfig) []map[string]interface{} { disk := map[string]interface{}{} data := map[string]interface{}{} diff --git a/google-beta/services/dataproc/resource_dataproc_cluster_migrate.go b/google-beta/services/dataproc/resource_dataproc_cluster_migrate.go index 2c0f21a5e7..a5ca8bc029 100644 --- a/google-beta/services/dataproc/resource_dataproc_cluster_migrate.go +++ b/google-beta/services/dataproc/resource_dataproc_cluster_migrate.go @@ -963,6 +963,74 @@ func resourceDataprocClusterResourceV0() *schema.Resource { Elem: &schema.Schema{Type: schema.TypeString}, Description: `List of preemptible instance names which have been assigned to the cluster.`, }, + "instance_flexibility_policy": { + Type: schema.TypeList, + Optional: true, + Computed: true, + Description: `Instance flexibility Policy allowing a mixture of VM shapes and provisioning models.`, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.num_instances", + "cluster_config.0.preemptible_worker_config.0.preemptibility", + "cluster_config.0.preemptible_worker_config.0.disk_config", + "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy", + }, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "instance_selection_list": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + AtLeastOneOf: []string{ + "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list", + }, + Description: `List of instance selection options that the group will use when creating new VMs.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_types": { + Type: schema.TypeList, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: `Full machine-type names, e.g. "n1-standard-16".`, + }, + "rank": { + Type: schema.TypeInt, + Computed: true, + Optional: true, + ForceNew: true, + Elem: &schema.Schema{Type: schema.TypeInt}, + Description: `Preference of this instance selection. Lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference.`, + }, + }, + }, + }, + "instance_selection_results": { + Type: schema.TypeList, + Computed: true, + Description: `A list of instance selection results in the group.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "machine_type": { + Type: schema.TypeString, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: `Full machine-type names, e.g. "n1-standard-16".`, + }, + "vm_count": { + Type: schema.TypeInt, + Computed: true, + Elem: &schema.Schema{Type: schema.TypeInt}, + Description: `Number of VM provisioned with the machine_type.`, + }, + }, + }, + }, + }, + }, + }, }, }, }, diff --git a/google-beta/services/dataproc/resource_dataproc_cluster_test.go b/google-beta/services/dataproc/resource_dataproc_cluster_test.go index f54560dfb9..6b418859b7 100644 --- a/google-beta/services/dataproc/resource_dataproc_cluster_test.go +++ b/google-beta/services/dataproc/resource_dataproc_cluster_test.go @@ -492,6 +492,29 @@ func TestAccDataprocCluster_spotSecondary(t *testing.T) { }) } +func TestAccDataprocCluster_spotWithInstanceFlexibilityPolicy(t *testing.T) { + t.Parallel() + + rnd := acctest.RandString(t, 10) + var cluster dataproc.Cluster + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataprocClusterDestroy(t), + Steps: []resource.TestStep{ + { + Config: testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd), + Check: resource.ComposeTestCheckFunc( + testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.spot_with_instance_flexibility_policy", &cluster), + resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.preemptibility", "SPOT"), + resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.machine_types.0", "n2d-standard-2"), + resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.instance_flexibility_policy.0.instance_selection_list.0.rank", "3"), + ), + }, + }, + }) +} + func TestAccDataprocCluster_withStagingBucket(t *testing.T) { t.Parallel() @@ -1816,6 +1839,47 @@ resource "google_dataproc_cluster" "spot_secondary" { `, rnd, subnetworkName) } +func testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd string) string { + return fmt.Sprintf(` +resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" { + name = "tf-test-dproc-%s" + region = "us-central1" + + cluster_config { + master_config { + num_instances = "1" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + worker_config { + num_instances = "2" + machine_type = "e2-medium" + disk_config { + boot_disk_size_gb = 35 + } + } + + preemptible_worker_config { + num_instances = "3" + preemptibility = "SPOT" + disk_config { + boot_disk_size_gb = 35 + } + instance_flexibility_policy { + instance_selection_list { + machine_types = ["n2d-standard-2"] + rank = 3 + } + } + } + } +} + `, rnd) +} + func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string { return fmt.Sprintf(` resource "google_storage_bucket" "bucket" { diff --git a/website/docs/r/dataproc_cluster.html.markdown b/website/docs/r/dataproc_cluster.html.markdown index e55b431059..32762397ff 100644 --- a/website/docs/r/dataproc_cluster.html.markdown +++ b/website/docs/r/dataproc_cluster.html.markdown @@ -14,7 +14,7 @@ Manages a Cloud Dataproc cluster resource within GCP. !> **Warning:** Due to limitations of the API, all arguments except -`labels`,`cluster_config.worker_config.num_instances` and `cluster_config.preemptible_worker_config.num_instances` are non-updatable. Changing others will cause recreation of the +`labels`,`cluster_config.worker_config.num_instances` and `cluster_config.preemptible_worker_config.num_instances` are non-updatable. Changing `cluster_config.worker_config.min_num_instances` will be ignored. Changing others will cause recreation of the whole cluster! ## Example Usage - Basic @@ -608,6 +608,16 @@ cluster_config { boot_disk_size_gb = 30 num_local_ssds = 1 } + instance_flexibility_policy { + instance_selection_list { + machine_types = ["n2-standard-2","n1-standard-2"] + rank = 1 + } + instance_selection_list { + machine_types = ["n2d-standard-2"] + rank = 3 + } + } } } ``` @@ -638,6 +648,13 @@ will be set for you based on whatever was set for the `worker_config.machine_typ * `num_local_ssds` - (Optional) The amount of local SSD disks that will be attached to each preemptible worker node. Defaults to 0. +* `instance_flexibility_policy` (Optional) Instance flexibility Policy allowing a mixture of VM shapes and provisioning models. + + * `instance_selection_list` - (Optional) List of instance selection options that the group will use when creating new VMs. + * `machine_types` - (Optional) Full machine-type names, e.g. `"n1-standard-16"`. + + * `rank` - (Optional) Preference of this instance selection. A lower number means higher preference. Dataproc will first try to create a VM based on the machine-type with priority rank and fallback to next rank based on availability. Machine types and instance selections with the same priority have the same preference. + - - - The `cluster_config.software_config` block supports: