From 058028c9a2e432f874744cc89bde9828b31fad98 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 22 Dec 2023 14:38:46 +0000
Subject: [PATCH 01/92] Set min nodes to 0 for worker and user.

---
 src/_nebari/stages/infrastructure/__init__.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 1e34cb05ef..82ce8fc3bd 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -438,10 +438,10 @@ class AmazonWebServicesProvider(schema.Base):
     node_groups: Dict[str, AWSNodeGroup] = {
         "general": AWSNodeGroup(instance="m5.2xlarge", min_nodes=1, max_nodes=1),
         "user": AWSNodeGroup(
-            instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False
+            instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False
         ),
         "worker": AWSNodeGroup(
-            instance="m5.xlarge", min_nodes=1, max_nodes=5, single_subnet=False
+            instance="m5.xlarge", min_nodes=0, max_nodes=5, single_subnet=False
         ),
     }
     existing_subnet_ids: List[str] = None

From 1b8a6c12a6e27f4ca1d8cb8c67920d027d163f9e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 3 Jan 2024 11:10:12 +0000
Subject: [PATCH 02/92] Move user scheduler to general and add tags and lables
 to asg and node groups to support scaling from zero.

---
 .../template/aws/modules/kubernetes/main.tf   | 35 +++++++++++++++++++
 .../kubernetes/services/jupyterhub/main.tf    |  4 +--
 2 files changed, 37 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index fd53a636c2..878d2f47db 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -39,6 +39,10 @@ resource "aws_eks_node_group" "main" {
     max_size     = var.node_groups[count.index].max_size
   }
 
+  labels = {
+    "dedicated" = var.node_groups[count.index].name
+  }
+
   lifecycle {
     ignore_changes = [
       scaling_config[0].desired_size,
@@ -54,9 +58,40 @@ resource "aws_eks_node_group" "main" {
 
   tags = merge({
     "kubernetes.io/cluster/${var.name}" = "shared"
+    "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
+    propagate_at_launch = true
   }, var.tags)
 }
 
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in aws_eks_node_group.main[1].resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value = var.node_groups[1].name
+    propagate_at_launch = true
+  }
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in aws_eks_node_group.main[2].resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value = var.node_groups[2].name
+    propagate_at_launch = true
+  }
+}
+
+
 data "aws_eks_cluster_auth" "main" {
   name = aws_eks_cluster.main.name
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 3c5822a461..2a3d139801 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -134,14 +134,14 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "${var.user-node-group.key}" = var.user-node-group.value
+          "dedicated" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "${var.user-node-group.key}" = var.user-node-group.value
+            "${var.user-node-group.key}" = var.general-node-group.value
           }
         }
       }

From 26150be549590dd42921e7ea59940bdb739db0a4 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 3 Jan 2024 17:54:48 +0000
Subject: [PATCH 03/92] Change the tag key for autoscheudler to work.

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py    | 2 +-
 .../template/modules/kubernetes/services/jupyterhub/main.tf     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index b1499efe44..a468757cbf 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,7 +115,7 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        config["worker-node-group"]["key"]: config["worker-node-group"]["value"]
+        "dedicated": config["worker-node-group"]["value"]
     }
 
     # check `worker_extra_pod_config` first
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 2a3d139801..b39cda62f7 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -141,7 +141,7 @@ resource "helm_release" "jupyterhub" {
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "${var.user-node-group.key}" = var.general-node-group.value
+            "dedicated" = var.general-node-group.value
           }
         }
       }

From 492835fa42a64d0e52efb82563a4d6ddec74c2a2 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 3 Jan 2024 17:55:32 +0000
Subject: [PATCH 04/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py  | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index a468757cbf..69cf3b5860 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -114,9 +114,7 @@ def list_dask_environments():
 
 
 def base_node_group(options):
-    default_node_group = {
-        "dedicated": config["worker-node-group"]["value"]
-    }
+    default_node_group = {"dedicated": config["worker-node-group"]["value"]}
 
     # check `worker_extra_pod_config` first
     worker_node_group = (

From 8123adc79317c42da7854992272abc2b2a142d47 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 3 Jan 2024 18:14:04 +0000
Subject: [PATCH 05/92] Retain previous node selectors.

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py    | 1 +
 .../template/modules/kubernetes/services/jupyterhub/main.tf     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index a468757cbf..cff5dbb680 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,6 +115,7 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
+        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
         "dedicated": config["worker-node-group"]["value"]
     }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index b39cda62f7..6df868b342 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -134,6 +134,7 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
+          "${var.user-node-group.key}" = var.user-node-group.value
           "dedicated" = var.user-node-group.value
         }
       }
@@ -141,6 +142,7 @@ resource "helm_release" "jupyterhub" {
       scheduling = {
         userScheduler = {
           nodeSelector = {
+            "${var.general-node-group.key}" = var.general-node-group.value
             "dedicated" = var.general-node-group.value
           }
         }

From fd141de556d3a2531ad4a42801d2e969f168173d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 3 Jan 2024 18:19:22 +0000
Subject: [PATCH 06/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index cff5dbb680..b5315b8ea8 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -116,7 +116,7 @@ def list_dask_environments():
 def base_node_group(options):
     default_node_group = {
         config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
-        "dedicated": config["worker-node-group"]["value"]
+        "dedicated": config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first

From 6533542845a815fb994bf423da11c042d0adf738 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 3 Jan 2024 18:26:38 +0000
Subject: [PATCH 07/92] Formatting changes.

---
 .../template/aws/modules/kubernetes/main.tf            | 10 +++++-----
 .../modules/kubernetes/services/jupyterhub/main.tf     |  4 ++--
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index 878d2f47db..040821fc3c 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -57,7 +57,7 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    "kubernetes.io/cluster/${var.name}" = "shared"
+    "kubernetes.io/cluster/${var.name}"                       = "shared"
     "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
     propagate_at_launch = true
   }, var.tags)
@@ -71,8 +71,8 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
   )
   autoscaling_group_name = each.value
   tag {
-    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value = var.node_groups[1].name
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = var.node_groups[1].name
     propagate_at_launch = true
   }
 }
@@ -85,8 +85,8 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
   )
   autoscaling_group_name = each.value
   tag {
-    key   = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value = var.node_groups[2].name
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = var.node_groups[2].name
     propagate_at_launch = true
   }
 }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 6df868b342..b15b8006d8 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -135,7 +135,7 @@ resource "helm_release" "jupyterhub" {
         image = var.jupyterlab-image
         nodeSelector = {
           "${var.user-node-group.key}" = var.user-node-group.value
-          "dedicated" = var.user-node-group.value
+          "dedicated"                  = var.user-node-group.value
         }
       }
 
@@ -143,7 +143,7 @@ resource "helm_release" "jupyterhub" {
         userScheduler = {
           nodeSelector = {
             "${var.general-node-group.key}" = var.general-node-group.value
-            "dedicated" = var.general-node-group.value
+            "dedicated"                     = var.general-node-group.value
           }
         }
       }

From 7d26843e6d8136c0070ee34e6eecf29e23a75260 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 3 Jan 2024 18:29:01 +0000
Subject: [PATCH 08/92] Formatting changes.

---
 .../infrastructure/template/aws/modules/kubernetes/main.tf      | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index 040821fc3c..067265e2eb 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -59,7 +59,7 @@ resource "aws_eks_node_group" "main" {
   tags = merge({
     "kubernetes.io/cluster/${var.name}"                       = "shared"
     "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
-    propagate_at_launch = true
+    propagate_at_launch                                       = true
   }, var.tags)
 }
 

From 3e5044f3271877b206959b7cb39439385a8bcd32 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 09:33:11 +0000
Subject: [PATCH 09/92] ASG needs to exist before its tagged, moving
 aws_autoscaling_group_tag to 03-kubernetes-initialize.

---
 .../template/aws/modules/kubernetes/main.tf   | 27 -------------
 .../modules/cluster-autoscaler/main.tf        | 38 +++++++++++++++++++
 2 files changed, 38 insertions(+), 27 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index 067265e2eb..a2238a41f8 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -63,33 +63,6 @@ resource "aws_eks_node_group" "main" {
   }, var.tags)
 }
 
-resource "aws_autoscaling_group_tag" "dedicated_user" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in aws_eks_node_group.main[1].resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = var.node_groups[1].name
-    propagate_at_launch = true
-  }
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_worker" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in aws_eks_node_group.main[2].resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = var.node_groups[2].name
-    propagate_at_launch = true
-  }
-}
 
 
 data "aws_eks_cluster_auth" "main" {
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index 29f982c86a..37526eb263 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -22,3 +22,41 @@ resource "helm_release" "autoscaler" {
     })
   ], var.overrides)
 }
+
+data "aws_eks_node_group" "user" {
+  cluster_name    = var.cluster-name
+  node_group_name = "user"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "user"
+    propagate_at_launch = true
+  }
+}
+
+data "aws_eks_node_group" "worker" {
+  cluster_name    = var.cluster-name
+  node_group_name = "worker"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "worker"
+    propagate_at_launch = true
+  }
+}

From 1197fe393dd0655104a50023d358468d242e3e5f Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 09:51:42 +0000
Subject: [PATCH 10/92] Remove origianl node selector. as it prevents the pod
 to match nodeselector.

---
 .../template/modules/kubernetes/services/jupyterhub/main.tf     | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index b15b8006d8..fcbbdeb863 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -134,7 +134,6 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "${var.user-node-group.key}" = var.user-node-group.value
           "dedicated"                  = var.user-node-group.value
         }
       }
@@ -142,7 +141,6 @@ resource "helm_release" "jupyterhub" {
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "${var.general-node-group.key}" = var.general-node-group.value
             "dedicated"                     = var.general-node-group.value
           }
         }

From 1d406a30c355a4a473e93b4e6a00b074645767a3 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 10:06:33 +0000
Subject: [PATCH 11/92] Terraform format changes.

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py  | 1 -
 .../template/modules/kubernetes/services/jupyterhub/main.tf   | 4 ++--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index b5315b8ea8..083741f2a1 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,7 +115,6 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
         "dedicated": config["worker-node-group"]["value"],
     }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index fcbbdeb863..b39cda62f7 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -134,14 +134,14 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "dedicated"                  = var.user-node-group.value
+          "dedicated" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "dedicated"                     = var.general-node-group.value
+            "dedicated" = var.general-node-group.value
           }
         }
       }

From 8ca133ba0ee96455ad934294a65e72f968537a4b Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 17:40:38 +0000
Subject: [PATCH 12/92] Minor formatting changes.

---
 .../template/modules/cluster-autoscaler/main.tf                  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index 37526eb263..896020b842 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -34,6 +34,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
       [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
     ) : asg.name]
   )
+
   autoscaling_group_name = each.value
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"

From a2c891a02eefb538efad1d98cf8528b7c7a85b5d Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 19:07:13 +0000
Subject: [PATCH 13/92] Upgrade provider version.

---
 src/_nebari/stages/infrastructure/template/aws/versions.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/versions.tf b/src/_nebari/stages/infrastructure/template/aws/versions.tf
index 54fc973d6a..a41b02b1b4 100644
--- a/src/_nebari/stages/infrastructure/template/aws/versions.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "3.73.0"
+      version = "5.12.0"
     }
   }
   required_version = ">= 1.0"

From 214b2f330fdc2e190ecc77650630fac386f52897 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 19:32:41 +0000
Subject: [PATCH 14/92] Pin hashicorp/aws version.

---
 src/_nebari/stages/kubernetes_initialize/template/versions.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index 341def1365..b349affe7c 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,6 +8,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.12.0"
+    }
   }
   required_version = ">= 1.0"
 }

From 9d825b6109bc9c22e2d80aa7765eb04c179e272d Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 20:17:15 +0000
Subject: [PATCH 15/92] Pin hashicorp/aws version.

---
 src/_nebari/stages/infrastructure/template/local/main.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf
index 9fd8bb2618..9e7bcd6950 100644
--- a/src/_nebari/stages/infrastructure/template/local/main.tf
+++ b/src/_nebari/stages/infrastructure/template/local/main.tf
@@ -12,6 +12,10 @@ terraform {
       source  = "gavinbunney/kubectl"
       version = ">= 1.7.0"
     }
+    aws = {
+      source  = "hashicorp/aws"
+      version = "5.12.0"
+    }
   }
 }
 

From 0d8b692b89bc6088519c393672d52cb9ebf1d107 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Fri, 5 Jan 2024 20:51:18 +0000
Subject: [PATCH 16/92] Revert Pin hashicorp/aws version.

---
 src/_nebari/stages/infrastructure/template/aws/versions.tf    | 2 +-
 src/_nebari/stages/infrastructure/template/local/main.tf      | 4 ----
 src/_nebari/stages/kubernetes_initialize/template/versions.tf | 4 ----
 3 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/versions.tf b/src/_nebari/stages/infrastructure/template/aws/versions.tf
index a41b02b1b4..54fc973d6a 100644
--- a/src/_nebari/stages/infrastructure/template/aws/versions.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/versions.tf
@@ -2,7 +2,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "5.12.0"
+      version = "3.73.0"
     }
   }
   required_version = ">= 1.0"
diff --git a/src/_nebari/stages/infrastructure/template/local/main.tf b/src/_nebari/stages/infrastructure/template/local/main.tf
index 9e7bcd6950..9fd8bb2618 100644
--- a/src/_nebari/stages/infrastructure/template/local/main.tf
+++ b/src/_nebari/stages/infrastructure/template/local/main.tf
@@ -12,10 +12,6 @@ terraform {
       source  = "gavinbunney/kubectl"
       version = ">= 1.7.0"
     }
-    aws = {
-      source  = "hashicorp/aws"
-      version = "5.12.0"
-    }
   }
 }
 
diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index b349affe7c..341def1365 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,10 +8,6 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
-    aws = {
-      source  = "hashicorp/aws"
-      version = "5.12.0"
-    }
   }
   required_version = ">= 1.0"
 }

From 4f82abf62542d5918e7e4c6b7033e5a2a2cd1692 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 11:00:53 +0000
Subject: [PATCH 17/92] Pin hashicorp/aws version.

---
 .../stages/kubernetes_initialize/template/versions.tf       | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index 341def1365..cc093e216b 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,6 +8,12 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
+    required_providers {
+      aws = {
+        source  = "hashicorp/aws"
+        version = "3.73.0"
+      }
+    }
   }
   required_version = ">= 1.0"
 }

From b04cd1bd8bee833dad66d8d604d995233d3009d1 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 11:11:12 +0000
Subject: [PATCH 18/92] Pin hashicorp/aws version.

---
 .../stages/kubernetes_initialize/template/versions.tf     | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index cc093e216b..33b30e0c1f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,11 +8,9 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
-    required_providers {
-      aws = {
-        source  = "hashicorp/aws"
-        version = "3.73.0"
-      }
+    aws = {
+      source  = "hashicorp/aws"
+      version = "3.73.0"
     }
   }
   required_version = ">= 1.0"

From 4855087090d4ea4b0ee75264cfc39cf8de22479b Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 13:04:07 +0000
Subject: [PATCH 19/92] Set aws region.

---
 .../template/modules/cluster-autoscaler/main.tf               | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index 896020b842..fb9231b967 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -1,3 +1,7 @@
+provider "aws" {
+  region = var.aws_region
+}
+
 resource "helm_release" "autoscaler" {
   name      = "cluster-autoscaler"
   namespace = var.namespace

From 1850624895b02a4013561448bc0113cbe24a4a6a Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 13:44:21 +0000
Subject: [PATCH 20/92] Move aws_autoscaling_group_tag to initilization stage.

---
 .../infrastructure/template/aws/main.tf       | 39 +++++++++++++++++++
 .../modules/cluster-autoscaler/main.tf        | 38 ------------------
 2 files changed, 39 insertions(+), 38 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf
index a2c0e929d7..6527239a2a 100644
--- a/src/_nebari/stages/infrastructure/template/aws/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/main.tf
@@ -95,3 +95,42 @@ module "kubernetes" {
   endpoint_private_access = var.eks_endpoint_private_access
   public_access_cidrs     = var.eks_public_access_cidrs
 }
+
+data "aws_eks_node_group" "user" {
+  cluster_name    = local.cluster_name
+  node_group_name = "user"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "user"
+    propagate_at_launch = true
+  }
+}
+
+data "aws_eks_node_group" "worker" {
+  cluster_name    = var.cluster-name
+  node_group_name = "worker"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "worker"
+    propagate_at_launch = true
+  }
+}
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index fb9231b967..53a9141191 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -27,41 +27,3 @@ resource "helm_release" "autoscaler" {
   ], var.overrides)
 }
 
-data "aws_eks_node_group" "user" {
-  cluster_name    = var.cluster-name
-  node_group_name = "user"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_user" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "user"
-    propagate_at_launch = true
-  }
-}
-
-data "aws_eks_node_group" "worker" {
-  cluster_name    = var.cluster-name
-  node_group_name = "worker"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_worker" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "worker"
-    propagate_at_launch = true
-  }
-}

From ee3b17141ecbd73b6204b8b433916c6309b4b5a8 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sat, 6 Jan 2024 13:44:37 +0000
Subject: [PATCH 21/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 .../template/modules/cluster-autoscaler/main.tf                  | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index 53a9141191..797993c297 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -26,4 +26,3 @@ resource "helm_release" "autoscaler" {
     })
   ], var.overrides)
 }
-

From 30c0025f0956209b90883b40cdf78f9db3d30331 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 14:21:24 +0000
Subject: [PATCH 22/92] Revert aws version change.

---
 src/_nebari/stages/kubernetes_initialize/template/versions.tf | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index 33b30e0c1f..341def1365 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,10 +8,6 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
-    aws = {
-      source  = "hashicorp/aws"
-      version = "3.73.0"
-    }
   }
   required_version = ">= 1.0"
 }

From 2e18b86b796380b84ec75d63a434b15aaa2fca23 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sat, 6 Jan 2024 14:25:17 +0000
Subject: [PATCH 23/92] Revert aws version change.

---
 .../template/modules/cluster-autoscaler/main.tf               | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
index 797993c297..29f982c86a 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/cluster-autoscaler/main.tf
@@ -1,7 +1,3 @@
-provider "aws" {
-  region = var.aws_region
-}
-
 resource "helm_release" "autoscaler" {
   name      = "cluster-autoscaler"
   namespace = var.namespace

From 73fbc6c2d9672d6ce0322fcf760d4c2402c4ae06 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 11:25:48 +0000
Subject: [PATCH 24/92] Fix terrafrom template.

---
 src/_nebari/stages/infrastructure/template/aws/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf
index 6527239a2a..e0c68ec43d 100644
--- a/src/_nebari/stages/infrastructure/template/aws/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/main.tf
@@ -117,7 +117,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
-  cluster_name    = var.cluster-name
+  cluster_name    = local.cluster_name
   node_group_name = "worker"
 }
 

From bfe9ce29aedf39eff8b4327af2e7aa62bfa8e619 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 12:52:15 +0000
Subject: [PATCH 25/92] Move aws_autoscaling_group_tag back to  kubernetes
 stage.

---
 .../infrastructure/template/aws/main.tf       | 38 --------------
 .../template/aws/modules/kubernetes/main.tf   | 52 +++++++++++++++++++
 .../aws/modules/kubernetes/outputs.tf         |  5 ++
 3 files changed, 57 insertions(+), 38 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf
index e0c68ec43d..4d129a2523 100644
--- a/src/_nebari/stages/infrastructure/template/aws/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/main.tf
@@ -96,41 +96,3 @@ module "kubernetes" {
   public_access_cidrs     = var.eks_public_access_cidrs
 }
 
-data "aws_eks_node_group" "user" {
-  cluster_name    = local.cluster_name
-  node_group_name = "user"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_user" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "user"
-    propagate_at_launch = true
-  }
-}
-
-data "aws_eks_node_group" "worker" {
-  cluster_name    = local.cluster_name
-  node_group_name = "worker"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_worker" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "worker"
-    propagate_at_launch = true
-  }
-}
diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index a2238a41f8..c2ed3cc4ae 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -95,3 +95,55 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" {
     var.tags
   )
 }
+data "aws_eks_node_group" "user" {
+  cluster_name    = aws_eks_cluster.main.name
+  node_group_name = "user"
+  depends_on = [
+    aws_eks_node_group.main
+  ]
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "user"
+    propagate_at_launch = true
+  }
+  depends_on = [
+    aws_eks_node_group.main,
+    data.aws_eks_node_group.user
+  ]
+}
+
+data "aws_eks_node_group" "worker" {
+  cluster_name    = aws_eks_cluster.main.name
+  node_group_name = "worker"
+  depends_on = [
+    aws_eks_node_group.main
+  ]
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "worker"
+    propagate_at_launch = true
+  }
+  depends_on = [
+    data.aws_eks_node_group.worker,
+    aws_eks_node_group.main
+  ]
+}
diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
index 48994bc7a2..ea41c8acd0 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
@@ -23,6 +23,11 @@ output "oidc_provider_arn" {
   value       = aws_iam_openid_connect_provider.oidc_provider.arn
 }
 
+output "cluster_name" {
+  description = "Name cluster"
+  value       = aws_eks_cluster.main.name
+}
+
 # /~https://github.com/terraform-aws-modules/terraform-aws-eks/blob/16f46db94b7158fd762d9133119206aaa7cf6d63/examples/self_managed_node_group/main.tf
 output "kubeconfig" {
   description = "Kubernetes connection configuration kubeconfig"

From 7996faa0b4f8e4c88543cc8ff3999503e6afa2c9 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 7 Jan 2024 12:52:28 +0000
Subject: [PATCH 26/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/infrastructure/template/aws/main.tf | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/main.tf b/src/_nebari/stages/infrastructure/template/aws/main.tf
index 4d129a2523..a2c0e929d7 100644
--- a/src/_nebari/stages/infrastructure/template/aws/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/main.tf
@@ -95,4 +95,3 @@ module "kubernetes" {
   endpoint_private_access = var.eks_endpoint_private_access
   public_access_cidrs     = var.eks_public_access_cidrs
 }
-

From 7c9f6ecdb9a2c6969b7ed2e05894ec94c5470173 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 18:31:02 +0000
Subject: [PATCH 27/92] Add a module for ASG tagging at
 03-kubernetes-initialize/aws-asg-tagging.

---
 .../template/aws/modules/kubernetes/main.tf   | 52 -------------------
 .../kubernetes_initialize/template/main.tf    |  6 +++
 .../modules/tagging/aws-asg-tagging.tf        | 44 ++++++++++++++++
 .../template/modules/tagging/variables.tf     |  4 ++
 4 files changed, 54 insertions(+), 52 deletions(-)
 create mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
 create mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index c2ed3cc4ae..a2238a41f8 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -95,55 +95,3 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" {
     var.tags
   )
 }
-data "aws_eks_node_group" "user" {
-  cluster_name    = aws_eks_cluster.main.name
-  node_group_name = "user"
-  depends_on = [
-    aws_eks_node_group.main
-  ]
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_user" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "user"
-    propagate_at_launch = true
-  }
-  depends_on = [
-    aws_eks_node_group.main,
-    data.aws_eks_node_group.user
-  ]
-}
-
-data "aws_eks_node_group" "worker" {
-  cluster_name    = aws_eks_cluster.main.name
-  node_group_name = "worker"
-  depends_on = [
-    aws_eks_node_group.main
-  ]
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_worker" {
-  for_each = toset(
-    [for asg in flatten(
-      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
-    ) : asg.name]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "worker"
-    propagate_at_launch = true
-  }
-  depends_on = [
-    data.aws_eks_node_group.worker,
-    aws_eks_node_group.main
-  ]
-}
diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 402c68fb3f..816d382692 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -29,3 +29,9 @@ module "nvidia-driver-installer" {
   gpu_enabled          = var.gpu_enabled
   gpu_node_group_names = var.gpu_node_group_names
 }
+
+module "tagging" {
+  count = var.cloud_provider == "aws" ? 1 : 0
+  source = "./modules/tagging"
+  cluster_name = local.cluster_name
+}
\ No newline at end of file
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
new file mode 100644
index 0000000000..27c30f786d
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -0,0 +1,44 @@
+data "aws_eks_node_group" "user" {
+  cluster_name    = var.cluster_name
+  node_group_name = "user"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_user" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "user"
+    propagate_at_launch = true
+  }
+  depends_on = [
+    data.aws_eks_node_group.user
+  ]
+}
+
+data "aws_eks_node_group" "worker" {
+  cluster_name    = var.cluster_name
+  node_group_name = "worker"
+}
+
+resource "aws_autoscaling_group_tag" "dedicated_worker" {
+  for_each = toset(
+    [for asg in flatten(
+      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
+    ) : asg.name]
+  )
+  autoscaling_group_name = each.value
+  tag {
+    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
+    value               = "worker"
+    propagate_at_launch = true
+  }
+  depends_on = [
+    data.aws_eks_node_group.worker,
+  ]
+}
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
new file mode 100644
index 0000000000..00b51c849f
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -0,0 +1,4 @@
+variable "cluster_name" {
+  description = "Name of the cluster."
+  type        = string
+}

From 2f33394ac35d1abab36ddcbb5914f2a5256cbd8d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 7 Jan 2024 18:31:17 +0000
Subject: [PATCH 28/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 816d382692..ebcdec3e43 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -34,4 +34,4 @@ module "tagging" {
   count = var.cloud_provider == "aws" ? 1 : 0
   source = "./modules/tagging"
   cluster_name = local.cluster_name
-}
\ No newline at end of file
+}

From 82289963388db2e3ebb609ca29067cdfb1b2896b Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 19:00:01 +0000
Subject: [PATCH 29/92] Terafromr formatting changes.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index ebcdec3e43..bb57243854 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -31,7 +31,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count = var.cloud_provider == "aws" ? 1 : 0
-  source = "./modules/tagging"
+  count        = var.cloud_provider == "aws" ? 1 : 0
+  source       = "./modules/tagging"
   cluster_name = local.cluster_name
 }

From c375059c79742568daf24ec3abf7906a556b9d50 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 21:29:54 +0000
Subject: [PATCH 30/92] Pin provider version.

---
 src/_nebari/stages/kubernetes_initialize/template/versions.tf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index 341def1365..33b30e0c1f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,6 +8,10 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
+    aws = {
+      source  = "hashicorp/aws"
+      version = "3.73.0"
+    }
   }
   required_version = ">= 1.0"
 }

From 9fe79a2fad861c4ae462672ff44c9481dc7423d5 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 21:54:08 +0000
Subject: [PATCH 31/92] Tagging aws region.

---
 .../stages/kubernetes_initialize/template/main.tf      |  2 ++
 .../template/modules/tagging/variables.tf              | 10 ++++++++++
 2 files changed, 12 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index bb57243854..51375d883c 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -34,4 +34,6 @@ module "tagging" {
   count        = var.cloud_provider == "aws" ? 1 : 0
   source       = "./modules/tagging"
   cluster_name = local.cluster_name
+  aws_region   = var.aws_region
+  cloud_provider       = var.cloud_provider
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index 00b51c849f..8185499bfa 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -2,3 +2,13 @@ variable "cluster_name" {
   description = "Name of the cluster."
   type        = string
 }
+
+variable "aws_region" {
+  description = "AWS region is cloud provider is AWS"
+  type        = string
+}
+
+variable "cloud_provider" {
+  description = "Cloud provider being used in deployment"
+  type        = string
+}

From 55200f3bf4577d8c326af245714845e082ffc9e3 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 22:00:33 +0000
Subject: [PATCH 32/92] FTerraform frmatting changes.

---
 .../stages/kubernetes_initialize/template/main.tf      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 51375d883c..f706f1d59e 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -31,9 +31,9 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count        = var.cloud_provider == "aws" ? 1 : 0
-  source       = "./modules/tagging"
-  cluster_name = local.cluster_name
-  aws_region   = var.aws_region
-  cloud_provider       = var.cloud_provider
+  count          = var.cloud_provider == "aws" ? 1 : 0
+  source         = "./modules/tagging"
+  cluster_name   = local.cluster_name
+  aws_region     = var.aws_region
+  cloud_provider = var.cloud_provider
 }

From 34f318f1c40e3cf1eaeda40db676e5125e5ae30e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Sun, 7 Jan 2024 23:16:09 +0000
Subject: [PATCH 33/92] Tagging changes.

---
 .../kubernetes_initialize/template/main.tf    | 19 ++++++-------------
 .../modules/tagging/aws-asg-tagging.tf        |  1 -
 .../template/modules/tagging/variables.tf     |  5 -----
 3 files changed, 6 insertions(+), 19 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index f706f1d59e..abac45ec36 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -1,17 +1,13 @@
 module "kubernetes-initialization" {
-  source = "./modules/initialization"
-
+  source    = "./modules/initialization"
   namespace = var.environment
   secrets   = []
 }
 
 module "kubernetes-autoscaling" {
-  count = var.cloud_provider == "aws" ? 1 : 0
-
-  source = "./modules/cluster-autoscaler"
-
-  namespace = var.environment
-
+  count        = var.cloud_provider == "aws" ? 1 : 0
+  source       = "./modules/cluster-autoscaler"
+  namespace    = var.environment
   aws_region   = var.aws_region
   cluster-name = local.cluster_name
 }
@@ -21,10 +17,8 @@ module "traefik-crds" {
 }
 
 module "nvidia-driver-installer" {
-  count = var.gpu_enabled ? 1 : 0
-
-  source = "./modules/nvidia-installer"
-
+  count                = var.gpu_enabled ? 1 : 0
+  source               = "./modules/nvidia-installer"
   cloud_provider       = var.cloud_provider
   gpu_enabled          = var.gpu_enabled
   gpu_node_group_names = var.gpu_node_group_names
@@ -34,6 +28,5 @@ module "tagging" {
   count          = var.cloud_provider == "aws" ? 1 : 0
   source         = "./modules/tagging"
   cluster_name   = local.cluster_name
-  aws_region     = var.aws_region
   cloud_provider = var.cloud_provider
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 27c30f786d..a37fe8056a 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -9,7 +9,6 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
       [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
     ) : asg.name]
   )
-
   autoscaling_group_name = each.value
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index 8185499bfa..cbf6bdf3b7 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -3,11 +3,6 @@ variable "cluster_name" {
   type        = string
 }
 
-variable "aws_region" {
-  description = "AWS region is cloud provider is AWS"
-  type        = string
-}
-
 variable "cloud_provider" {
   description = "Cloud provider being used in deployment"
   type        = string

From 925755574af8b9cde893fefab904a571597071a2 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <pt247@users.noreply.github.com>
Date: Mon, 8 Jan 2024 08:38:37 +0000
Subject: [PATCH 34/92] Update versions.tf provide region in AWS configuration

---
 src/_nebari/stages/kubernetes_initialize/template/versions.tf | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index 33b30e0c1f..cedca51bf6 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -10,7 +10,8 @@ terraform {
     }
     aws = {
       source  = "hashicorp/aws"
-      version = "3.73.0"
+      version = "3.73.0",
+      region = var.aws_region
     }
   }
   required_version = ">= 1.0"

From cd9b5f8704452133ad7aa56e4d55bc4e3ffed854 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <pt247@users.noreply.github.com>
Date: Mon, 8 Jan 2024 08:50:48 +0000
Subject: [PATCH 35/92] Update versions.tf removed AWS from stage level

---
 .../stages/kubernetes_initialize/template/versions.tf        | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
index cedca51bf6..341def1365 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/versions.tf
@@ -8,11 +8,6 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = "2.20.0"
     }
-    aws = {
-      source  = "hashicorp/aws"
-      version = "3.73.0",
-      region = var.aws_region
-    }
   }
   required_version = ">= 1.0"
 }

From d4c4251db3ad3bc4a8dd53439315532ee63fce49 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <pt247@users.noreply.github.com>
Date: Mon, 8 Jan 2024 08:55:26 +0000
Subject: [PATCH 36/92] Update versions.tf move AWS provider to level of
 tagging

---
 .../template/modules/tagging/versions.tf                  | 8 ++++++++
 1 file changed, 8 insertions(+)
 create mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
new file mode 100644
index 0000000000..9de6656699
--- /dev/null
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+  aws = {
+      source  = "hashicorp/aws"
+      version = "3.73.0"
+    }
+  required_version = ">= 1.0"
+}

From 8a4ec1254db885dd4a8e04319505f766639fbb9f Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <pt247@users.noreply.github.com>
Date: Mon, 8 Jan 2024 08:58:20 +0000
Subject: [PATCH 37/92] Update versions.tf

---
 .../kubernetes_initialize/template/modules/tagging/versions.tf   | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
index 9de6656699..f70bc664a5 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
@@ -4,5 +4,6 @@ terraform {
       source  = "hashicorp/aws"
       version = "3.73.0"
     }
+  }
   required_version = ">= 1.0"
 }

From 2bac3bea194a68e9ddc699a09228e0d720d3ec9f Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 10:31:51 +0000
Subject: [PATCH 38/92] Add region to input vars of kubernetes_initialize.

---
 src/_nebari/stages/kubernetes_initialize/__init__.py            | 1 +
 .../kubernetes_initialize/template/modules/tagging/versions.tf  | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index e40a69ed0f..521a5c28b3 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -78,6 +78,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             name=self.config.project_name,
             environment=self.config.namespace,
             cloud_provider=self.config.provider.value,
+            aws_region=self.config.provider.region.value,
             external_container_reg=self.config.external_container_reg.dict(),
         )
 
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
index f70bc664a5..54fc973d6a 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
@@ -1,6 +1,6 @@
 terraform {
   required_providers {
-  aws = {
+    aws = {
       source  = "hashicorp/aws"
       version = "3.73.0"
     }

From 30d0e5bc4a0625dfc783818e65040b272d5dc177 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 10:52:11 +0000
Subject: [PATCH 39/92] Revert version changes.

---
 .../stages/kubernetes_initialize/__init__.py   |  1 -
 .../template/modules/tagging/versions.tf       | 18 +++++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index 521a5c28b3..e40a69ed0f 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -78,7 +78,6 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             name=self.config.project_name,
             environment=self.config.namespace,
             cloud_provider=self.config.provider.value,
-            aws_region=self.config.provider.region.value,
             external_container_reg=self.config.external_container_reg.dict(),
         )
 
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
index 54fc973d6a..3309e058ba 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
@@ -1,9 +1,9 @@
-terraform {
-  required_providers {
-    aws = {
-      source  = "hashicorp/aws"
-      version = "3.73.0"
-    }
-  }
-  required_version = ">= 1.0"
-}
+#terraform {
+#  required_providers {
+#    aws = {
+#      source  = "hashicorp/aws"
+#      version = "3.73.0"
+#    }
+#  }
+#  required_version = ">= 1.0"
+#}

From 433e9cbe4c3c6045a339c9043a960e3559100f0c Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 11:07:18 +0000
Subject: [PATCH 40/92] Fix provider error.

---
 .../stages/kubernetes_initialize/template/main.tf      |  1 -
 .../template/modules/tagging/aws-asg-tagging.tf        | 10 ++++++++--
 2 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index abac45ec36..6de2df76ad 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,7 +25,6 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count          = var.cloud_provider == "aws" ? 1 : 0
   source         = "./modules/tagging"
   cluster_name   = local.cluster_name
   cloud_provider = var.cloud_provider
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index a37fe8056a..765919627f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,4 +1,5 @@
 data "aws_eks_node_group" "user" {
+  count = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "user"
 }
@@ -7,7 +8,9 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
       [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
-    ) : asg.name]
+    ) : asg.name
+      if var.cloud_provider == "aws" ? 1 : 0
+    ]
   )
   autoscaling_group_name = each.value
   tag {
@@ -21,6 +24,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
+  count = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }
@@ -29,7 +33,9 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
       [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
-    ) : asg.name]
+    ) : asg.name
+      if var.cloud_provider == "aws" ? 1 : 0
+    ]
   )
   autoscaling_group_name = each.value
   tag {

From c123badfb7402d01d3a54c07f1a2cee24dd69c77 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 11:26:31 +0000
Subject: [PATCH 41/92] Remove if condition from  for each.

---
 .../template/modules/tagging/aws-asg-tagging.tf                 | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 765919627f..5a598a5d5d 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -9,7 +9,6 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
     [for asg in flatten(
       [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
     ) : asg.name
-      if var.cloud_provider == "aws" ? 1 : 0
     ]
   )
   autoscaling_group_name = each.value
@@ -34,7 +33,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
     [for asg in flatten(
       [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
     ) : asg.name
-      if var.cloud_provider == "aws" ? 1 : 0
     ]
   )
   autoscaling_group_name = each.value

From 5d8c344d134245e236299f18d86de0dff88d23cf Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 11:29:37 +0000
Subject: [PATCH 42/92] Fix aws-asg-taggin.tf.

---
 .../template/modules/tagging/aws-asg-tagging.tf           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 5a598a5d5d..e8f8ed5e01 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -7,7 +7,7 @@ data "aws_eks_node_group" "user" {
 resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
-      [for resources in data.aws_eks_node_group.user.resources : resources.autoscaling_groups]
+      [for resources in data.aws_eks_node_group[0].user.resources : resources.autoscaling_groups]
     ) : asg.name
     ]
   )
@@ -18,7 +18,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.user
+    data.aws_eks_node_group[0].user
   ]
 }
 
@@ -31,7 +31,7 @@ data "aws_eks_node_group" "worker" {
 resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
-      [for resources in data.aws_eks_node_group.worker.resources : resources.autoscaling_groups]
+      [for resources in data.aws_eks_node_group[0].worker.resources : resources.autoscaling_groups]
     ) : asg.name
     ]
   )
@@ -42,6 +42,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.worker,
+    data.aws_eks_node_group[0].worker,
   ]
 }

From 342774d22908afcafb8e5f8ff92217d05a42409e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 11:31:55 +0000
Subject: [PATCH 43/92] Formatting changes.

---
 .../template/modules/tagging/aws-asg-tagging.tf           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index e8f8ed5e01..737dfab6c4 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,5 +1,5 @@
 data "aws_eks_node_group" "user" {
-  count = var.cloud_provider == "aws" ? 1 : 0
+  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "user"
 }
@@ -8,7 +8,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
       [for resources in data.aws_eks_node_group[0].user.resources : resources.autoscaling_groups]
-    ) : asg.name
+      ) : asg.name
     ]
   )
   autoscaling_group_name = each.value
@@ -23,7 +23,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
-  count = var.cloud_provider == "aws" ? 1 : 0
+  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }
@@ -32,7 +32,7 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
       [for resources in data.aws_eks_node_group[0].worker.resources : resources.autoscaling_groups]
-    ) : asg.name
+      ) : asg.name
     ]
   )
   autoscaling_group_name = each.value

From 3a12e602a36be1f11193d50d13919f3901a077c3 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 11:45:24 +0000
Subject: [PATCH 44/92] Fix aws-asg-taggin.tf.

---
 .../template/modules/tagging/aws-asg-tagging.tf  | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 737dfab6c4..6e9133c71b 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -7,7 +7,11 @@ data "aws_eks_node_group" "user" {
 resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
-      [for resources in data.aws_eks_node_group[0].user.resources : resources.autoscaling_groups]
+      [
+        for resources in data.aws_eks_node_group.user[0].resources :
+        resources.autoscaling_groups
+        if length(data.aws_eks_node_group.user) > 0
+      ]
       ) : asg.name
     ]
   )
@@ -18,7 +22,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group[0].user
+    data.aws_eks_node_group.user[0]
   ]
 }
 
@@ -31,7 +35,11 @@ data "aws_eks_node_group" "worker" {
 resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
-      [for resources in data.aws_eks_node_group[0].worker.resources : resources.autoscaling_groups]
+      [
+        for resources in data.aws_eks_node_group.worker[0].resources :
+        resources.autoscaling_groups
+        if length(data.aws_eks_node_group.worker) > 0
+      ]
       ) : asg.name
     ]
   )
@@ -42,6 +50,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group[0].worker,
+    data.aws_eks_node_group.worker[0],
   ]
 }

From 033d76b9b75e1b748ae9de567cebd0ade3655e95 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 12:08:02 +0000
Subject: [PATCH 45/92] Add aws validation for tagging.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf     | 1 +
 .../template/modules/tagging/aws-asg-tagging.tf               | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 6de2df76ad..a9f5f22f91 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,6 +25,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
+  count        = var.cloud_provider == "aws" ? 1 : 0
   source         = "./modules/tagging"
   cluster_name   = local.cluster_name
   cloud_provider = var.cloud_provider
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 6e9133c71b..5d8c048a08 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -10,7 +10,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
       [
         for resources in data.aws_eks_node_group.user[0].resources :
         resources.autoscaling_groups
-        if length(data.aws_eks_node_group.user) > 0
+        if data.aws_eks_node_group.user.count > 0
       ]
       ) : asg.name
     ]
@@ -38,7 +38,7 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
       [
         for resources in data.aws_eks_node_group.worker[0].resources :
         resources.autoscaling_groups
-        if length(data.aws_eks_node_group.worker) > 0
+        if data.aws_eks_node_group.worker.count > 0
       ]
       ) : asg.name
     ]

From 8d53afd09ce656529eeed0569f3570547ee51595 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 13:16:03 +0000
Subject: [PATCH 46/92] Count not supported using lenght.

---
 .../template/modules/tagging/aws-asg-tagging.tf               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 5d8c048a08..6e9133c71b 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -10,7 +10,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
       [
         for resources in data.aws_eks_node_group.user[0].resources :
         resources.autoscaling_groups
-        if data.aws_eks_node_group.user.count > 0
+        if length(data.aws_eks_node_group.user) > 0
       ]
       ) : asg.name
     ]
@@ -38,7 +38,7 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
       [
         for resources in data.aws_eks_node_group.worker[0].resources :
         resources.autoscaling_groups
-        if data.aws_eks_node_group.worker.count > 0
+        if length(data.aws_eks_node_group.worker) > 0
       ]
       ) : asg.name
     ]

From 0acd0714247126e7befd32c7735e4cb4bccb46be Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 15:33:23 +0000
Subject: [PATCH 47/92] Revert module level aws filtering.

---
 .../stages/kubernetes_initialize/template/main.tf        | 2 +-
 .../template/modules/tagging/aws-asg-tagging.tf          | 4 ----
 .../template/modules/tagging/versions.tf                 | 9 ---------
 3 files changed, 1 insertion(+), 14 deletions(-)
 delete mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index a9f5f22f91..abac45ec36 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,7 +25,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count        = var.cloud_provider == "aws" ? 1 : 0
+  count          = var.cloud_provider == "aws" ? 1 : 0
   source         = "./modules/tagging"
   cluster_name   = local.cluster_name
   cloud_provider = var.cloud_provider
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 6e9133c71b..8edcd57a71 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,5 +1,4 @@
 data "aws_eks_node_group" "user" {
-  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "user"
 }
@@ -10,7 +9,6 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
       [
         for resources in data.aws_eks_node_group.user[0].resources :
         resources.autoscaling_groups
-        if length(data.aws_eks_node_group.user) > 0
       ]
       ) : asg.name
     ]
@@ -27,7 +25,6 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
-  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }
@@ -38,7 +35,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
       [
         for resources in data.aws_eks_node_group.worker[0].resources :
         resources.autoscaling_groups
-        if length(data.aws_eks_node_group.worker) > 0
       ]
       ) : asg.name
     ]
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
deleted file mode 100644
index 3309e058ba..0000000000
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/versions.tf
+++ /dev/null
@@ -1,9 +0,0 @@
-#terraform {
-#  required_providers {
-#    aws = {
-#      source  = "hashicorp/aws"
-#      version = "3.73.0"
-#    }
-#  }
-#  required_version = ">= 1.0"
-#}

From 28eef1f77d1d072abf4acf961668d8c126e51f36 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 15:35:23 +0000
Subject: [PATCH 48/92] Revert module level aws filtering.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf    | 1 -
 .../template/modules/tagging/variables.tf                    | 5 -----
 2 files changed, 6 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index abac45ec36..61a2640fcb 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -28,5 +28,4 @@ module "tagging" {
   count          = var.cloud_provider == "aws" ? 1 : 0
   source         = "./modules/tagging"
   cluster_name   = local.cluster_name
-  cloud_provider = var.cloud_provider
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index cbf6bdf3b7..00b51c849f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -2,8 +2,3 @@ variable "cluster_name" {
   description = "Name of the cluster."
   type        = string
 }
-
-variable "cloud_provider" {
-  description = "Cloud provider being used in deployment"
-  type        = string
-}

From f25decc6ca3b96968e309ea9e422033e90dada5e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 15:37:47 +0000
Subject: [PATCH 49/92] Terraform format changes.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 61a2640fcb..de9d36c08b 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,7 +25,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count          = var.cloud_provider == "aws" ? 1 : 0
-  source         = "./modules/tagging"
-  cluster_name   = local.cluster_name
+  count        = var.cloud_provider == "aws" ? 1 : 0
+  source       = "./modules/tagging"
+  cluster_name = local.cluster_name
 }

From 9157bdd0fa91adaaf86d2724732031210fe77622 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 16:12:40 +0000
Subject: [PATCH 50/92] Fix aws-asg-tagging.tf.

---
 .../template/modules/tagging/aws-asg-tagging.tf           | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 8edcd57a71..0d1ddff61f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -7,7 +7,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
       [
-        for resources in data.aws_eks_node_group.user[0].resources :
+        for resources in data.aws_eks_node_group.user.resources :
         resources.autoscaling_groups
       ]
       ) : asg.name
@@ -20,7 +20,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.user[0]
+    data.aws_eks_node_group.user
   ]
 }
 
@@ -33,7 +33,7 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
       [
-        for resources in data.aws_eks_node_group.worker[0].resources :
+        for resources in data.aws_eks_node_group.worker.resources :
         resources.autoscaling_groups
       ]
       ) : asg.name
@@ -46,6 +46,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.worker[0],
+    data.aws_eks_node_group.worker,
   ]
 }

From cff6ca3d25eda25b72b5a71b08dab7a30e1c3d5c Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 18:06:12 +0000
Subject: [PATCH 51/92] Add aws provider region.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf    | 1 +
 .../template/modules/tagging/aws-asg-tagging.tf              | 5 +++++
 .../template/modules/tagging/variables.tf                    | 4 ++++
 3 files changed, 10 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index de9d36c08b..6025e400d4 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -28,4 +28,5 @@ module "tagging" {
   count        = var.cloud_provider == "aws" ? 1 : 0
   source       = "./modules/tagging"
   cluster_name = local.cluster_name
+  aws_region   = var.aws_region
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 0d1ddff61f..a912a73337 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -3,6 +3,11 @@ data "aws_eks_node_group" "user" {
   node_group_name = "user"
 }
 
+provider "aws" {
+  region = var.aws_region
+
+}
+
 resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index 00b51c849f..a02b1baaf5 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -2,3 +2,7 @@ variable "cluster_name" {
   description = "Name of the cluster."
   type        = string
 }
+variable "aws_region" {
+  description = "AWS Region that cluster autoscaler is running"
+  type        = string
+}

From 9307175c1a2af0cf313e2e6afddf0f00dfd58b11 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 18:38:23 +0000
Subject: [PATCH 52/92] Remove aws filter from stage level and add it to module
 level.

---
 .../stages/kubernetes_initialize/template/main.tf |  8 ++++----
 .../template/modules/tagging/aws-asg-tagging.tf   | 15 ++++++---------
 2 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 6025e400d4..5356959d1c 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,8 +25,8 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count        = var.cloud_provider == "aws" ? 1 : 0
-  source       = "./modules/tagging"
-  cluster_name = local.cluster_name
-  aws_region   = var.aws_region
+  source         = "./modules/tagging"
+  cluster_name   = local.cluster_name
+  aws_region     = var.aws_region
+  cloud_provider = var.cloud_provider
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index a912a73337..8e0d3d76f3 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,18 +1,14 @@
 data "aws_eks_node_group" "user" {
+  count = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "user"
 }
 
-provider "aws" {
-  region = var.aws_region
-
-}
-
 resource "aws_autoscaling_group_tag" "dedicated_user" {
   for_each = toset(
     [for asg in flatten(
       [
-        for resources in data.aws_eks_node_group.user.resources :
+        for resources in data.aws_eks_node_group.user[0].resources :
         resources.autoscaling_groups
       ]
       ) : asg.name
@@ -25,11 +21,12 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.user
+    data.aws_eks_node_group.user[0]
   ]
 }
 
 data "aws_eks_node_group" "worker" {
+  count = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }
@@ -38,7 +35,7 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
   for_each = toset(
     [for asg in flatten(
       [
-        for resources in data.aws_eks_node_group.worker.resources :
+        for resources in data.aws_eks_node_group.worker[0].resources :
         resources.autoscaling_groups
       ]
       ) : asg.name
@@ -51,6 +48,6 @@ resource "aws_autoscaling_group_tag" "dedicated_worker" {
     propagate_at_launch = true
   }
   depends_on = [
-    data.aws_eks_node_group.worker,
+    data.aws_eks_node_group.worker[0],
   ]
 }

From ef3b2f75f3f48b1ab3e753da5ce25fdd8764d52d Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 18:41:18 +0000
Subject: [PATCH 53/92] Terraform format changes.

---
 .../template/modules/tagging/aws-asg-tagging.tf               | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 8e0d3d76f3..c315d850b0 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,5 +1,5 @@
 data "aws_eks_node_group" "user" {
-  count = var.cloud_provider == "aws" ? 1 : 0
+  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "user"
 }
@@ -26,7 +26,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
-  count = var.cloud_provider == "aws" ? 1 : 0
+  count =         var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }

From 1a00157d93ba423ae2a6753722fb4ced6af27666 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 19:02:00 +0000
Subject: [PATCH 54/92] Remove aws filter from stage level and add it to module
 level.

---
 .../template/modules/tagging/variables.tf                    | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index a02b1baaf5..7d1b08c90b 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -6,3 +6,8 @@ variable "aws_region" {
   description = "AWS Region that cluster autoscaler is running"
   type        = string
 }
+
+variable "cloud_provider" {
+  description = "Cloud provider being used in deployment"
+  type        = string
+}

From 2db4db9b6faf8b344da063bd10f9e017ae45e5b9 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 8 Jan 2024 19:03:27 +0000
Subject: [PATCH 55/92] Terraform format changes.

---
 .../template/modules/tagging/aws-asg-tagging.tf                 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index c315d850b0..a67c161706 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -26,7 +26,7 @@ resource "aws_autoscaling_group_tag" "dedicated_user" {
 }
 
 data "aws_eks_node_group" "worker" {
-  count =         var.cloud_provider == "aws" ? 1 : 0
+  count           = var.cloud_provider == "aws" ? 1 : 0
   cluster_name    = var.cluster_name
   node_group_name = "worker"
 }

From e3a3ea397096b5361362f4d7b67014c8fccef958 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 10:16:01 +0000
Subject: [PATCH 56/92] Get aws_get_asg_node_group_mapping from stage 2 output.

---
 .../provider/cloud/amazon_web_services.py     | 21 ++++++++
 src/_nebari/stages/infrastructure/__init__.py | 16 ++++++
 .../stages/kubernetes_initialize/__init__.py  |  1 +
 .../kubernetes_initialize/template/main.tf    |  6 +--
 .../modules/tagging/aws-asg-tagging.tf        | 52 ++-----------------
 .../template/modules/tagging/variables.tf     | 17 ++----
 .../template/variables.tf                     |  5 ++
 7 files changed, 53 insertions(+), 65 deletions(-)

diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py
index 576f72c1c6..5194b162ea 100644
--- a/src/_nebari/provider/cloud/amazon_web_services.py
+++ b/src/_nebari/provider/cloud/amazon_web_services.py
@@ -143,6 +143,27 @@ def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]:
     return None
 
 
+def aws_get_asg_node_group_mapping(name: str, namespace: str, region: str) -> Dict[str, str]:
+    """Return a dictionary of autoscaling groups and their associated node groups."""
+    asg_node_group_mapping = {}
+    session = aws_session(region=region)
+    eks = session.client("eks")
+    node_groups_response = eks.list_nodegroups(
+        clusterName=f"{name}-{namespace}",
+    )
+    node_groups = node_groups_response.get("nodegroups", [])
+    for nodegroup in node_groups:
+        response = eks.describe_nodegroup(
+            clusterName=f"{name}-{namespace}",
+            nodegroupName=nodegroup
+        )
+        node_group_name = response["nodegroup"]["nodegroupName"]
+        auto_scaling_groups = response["nodegroup"]["resources"]["autoScalingGroups"]
+        for auto_scaling_group in auto_scaling_groups:
+            asg_node_group_mapping[auto_scaling_group["name"]] = node_group_name
+    return asg_node_group_mapping
+
+
 def aws_get_subnet_ids(name: str, namespace: str, region: str) -> List[str]:
     """Return list of subnet IDs for the EKS cluster named `{name}-{namespace}`."""
     session = aws_session(region=region)
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 82ce8fc3bd..58b27d36b9 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -145,6 +145,16 @@ class AWSInputVars(schema.Base):
     tags: Dict[str, str] = {}
 
 
+def _calculate_asg_node_group_map(config: schema.Main):
+    if config.provider == schema.ProviderEnum.aws:
+        return amazon_web_services.aws_get_asg_node_group_mapping(
+            config.project_name,
+            config.namespace,
+            config.amazon_web_services.region,
+        )
+    else:
+        return {}
+
 def _calculate_node_groups(config: schema.Main):
     if config.provider == schema.ProviderEnum.aws:
         return {
@@ -571,6 +581,10 @@ class NodeSelectorKeyValue(schema.Base):
     key: str
     value: str
 
+class AsgNodeGroupMap(schema.Base):
+    key: str
+    value: str
+
 
 class KubernetesCredentials(schema.Base):
     host: str
@@ -589,6 +603,7 @@ class OutputSchema(schema.Base):
     kubernetes_credentials: KubernetesCredentials
     kubeconfig_filename: str
     nfs_endpoint: Optional[str]
+    asg_node_group_map: Optional[AsgNodeGroupMap]
 
 
 class KubernetesInfrastructureStage(NebariTerraformStage):
@@ -812,6 +827,7 @@ def set_outputs(
         self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]
     ):
         outputs["node_selectors"] = _calculate_node_groups(self.config)
+        outputs["asg_node_group_map"] = _calculate_asg_node_group_map(self.config)
         super().set_outputs(stage_outputs, outputs)
 
     @contextlib.contextmanager
diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index e40a69ed0f..883afb92ea 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -49,6 +49,7 @@ class InputVars(schema.Base):
     external_container_reg: Union[ExtContainerReg, None] = None
     gpu_enabled: bool = False
     gpu_node_group_names: List[str] = []
+    asg_node_group_map: Dict[str, str] = {}
 
 
 class InputSchema(schema.Base):
diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 5356959d1c..dd1195affc 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,8 +25,6 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  source         = "./modules/tagging"
-  cluster_name   = local.cluster_name
-  aws_region     = var.aws_region
-  cloud_provider = var.cloud_provider
+  source             = "./modules/tagging"
+  asg_node_group_map = var.asg_node_group_map
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index a67c161706..ca205fc44a 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,53 +1,9 @@
-data "aws_eks_node_group" "user" {
-  count           = var.cloud_provider == "aws" ? 1 : 0
-  cluster_name    = var.cluster_name
-  node_group_name = "user"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_user" {
-  for_each = toset(
-    [for asg in flatten(
-      [
-        for resources in data.aws_eks_node_group.user[0].resources :
-        resources.autoscaling_groups
-      ]
-      ) : asg.name
-    ]
-  )
-  autoscaling_group_name = each.value
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "user"
-    propagate_at_launch = true
-  }
-  depends_on = [
-    data.aws_eks_node_group.user[0]
-  ]
-}
-
-data "aws_eks_node_group" "worker" {
-  count           = var.cloud_provider == "aws" ? 1 : 0
-  cluster_name    = var.cluster_name
-  node_group_name = "worker"
-}
-
-resource "aws_autoscaling_group_tag" "dedicated_worker" {
-  for_each = toset(
-    [for asg in flatten(
-      [
-        for resources in data.aws_eks_node_group.worker[0].resources :
-        resources.autoscaling_groups
-      ]
-      ) : asg.name
-    ]
-  )
-  autoscaling_group_name = each.value
+resource "aws_autoscaling_group_tag" "aws_eks_scaling_from_zero_labeling" {
+  count = length(var.asg_node_group_map)
+  autoscaling_group_name = var.asg_node_group_map[element(keys(var.asg_node_group_map), count.index)]
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = "worker"
+    value               = element(keys(var.asg_node_group_map), count.index)
     propagate_at_launch = true
   }
-  depends_on = [
-    data.aws_eks_node_group.worker[0],
-  ]
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index 7d1b08c90b..679117102f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -1,13 +1,4 @@
-variable "cluster_name" {
-  description = "Name of the cluster."
-  type        = string
-}
-variable "aws_region" {
-  description = "AWS Region that cluster autoscaler is running"
-  type        = string
-}
-
-variable "cloud_provider" {
-  description = "Cloud provider being used in deployment"
-  type        = string
-}
+variable "asg_node_group_map" {
+  description = "Mapping between asg name and node group name."
+  type = map(string)
+}
\ No newline at end of file
diff --git a/src/_nebari/stages/kubernetes_initialize/template/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
index f169f5bcf2..d80a989855 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
@@ -30,3 +30,8 @@ variable "gpu_enabled" {
 variable "gpu_node_group_names" {
   description = "Names of node groups with GPU"
 }
+
+variable "asg_node_group_map" {
+  description = "Mapping between asg name and node group name."
+  type = map(string)
+}

From 192020b9fafb0ed691683a15875fb670bbaf937d Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 9 Jan 2024 10:17:46 +0000
Subject: [PATCH 57/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/provider/cloud/amazon_web_services.py          | 7 ++++---
 src/_nebari/stages/infrastructure/__init__.py              | 2 ++
 .../template/modules/tagging/variables.tf                  | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py
index 5194b162ea..24624fabe7 100644
--- a/src/_nebari/provider/cloud/amazon_web_services.py
+++ b/src/_nebari/provider/cloud/amazon_web_services.py
@@ -143,7 +143,9 @@ def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]:
     return None
 
 
-def aws_get_asg_node_group_mapping(name: str, namespace: str, region: str) -> Dict[str, str]:
+def aws_get_asg_node_group_mapping(
+    name: str, namespace: str, region: str
+) -> Dict[str, str]:
     """Return a dictionary of autoscaling groups and their associated node groups."""
     asg_node_group_mapping = {}
     session = aws_session(region=region)
@@ -154,8 +156,7 @@ def aws_get_asg_node_group_mapping(name: str, namespace: str, region: str) -> Di
     node_groups = node_groups_response.get("nodegroups", [])
     for nodegroup in node_groups:
         response = eks.describe_nodegroup(
-            clusterName=f"{name}-{namespace}",
-            nodegroupName=nodegroup
+            clusterName=f"{name}-{namespace}", nodegroupName=nodegroup
         )
         node_group_name = response["nodegroup"]["nodegroupName"]
         auto_scaling_groups = response["nodegroup"]["resources"]["autoScalingGroups"]
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 58b27d36b9..b4a6ea06a5 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -155,6 +155,7 @@ def _calculate_asg_node_group_map(config: schema.Main):
     else:
         return {}
 
+
 def _calculate_node_groups(config: schema.Main):
     if config.provider == schema.ProviderEnum.aws:
         return {
@@ -581,6 +582,7 @@ class NodeSelectorKeyValue(schema.Base):
     key: str
     value: str
 
+
 class AsgNodeGroupMap(schema.Base):
     key: str
     value: str
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index 679117102f..e6d478b1be 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -1,4 +1,4 @@
 variable "asg_node_group_map" {
   description = "Mapping between asg name and node group name."
   type = map(string)
-}
\ No newline at end of file
+}

From 560fc1a0671c0741b5499d8fd717dbd8fef521ba Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 10:27:34 +0000
Subject: [PATCH 58/92] Get aws_get_asg_node_group_mapping from stage 2 output.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index dd1195affc..16a75a5108 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,6 +25,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
+  count              = length(var.asg_node_group_map) > 0 ? 1 : 0
   source             = "./modules/tagging"
   asg_node_group_map = var.asg_node_group_map
 }

From 970b2a5636a9e37bd6f03e27d3e34f1be2b997a8 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 15:23:48 +0000
Subject: [PATCH 59/92] Get aws_get_asg_node_group_mapping from stage 2 output.

---
 src/_nebari/stages/kubernetes_initialize/__init__.py          | 2 +-
 .../template/modules/tagging/aws-asg-tagging.tf               | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index 883afb92ea..a8bfefbf02 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -97,7 +97,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 group for group in self.config.amazon_web_services.node_groups.keys()
             ]
             input_vars.aws_region = self.config.amazon_web_services.region
-
+            input_vars.asg_node_group_map = stage_outputs['stages/02-infrastructure']['asg_node_group_map']
         return input_vars.dict()
 
     def check(
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index ca205fc44a..40de1aece1 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,9 +1,9 @@
 resource "aws_autoscaling_group_tag" "aws_eks_scaling_from_zero_labeling" {
   count = length(var.asg_node_group_map)
-  autoscaling_group_name = var.asg_node_group_map[element(keys(var.asg_node_group_map), count.index)]
+  autoscaling_group_name = element(keys(var.asg_node_group_map), count.index)
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = element(keys(var.asg_node_group_map), count.index)
+    value               = var.asg_node_group_map[element(keys(var.asg_node_group_map), count.index)]
     propagate_at_launch = true
   }
 }

From a0315c31886fc5dd8c4e94e0c3cfde28dee01bcc Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 9 Jan 2024 15:24:29 +0000
Subject: [PATCH 60/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/kubernetes_initialize/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index a8bfefbf02..04f2b60415 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -97,7 +97,9 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 group for group in self.config.amazon_web_services.node_groups.keys()
             ]
             input_vars.aws_region = self.config.amazon_web_services.region
-            input_vars.asg_node_group_map = stage_outputs['stages/02-infrastructure']['asg_node_group_map']
+            input_vars.asg_node_group_map = stage_outputs["stages/02-infrastructure"][
+                "asg_node_group_map"
+            ]
         return input_vars.dict()
 
     def check(

From 29e7f65f3e8b449accf515a4ea62be8af4784828 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 15:29:31 +0000
Subject: [PATCH 61/92] Terraform format changes.

---
 .../template/modules/tagging/aws-asg-tagging.tf                 | 2 +-
 .../kubernetes_initialize/template/modules/tagging/variables.tf | 2 +-
 src/_nebari/stages/kubernetes_initialize/template/variables.tf  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 40de1aece1..4b28708981 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,5 +1,5 @@
 resource "aws_autoscaling_group_tag" "aws_eks_scaling_from_zero_labeling" {
-  count = length(var.asg_node_group_map)
+  count                  = length(var.asg_node_group_map)
   autoscaling_group_name = element(keys(var.asg_node_group_map), count.index)
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
index e6d478b1be..e5c82f3dc5 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
@@ -1,4 +1,4 @@
 variable "asg_node_group_map" {
   description = "Mapping between asg name and node group name."
-  type = map(string)
+  type        = map(string)
 }
diff --git a/src/_nebari/stages/kubernetes_initialize/template/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
index d80a989855..987e47b729 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
@@ -33,5 +33,5 @@ variable "gpu_node_group_names" {
 
 variable "asg_node_group_map" {
   description = "Mapping between asg name and node group name."
-  type = map(string)
+  type        = map(string)
 }

From 852a92915d6c2e146b332ee0f05f64b8f6714ef3 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 15:30:45 +0000
Subject: [PATCH 62/92] Fix stage level filter for tagging.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 16a75a5108..abddb61439 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -25,7 +25,7 @@ module "nvidia-driver-installer" {
 }
 
 module "tagging" {
-  count              = length(var.asg_node_group_map) > 0 ? 1 : 0
+  count              = var.cloud_provider == "aws" ? 1 : 0
   source             = "./modules/tagging"
   asg_node_group_map = var.asg_node_group_map
 }

From 301a2db94557e66bedb7e63bd35700cb2f504b5d Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 16:13:20 +0000
Subject: [PATCH 63/92] Try and remove tagging from pahse-03

---
 .../stages/kubernetes_initialize/template/main.tf      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index abddb61439..ee1cdcce31 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -24,8 +24,8 @@ module "nvidia-driver-installer" {
   gpu_node_group_names = var.gpu_node_group_names
 }
 
-module "tagging" {
-  count              = var.cloud_provider == "aws" ? 1 : 0
-  source             = "./modules/tagging"
-  asg_node_group_map = var.asg_node_group_map
-}
+#module "tagging" {
+#  count              = var.cloud_provider == "aws" ? 1 : 0
+#  source             = "./modules/tagging"
+#  asg_node_group_map = var.asg_node_group_map
+#}

From 9b060913bb30e219d0f81eafcce70e95691d38d0 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Tue, 9 Jan 2024 16:40:10 +0000
Subject: [PATCH 64/92] Add back tagging config to  phase-03.

---
 .../stages/kubernetes_initialize/template/main.tf      | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index ee1cdcce31..abddb61439 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -24,8 +24,8 @@ module "nvidia-driver-installer" {
   gpu_node_group_names = var.gpu_node_group_names
 }
 
-#module "tagging" {
-#  count              = var.cloud_provider == "aws" ? 1 : 0
-#  source             = "./modules/tagging"
-#  asg_node_group_map = var.asg_node_group_map
-#}
+module "tagging" {
+  count              = var.cloud_provider == "aws" ? 1 : 0
+  source             = "./modules/tagging"
+  asg_node_group_map = var.asg_node_group_map
+}

From a2206cd558b223a8590feff9733a728c25ac0241 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 08:45:43 +0000
Subject: [PATCH 65/92] Add precondition check if provider is aws for testing.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index abddb61439..9d21e70c3f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -28,4 +28,10 @@ module "tagging" {
   count              = var.cloud_provider == "aws" ? 1 : 0
   source             = "./modules/tagging"
   asg_node_group_map = var.asg_node_group_map
+  lifecycle {
+    precondition {
+      condition = var.cloud_provider == "aws"
+      error_message = "Cloud_provider ${var.cloud_provider} shoule not be aws."
+    }
+  }
 }

From 5970eefcfdccd0a5f63c67ee201e36ee7f8a60bd Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 11 Jan 2024 08:45:57 +0000
Subject: [PATCH 66/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 9d21e70c3f..3fbffb1be0 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -31,7 +31,7 @@ module "tagging" {
   lifecycle {
     precondition {
       condition = var.cloud_provider == "aws"
-      error_message = "Cloud_provider ${var.cloud_provider} shoule not be aws."
+      error_message = "Cloud_provider ${var.cloud_provider} should not be aws."
     }
   }
 }

From 4a253d3e2cd8156c8ce1e4c9417ddbfaf927490d Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 09:19:03 +0000
Subject: [PATCH 67/92] Terrafrom fmt changes.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 3fbffb1be0..618ce3381c 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -30,7 +30,7 @@ module "tagging" {
   asg_node_group_map = var.asg_node_group_map
   lifecycle {
     precondition {
-      condition = var.cloud_provider == "aws"
+      condition     = var.cloud_provider == "aws"
       error_message = "Cloud_provider ${var.cloud_provider} should not be aws."
     }
   }

From 5764dc940f6522f58deea80b3ed1554ca17608ed Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 09:26:09 +0000
Subject: [PATCH 68/92] Add debugging.

---
 .github/workflows/test_local_integration.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 8ddc1f9690..a82e77f557 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -98,7 +98,13 @@ jobs:
 
           cat nebari-config.yaml
 
+      - name Terrafrom version check
+        run: |
+          terraform --version
+
       - name: Deploy Nebari
+        env:
+          TF_LOG: TRACE
         working-directory: local-deployment
         run: |
           nebari deploy --config nebari-config.yaml --disable-prompt

From b5e1c9fbab0cb0d7a4a14b93f1c46a26bc530c62 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 09:31:07 +0000
Subject: [PATCH 69/92] Add debugging.

---
 .github/workflows/test_local_integration.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index a82e77f557..2c5f508b73 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -98,7 +98,7 @@ jobs:
 
           cat nebari-config.yaml
 
-      - name Terrafrom version check
+      - name: Terrafrom version check
         run: |
           terraform --version
 

From 9fd765b6535b555bcb18c64a4fc56fcc59feb1ca Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 09:36:02 +0000
Subject: [PATCH 70/92] Add debugging.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index 618ce3381c..abddb61439 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -28,10 +28,4 @@ module "tagging" {
   count              = var.cloud_provider == "aws" ? 1 : 0
   source             = "./modules/tagging"
   asg_node_group_map = var.asg_node_group_map
-  lifecycle {
-    precondition {
-      condition     = var.cloud_provider == "aws"
-      error_message = "Cloud_provider ${var.cloud_provider} should not be aws."
-    }
-  }
 }

From 3faba0d90c2730867d6a201fc05f45563a635a87 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 09:40:13 +0000
Subject: [PATCH 71/92] Add debugging.

---
 .github/workflows/test_local_integration.yaml | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 2c5f508b73..fd3be3faa8 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -98,10 +98,6 @@ jobs:
 
           cat nebari-config.yaml
 
-      - name: Terrafrom version check
-        run: |
-          terraform --version
-
       - name: Deploy Nebari
         env:
           TF_LOG: TRACE

From a47124503975be4c399ff7b7e5851b514046f458 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 11:57:40 +0000
Subject: [PATCH 72/92] Send empty asg_node_group_map to tagging module.

---
 src/_nebari/stages/kubernetes_initialize/template/main.tf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index abddb61439..f091a593cd 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -27,5 +27,5 @@ module "nvidia-driver-installer" {
 module "tagging" {
   count              = var.cloud_provider == "aws" ? 1 : 0
   source             = "./modules/tagging"
-  asg_node_group_map = var.asg_node_group_map
+  asg_node_group_map = var.cloud_provider == "aws" ? var.asg_node_group_map : {}
 }

From ed1b591ac0b659909ac6a5653fa49b41f96397a9 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 11 Jan 2024 11:58:35 +0000
Subject: [PATCH 73/92] Remove trace logs.

---
 .github/workflows/test_local_integration.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index fd3be3faa8..9d75c05e02 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -99,8 +99,8 @@ jobs:
           cat nebari-config.yaml
 
       - name: Deploy Nebari
-        env:
-          TF_LOG: TRACE
+        # env:
+        #   TF_LOG: TRACE
         working-directory: local-deployment
         run: |
           nebari deploy --config nebari-config.yaml --disable-prompt

From 13b69e97ec94a404002a7be94631319352a2a867 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Mon, 15 Jan 2024 10:56:34 +0000
Subject: [PATCH 74/92] Minor change to trigger CI.

---
 .../template/modules/tagging/aws-asg-tagging.tf                  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
index 4b28708981..c1abf21250 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
@@ -1,6 +1,7 @@
 resource "aws_autoscaling_group_tag" "aws_eks_scaling_from_zero_labeling" {
   count                  = length(var.asg_node_group_map)
   autoscaling_group_name = element(keys(var.asg_node_group_map), count.index)
+
   tag {
     key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
     value               = var.asg_node_group_map[element(keys(var.asg_node_group_map), count.index)]

From a6b2d98419813831ab774a010ec1aa1ea902f5e0 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 11:06:47 +0000
Subject: [PATCH 75/92] Add tagging in post deploy of phase 3, and remove
 tagging in phase 3.

---
 .../provider/cloud/amazon_web_services.py     | 22 ++++++++++++++++---
 src/_nebari/stages/infrastructure/__init__.py | 11 ++++++++++
 .../kubernetes_initialize/template/main.tf    | 10 ++++-----
 3 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py
index 24624fabe7..89bb48a158 100644
--- a/src/_nebari/provider/cloud/amazon_web_services.py
+++ b/src/_nebari/provider/cloud/amazon_web_services.py
@@ -143,9 +143,25 @@ def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]:
     return None
 
 
-def aws_get_asg_node_group_mapping(
-    name: str, namespace: str, region: str
-) -> Dict[str, str]:
+def set_asg_tags(asg_node_group_map: Dict[str, str], region: str) -> None:
+    """Set tags for AWS node scaling from zero to work."""
+    session = aws_session(region=region)
+    autoscaling_client = session.client("autoscaling")
+    tags = []
+    for asg_name, node_group in asg_node_group_map.items():
+        tags.append(
+            {
+                "Key": "k8s.io/cluster-autoscaler/node-template/label/dedicated",
+                "Value": node_group,
+                "ResourceId": asg_name,
+                "ResourceType": "auto-scaling-group",
+                "PropagateAtLaunch": True,
+            }
+        )
+    autoscaling_client.create_or_update_tags(Tags=tags)
+
+
+def aws_get_asg_node_group_mapping(name: str, namespace: str, region: str) -> Dict[str, str]:
     """Return a dictionary of autoscaling groups and their associated node groups."""
     asg_node_group_mapping = {}
     session = aws_session(region=region)
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index b4a6ea06a5..9328f7cff4 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -832,6 +832,17 @@ def set_outputs(
         outputs["asg_node_group_map"] = _calculate_asg_node_group_map(self.config)
         super().set_outputs(stage_outputs, outputs)
 
+
+    @contextlib.contextmanager
+    def post_deploy(
+            self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+    ):
+        if stage_outputs["asg_node_group_map"]:
+            amazon_web_services.set_asg_tags(
+                stage_outputs["asg_node_group_map"],
+                self.config.amazon_web_services.region)
+
+
     @contextlib.contextmanager
     def deploy(
         self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index f091a593cd..fe1b27a48e 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -24,8 +24,8 @@ module "nvidia-driver-installer" {
   gpu_node_group_names = var.gpu_node_group_names
 }
 
-module "tagging" {
-  count              = var.cloud_provider == "aws" ? 1 : 0
-  source             = "./modules/tagging"
-  asg_node_group_map = var.cloud_provider == "aws" ? var.asg_node_group_map : {}
-}
+#module "tagging" {
+#  count              = var.cloud_provider == "aws" ? 1 : 0
+#  source             = "./modules/tagging"
+#  asg_node_group_map = var.cloud_provider == "aws" ? var.asg_node_group_map : {}
+#}

From 4ea6b454f85b86bd43a32ffc6798014d41663623 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:07:02 +0000
Subject: [PATCH 76/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/provider/cloud/amazon_web_services.py | 4 +++-
 src/_nebari/stages/infrastructure/__init__.py     | 7 +++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/provider/cloud/amazon_web_services.py b/src/_nebari/provider/cloud/amazon_web_services.py
index 89bb48a158..2bf905bfcb 100644
--- a/src/_nebari/provider/cloud/amazon_web_services.py
+++ b/src/_nebari/provider/cloud/amazon_web_services.py
@@ -161,7 +161,9 @@ def set_asg_tags(asg_node_group_map: Dict[str, str], region: str) -> None:
     autoscaling_client.create_or_update_tags(Tags=tags)
 
 
-def aws_get_asg_node_group_mapping(name: str, namespace: str, region: str) -> Dict[str, str]:
+def aws_get_asg_node_group_mapping(
+    name: str, namespace: str, region: str
+) -> Dict[str, str]:
     """Return a dictionary of autoscaling groups and their associated node groups."""
     asg_node_group_mapping = {}
     session = aws_session(region=region)
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 9328f7cff4..d567d24684 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -832,16 +832,15 @@ def set_outputs(
         outputs["asg_node_group_map"] = _calculate_asg_node_group_map(self.config)
         super().set_outputs(stage_outputs, outputs)
 
-
     @contextlib.contextmanager
     def post_deploy(
-            self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
     ):
         if stage_outputs["asg_node_group_map"]:
             amazon_web_services.set_asg_tags(
                 stage_outputs["asg_node_group_map"],
-                self.config.amazon_web_services.region)
-
+                self.config.amazon_web_services.region,
+            )
 
     @contextlib.contextmanager
     def deploy(

From 703bce558383dcd6e2a1057e28d2304dd84655d7 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 11:56:57 +0000
Subject: [PATCH 77/92] Fix post deploy for stage 2.

---
 src/_nebari/stages/infrastructure/__init__.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index d567d24684..caf916031e 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -833,13 +833,11 @@ def set_outputs(
         super().set_outputs(stage_outputs, outputs)
 
     @contextlib.contextmanager
-    def post_deploy(
-        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
-    ):
-        if stage_outputs["asg_node_group_map"]:
+    def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False):
+        asg_node_group_map = _calculate_asg_node_group_map(self.config)
+        if asg_node_group_map:
             amazon_web_services.set_asg_tags(
-                stage_outputs["asg_node_group_map"],
-                self.config.amazon_web_services.region,
+                asg_node_group_map, self.config.amazon_web_services.region
             )
 
     @contextlib.contextmanager

From 023513ec637bec88381725582d118b303b436fb4 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 17 Jan 2024 11:58:23 +0000
Subject: [PATCH 78/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/infrastructure/__init__.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index caf916031e..53c4731607 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -833,7 +833,9 @@ def set_outputs(
         super().set_outputs(stage_outputs, outputs)
 
     @contextlib.contextmanager
-    def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False):
+    def post_deploy(
+        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+    ):
         asg_node_group_map = _calculate_asg_node_group_map(self.config)
         if asg_node_group_map:
             amazon_web_services.set_asg_tags(

From 5b41feb5fe1ff02293f61dcd7ee8afe8190eb74a Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 11:55:03 +0000
Subject: [PATCH 79/92] Fix post deploy for stage 2.

---
 src/_nebari/stages/infrastructure/__init__.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 9328f7cff4..a774058261 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -834,14 +834,12 @@ def set_outputs(
 
 
     @contextlib.contextmanager
-    def post_deploy(
-            self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
-    ):
-        if stage_outputs["asg_node_group_map"]:
+    def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False):
+        asg_node_group_map = _calculate_asg_node_group_map(self.config)
+        if asg_node_group_map:
             amazon_web_services.set_asg_tags(
-                stage_outputs["asg_node_group_map"],
-                self.config.amazon_web_services.region)
-
+                asg_node_group_map, self.config.amazon_web_services.region
+            )
 
     @contextlib.contextmanager
     def deploy(

From 019704fd8cb13b01460047a27b4b0c106c7ebf9f Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Wed, 17 Jan 2024 12:03:37 +0000
Subject: [PATCH 80/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/infrastructure/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index a774058261..53c4731607 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -832,9 +832,10 @@ def set_outputs(
         outputs["asg_node_group_map"] = _calculate_asg_node_group_map(self.config)
         super().set_outputs(stage_outputs, outputs)
 
-
     @contextlib.contextmanager
-    def post_deploy(self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False):
+    def post_deploy(
+        self, stage_outputs: Dict[str, Dict[str, Any]], disable_prompt: bool = False
+    ):
         asg_node_group_map = _calculate_asg_node_group_map(self.config)
         if asg_node_group_map:
             amazon_web_services.set_asg_tags(

From 32465f059673c2e195bcd4485ecb85126d842bee Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 13:22:05 +0000
Subject: [PATCH 81/92] Remove changes that are no longer needed.

---
 .github/workflows/test_local_integration.yaml |  2 --
 src/_nebari/stages/infrastructure/__init__.py |  7 ------
 .../template/aws/modules/kubernetes/main.tf   |  2 --
 .../aws/modules/kubernetes/outputs.tf         |  5 ----
 .../stages/kubernetes_initialize/__init__.py  |  5 +---
 .../kubernetes_initialize/template/main.tf    | 24 +++++++++----------
 .../template/variables.tf                     |  5 ----
 7 files changed, 13 insertions(+), 37 deletions(-)

diff --git a/.github/workflows/test_local_integration.yaml b/.github/workflows/test_local_integration.yaml
index 9d75c05e02..8ddc1f9690 100644
--- a/.github/workflows/test_local_integration.yaml
+++ b/.github/workflows/test_local_integration.yaml
@@ -99,8 +99,6 @@ jobs:
           cat nebari-config.yaml
 
       - name: Deploy Nebari
-        # env:
-        #   TF_LOG: TRACE
         working-directory: local-deployment
         run: |
           nebari deploy --config nebari-config.yaml --disable-prompt
diff --git a/src/_nebari/stages/infrastructure/__init__.py b/src/_nebari/stages/infrastructure/__init__.py
index 53c4731607..5c1aa77f77 100644
--- a/src/_nebari/stages/infrastructure/__init__.py
+++ b/src/_nebari/stages/infrastructure/__init__.py
@@ -583,11 +583,6 @@ class NodeSelectorKeyValue(schema.Base):
     value: str
 
 
-class AsgNodeGroupMap(schema.Base):
-    key: str
-    value: str
-
-
 class KubernetesCredentials(schema.Base):
     host: str
     cluster_ca_certifiate: str
@@ -605,7 +600,6 @@ class OutputSchema(schema.Base):
     kubernetes_credentials: KubernetesCredentials
     kubeconfig_filename: str
     nfs_endpoint: Optional[str]
-    asg_node_group_map: Optional[AsgNodeGroupMap]
 
 
 class KubernetesInfrastructureStage(NebariTerraformStage):
@@ -829,7 +823,6 @@ def set_outputs(
         self, stage_outputs: Dict[str, Dict[str, Any]], outputs: Dict[str, Any]
     ):
         outputs["node_selectors"] = _calculate_node_groups(self.config)
-        outputs["asg_node_group_map"] = _calculate_asg_node_group_map(self.config)
         super().set_outputs(stage_outputs, outputs)
 
     @contextlib.contextmanager
diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index a2238a41f8..43b1c0f3c9 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -63,8 +63,6 @@ resource "aws_eks_node_group" "main" {
   }, var.tags)
 }
 
-
-
 data "aws_eks_cluster_auth" "main" {
   name = aws_eks_cluster.main.name
 }
diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
index ea41c8acd0..48994bc7a2 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/outputs.tf
@@ -23,11 +23,6 @@ output "oidc_provider_arn" {
   value       = aws_iam_openid_connect_provider.oidc_provider.arn
 }
 
-output "cluster_name" {
-  description = "Name cluster"
-  value       = aws_eks_cluster.main.name
-}
-
 # /~https://github.com/terraform-aws-modules/terraform-aws-eks/blob/16f46db94b7158fd762d9133119206aaa7cf6d63/examples/self_managed_node_group/main.tf
 output "kubeconfig" {
   description = "Kubernetes connection configuration kubeconfig"
diff --git a/src/_nebari/stages/kubernetes_initialize/__init__.py b/src/_nebari/stages/kubernetes_initialize/__init__.py
index 04f2b60415..e40a69ed0f 100644
--- a/src/_nebari/stages/kubernetes_initialize/__init__.py
+++ b/src/_nebari/stages/kubernetes_initialize/__init__.py
@@ -49,7 +49,6 @@ class InputVars(schema.Base):
     external_container_reg: Union[ExtContainerReg, None] = None
     gpu_enabled: bool = False
     gpu_node_group_names: List[str] = []
-    asg_node_group_map: Dict[str, str] = {}
 
 
 class InputSchema(schema.Base):
@@ -97,9 +96,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 group for group in self.config.amazon_web_services.node_groups.keys()
             ]
             input_vars.aws_region = self.config.amazon_web_services.region
-            input_vars.asg_node_group_map = stage_outputs["stages/02-infrastructure"][
-                "asg_node_group_map"
-            ]
+
         return input_vars.dict()
 
     def check(
diff --git a/src/_nebari/stages/kubernetes_initialize/template/main.tf b/src/_nebari/stages/kubernetes_initialize/template/main.tf
index fe1b27a48e..402c68fb3f 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/main.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/main.tf
@@ -1,13 +1,17 @@
 module "kubernetes-initialization" {
-  source    = "./modules/initialization"
+  source = "./modules/initialization"
+
   namespace = var.environment
   secrets   = []
 }
 
 module "kubernetes-autoscaling" {
-  count        = var.cloud_provider == "aws" ? 1 : 0
-  source       = "./modules/cluster-autoscaler"
-  namespace    = var.environment
+  count = var.cloud_provider == "aws" ? 1 : 0
+
+  source = "./modules/cluster-autoscaler"
+
+  namespace = var.environment
+
   aws_region   = var.aws_region
   cluster-name = local.cluster_name
 }
@@ -17,15 +21,11 @@ module "traefik-crds" {
 }
 
 module "nvidia-driver-installer" {
-  count                = var.gpu_enabled ? 1 : 0
-  source               = "./modules/nvidia-installer"
+  count = var.gpu_enabled ? 1 : 0
+
+  source = "./modules/nvidia-installer"
+
   cloud_provider       = var.cloud_provider
   gpu_enabled          = var.gpu_enabled
   gpu_node_group_names = var.gpu_node_group_names
 }
-
-#module "tagging" {
-#  count              = var.cloud_provider == "aws" ? 1 : 0
-#  source             = "./modules/tagging"
-#  asg_node_group_map = var.cloud_provider == "aws" ? var.asg_node_group_map : {}
-#}
diff --git a/src/_nebari/stages/kubernetes_initialize/template/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
index 987e47b729..f169f5bcf2 100644
--- a/src/_nebari/stages/kubernetes_initialize/template/variables.tf
+++ b/src/_nebari/stages/kubernetes_initialize/template/variables.tf
@@ -30,8 +30,3 @@ variable "gpu_enabled" {
 variable "gpu_node_group_names" {
   description = "Names of node groups with GPU"
 }
-
-variable "asg_node_group_map" {
-  description = "Mapping between asg name and node group name."
-  type        = map(string)
-}

From 240f36fc91c0950b432231529a5b806a3e323779 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 14:47:43 +0000
Subject: [PATCH 82/92] Add back original tags.

---
 .../services/dask-gateway/files/gateway_config.py           | 1 +
 .../template/modules/kubernetes/services/jupyterhub/main.tf | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 083741f2a1..76f1dfbc65 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -116,6 +116,7 @@ def list_dask_environments():
 def base_node_group(options):
     default_node_group = {
         "dedicated": config["worker-node-group"]["value"],
+        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 9c8a3b7d46..2ab24042f2 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -173,14 +173,16 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "dedicated" = var.user-node-group.value
+          "dedicated"                  = var.user-node-group.value
+          "${var.user-node-group.key}" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "dedicated" = var.general-node-group.value
+            "dedicated"                     = var.general-node-group.value
+            "${var.general-node-group.key}" = var.general-node-group.value
           }
         }
       }

From 703763781d429f06dbe1fbe5cb9ef81a6c872f9e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 14:48:51 +0000
Subject: [PATCH 83/92] Remove unused tagging from stage 3.

---
 .../template/modules/tagging/aws-asg-tagging.tf        | 10 ----------
 .../template/modules/tagging/variables.tf              |  4 ----
 2 files changed, 14 deletions(-)
 delete mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
 delete mode 100644 src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf

diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
deleted file mode 100644
index c1abf21250..0000000000
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/aws-asg-tagging.tf
+++ /dev/null
@@ -1,10 +0,0 @@
-resource "aws_autoscaling_group_tag" "aws_eks_scaling_from_zero_labeling" {
-  count                  = length(var.asg_node_group_map)
-  autoscaling_group_name = element(keys(var.asg_node_group_map), count.index)
-
-  tag {
-    key                 = "k8s.io/cluster-autoscaler/node-template/label/dedicated"
-    value               = var.asg_node_group_map[element(keys(var.asg_node_group_map), count.index)]
-    propagate_at_launch = true
-  }
-}
diff --git a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf b/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
deleted file mode 100644
index e5c82f3dc5..0000000000
--- a/src/_nebari/stages/kubernetes_initialize/template/modules/tagging/variables.tf
+++ /dev/null
@@ -1,4 +0,0 @@
-variable "asg_node_group_map" {
-  description = "Mapping between asg name and node group name."
-  type        = map(string)
-}

From fa31975318cfaac8e9405d2cc56031ab2f21c16e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 15:56:22 +0000
Subject: [PATCH 84/92] Remove extra config.

---
 .../template/aws/modules/kubernetes/main.tf               | 6 +++---
 .../services/dask-gateway/files/gateway_config.py         | 2 +-
 .../modules/kubernetes/services/jupyterhub/main.tf        | 8 ++++----
 3 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index 43b1c0f3c9..5341d11ef3 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -57,9 +57,9 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    "kubernetes.io/cluster/${var.name}"                       = "shared"
-    "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
-    propagate_at_launch                                       = true
+    "kubernetes.io/cluster/${var.name}" = "shared"
+    # "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
+    # propagate_at_launch = true
   }, var.tags)
 }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 76f1dfbc65..069fb98d8f 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,7 +115,7 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        "dedicated": config["worker-node-group"]["value"],
+        # "dedicated": config["worker-node-group"]["value"],
         config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 2ab24042f2..90db0ad4f4 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -173,16 +173,16 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "dedicated"                  = var.user-node-group.value
-          "${var.user-node-group.key}" = var.user-node-group.value
+          "dedicated" = var.user-node-group.value
+          # "${var.user-node-group.key}" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "dedicated"                     = var.general-node-group.value
-            "${var.general-node-group.key}" = var.general-node-group.value
+            "dedicated" = var.general-node-group.value
+            # "${var.general-node-group.key}" = var.general-node-group.value
           }
         }
       }

From 46bb93cd0fb07b33fc0e5a25e9af29c3dcb6530e Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 16:05:10 +0000
Subject: [PATCH 85/92] Fix dask config.

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 069fb98d8f..85fa979b68 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,8 +115,8 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        # "dedicated": config["worker-node-group"]["value"],
-        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
+        "dedicated": config["worker-node-group"]["value"],
+        # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first

From 94b2ba22911eef7869cc98d979ba0d2fcc31aa44 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Wed, 17 Jan 2024 22:35:15 +0000
Subject: [PATCH 86/92] Revert changes for autoscaling.

---
 .../infrastructure/template/aws/modules/kubernetes/main.tf  | 6 +++---
 .../services/dask-gateway/files/gateway_config.py           | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
index 5341d11ef3..848d1c0471 100644
--- a/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
+++ b/src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf
@@ -57,9 +57,9 @@ resource "aws_eks_node_group" "main" {
   ]
 
   tags = merge({
-    "kubernetes.io/cluster/${var.name}" = "shared"
-    # "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
-    # propagate_at_launch = true
+    #    "kubernetes.io/cluster/${var.name}"                       = "shared"
+    "k8s.io/cluster-autoscaler/node-template/label/dedicated" = var.node_groups[count.index].name
+    propagate_at_launch                                       = true
   }, var.tags)
 }
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 85fa979b68..069fb98d8f 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,8 +115,8 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        "dedicated": config["worker-node-group"]["value"],
-        # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
+        # "dedicated": config["worker-node-group"]["value"],
+        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first

From 7e8b00304ce8bee488c5707a0f9bbbefa40b8d7a Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 18 Jan 2024 10:58:39 +0000
Subject: [PATCH 87/92] Revert changes to jupyerhub and dask servers, temprarly
 placing scheduler back on user node to check local deployment is effected.

---
 .../services/dask-gateway/files/gateway_config.py         | 4 ++--
 .../modules/kubernetes/services/jupyterhub/main.tf        | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 069fb98d8f..85fa979b68 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -115,8 +115,8 @@ def list_dask_environments():
 
 def base_node_group(options):
     default_node_group = {
-        # "dedicated": config["worker-node-group"]["value"],
-        config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
+        "dedicated": config["worker-node-group"]["value"],
+        # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 
     # check `worker_extra_pod_config` first
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 90db0ad4f4..40975441c3 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -173,16 +173,16 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          "dedicated" = var.user-node-group.value
-          # "${var.user-node-group.key}" = var.user-node-group.value
+          #          "dedicated" = var.user-node-group.value
+          "${var.user-node-group.key}" = var.user-node-group.value
         }
       }
 
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            "dedicated" = var.general-node-group.value
-            # "${var.general-node-group.key}" = var.general-node-group.value
+            # "dedicated" = var.general-node-group.value
+            "${var.general-node-group.key}" = var.user-node-group.value
           }
         }
       }

From 2bd5686682dbc1b8ffb173831414ce259e977fd4 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 18 Jan 2024 11:29:07 +0000
Subject: [PATCH 88/92] Fix revert.

---
 .../template/modules/kubernetes/services/jupyterhub/main.tf     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 40975441c3..7bdddb5cd6 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -182,7 +182,7 @@ resource "helm_release" "jupyterhub" {
         userScheduler = {
           nodeSelector = {
             # "dedicated" = var.general-node-group.value
-            "${var.general-node-group.key}" = var.user-node-group.value
+            "${var.user-node-group.key}" = var.user-node-group.value
           }
         }
       }

From e395098e4d7faa6fbc1ad4b475f60b7320288398 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 18 Jan 2024 14:16:37 +0000
Subject: [PATCH 89/92] Conditionally add lables for AWS.

---
 src/_nebari/stages/kubernetes_services/__init__.py  |  6 ++++--
 .../kubernetes_services/template/jupyterhub.tf      |  6 ++++++
 .../modules/kubernetes/services/jupyterhub/main.tf  | 13 +++++++++----
 .../kubernetes/services/jupyterhub/variables.tf     |  5 +++++
 4 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 212b9e6823..2ded180421 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -7,8 +7,6 @@
 from urllib.parse import urlencode
 
 import pydantic
-from pydantic import Field
-
 from _nebari import constants
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.tf_objects import (
@@ -20,6 +18,7 @@
 from _nebari.version import __version__
 from nebari import schema
 from nebari.hookspecs import NebariStage, hookimpl
+from pydantic import Field
 
 # check and retry settings
 NUM_ATTEMPTS = 10
@@ -353,6 +352,7 @@ class JupyterhubInputVars(schema.Base):
     idle_culler_settings: Dict[str, Any] = Field(alias="idle-culler-settings")
     argo_workflows_enabled: bool = Field(alias="argo-workflows-enabled")
     jhub_apps_enabled: bool = Field(alias="jhub-apps-enabled")
+    cloud_provider: str = Field(alias="cloud-provider")
 
 
 class DaskGatewayInputVars(schema.Base):
@@ -402,6 +402,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
         realm_id = stage_outputs["stages/06-kubernetes-keycloak-configuration"][
             "realm_id"
         ]["value"]
+        cloud_provider = self.config.provider.value
         jupyterhub_shared_endpoint = (
             stage_outputs["stages/02-infrastructure"]
             .get("nfs_endpoint", {})
@@ -477,6 +478,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
             ),
             jupyterhub_stared_storage=self.config.storage.shared_filesystem,
             jupyterhub_shared_endpoint=jupyterhub_shared_endpoint,
+            cloud_provider=cloud_provider,
             jupyterhub_profiles=self.config.profiles.dict()["jupyterlab"],
             jupyterhub_image=_split_docker_image_name(
                 self.config.default_images.jupyterhub
diff --git a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
index 9bc88ea22d..396f0e3677 100644
--- a/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
+++ b/src/_nebari/stages/kubernetes_services/template/jupyterhub.tf
@@ -50,6 +50,10 @@ variable "idle-culler-settings" {
   type        = any
 }
 
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}
 
 module "kubernetes-nfs-server" {
   count = var.jupyterhub-shared-endpoint == null ? 1 : 0
@@ -83,6 +87,8 @@ module "jupyterhub" {
   name      = var.name
   namespace = var.environment
 
+  cloud-provider = var.cloud-provider
+
   external-url = var.endpoint
   realm_id     = var.realm_id
 
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 7bdddb5cd6..01f73aa72f 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -16,8 +16,11 @@ resource "random_password" "jhub_apps_jwt_secret" {
 }
 
 locals {
-  jhub_apps_secrets_name = "jhub-apps-secrets"
-  jhub_apps_env_var_name = "JHUB_APP_JWT_SECRET_KEY"
+  jhub_apps_secrets_name           = "jhub-apps-secrets"
+  jhub_apps_env_var_name           = "JHUB_APP_JWT_SECRET_KEY"
+  singleuser_nodeselector_key      = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
+  userscheduler_nodeselector_key   = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
+  userscheduler_nodeselector_value = var.cloud-provider == "aws" ? var.general-node-group.value : var.user-node-group.key
 }
 
 resource "kubernetes_secret" "jhub_apps_secrets" {
@@ -174,7 +177,8 @@ resource "helm_release" "jupyterhub" {
         image = var.jupyterlab-image
         nodeSelector = {
           #          "dedicated" = var.user-node-group.value
-          "${var.user-node-group.key}" = var.user-node-group.value
+          # "${var.user-node-group.key}" = var.user-node-group.value
+          "${local.singleuser_nodeselector_key}" = var.user-node-group.value
         }
       }
 
@@ -182,7 +186,8 @@ resource "helm_release" "jupyterhub" {
         userScheduler = {
           nodeSelector = {
             # "dedicated" = var.general-node-group.value
-            "${var.user-node-group.key}" = var.user-node-group.value
+            # "${var.user-node-group.key}" = var.user-node-group.value
+            "${local.userscheduler_nodeselector_key}" = local.userscheduler_nodeselector_value
           }
         }
       }
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
index f048292b36..d268bbe944 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
@@ -167,3 +167,8 @@ variable "jupyterlab-pioneer-log-format" {
   description = "Logging format for JupyterLab Pioneer"
   type        = string
 }
+
+variable "cloud-provider" {
+  description = "Name of cloud provider."
+  type        = string
+}
\ No newline at end of file

From 4235594a97cfddd9e2ae4bf74090da219e3bdc03 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Thu, 18 Jan 2024 14:17:15 +0000
Subject: [PATCH 90/92] [pre-commit.ci] Apply automatic pre-commit fixes

---
 src/_nebari/stages/kubernetes_services/__init__.py             | 3 ++-
 .../modules/kubernetes/services/jupyterhub/variables.tf        | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py
index 2ded180421..4fbed53ca1 100644
--- a/src/_nebari/stages/kubernetes_services/__init__.py
+++ b/src/_nebari/stages/kubernetes_services/__init__.py
@@ -7,6 +7,8 @@
 from urllib.parse import urlencode
 
 import pydantic
+from pydantic import Field
+
 from _nebari import constants
 from _nebari.stages.base import NebariTerraformStage
 from _nebari.stages.tf_objects import (
@@ -18,7 +20,6 @@
 from _nebari.version import __version__
 from nebari import schema
 from nebari.hookspecs import NebariStage, hookimpl
-from pydantic import Field
 
 # check and retry settings
 NUM_ATTEMPTS = 10
diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
index d268bbe944..1897735307 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/variables.tf
@@ -171,4 +171,4 @@ variable "jupyterlab-pioneer-log-format" {
 variable "cloud-provider" {
   description = "Name of cloud provider."
   type        = string
-}
\ No newline at end of file
+}

From fd146452f0bfb96e60a89567da1be473d76a9e20 Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 18 Jan 2024 16:19:23 +0000
Subject: [PATCH 91/92] Fix dask config.

---
 .../kubernetes/services/dask-gateway/files/gateway_config.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
index 85fa979b68..12416e2527 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/dask-gateway/files/gateway_config.py
@@ -114,8 +114,11 @@ def list_dask_environments():
 
 
 def base_node_group(options):
+    key = config["worker-node-group"]["key"]
+    if config.provider.value == "aws":
+        key = "dedicated"
     default_node_group = {
-        "dedicated": config["worker-node-group"]["value"],
+        key: config["worker-node-group"]["value"],
         # config["worker-node-group"]["key"]: config["worker-node-group"]["value"],
     }
 

From 5c45b2d11fca1ccc69a68b2028fec9916fa197ef Mon Sep 17 00:00:00 2001
From: Prashant Tiwari <prashant.tiwari@genesisoilandgas.com>
Date: Thu, 18 Jan 2024 16:29:55 +0000
Subject: [PATCH 92/92] Fix jupyterhub nodeselectors.

---
 .../template/modules/kubernetes/services/jupyterhub/main.tf | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
index 6c1b990a4d..af690112f6 100644
--- a/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
+++ b/src/_nebari/stages/kubernetes_services/template/modules/kubernetes/services/jupyterhub/main.tf
@@ -20,7 +20,7 @@ locals {
   jhub_apps_env_var_name           = "JHUB_APP_JWT_SECRET_KEY"
   singleuser_nodeselector_key      = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
   userscheduler_nodeselector_key   = var.cloud-provider == "aws" ? "dedicated" : var.user-node-group.key
-  userscheduler_nodeselector_value = var.cloud-provider == "aws" ? var.general-node-group.value : var.user-node-group.key
+  userscheduler_nodeselector_value = var.cloud-provider == "aws" ? var.general-node-group.value : var.user-node-group.value
 }
 
 resource "kubernetes_secret" "jhub_apps_secrets" {
@@ -177,8 +177,6 @@ resource "helm_release" "jupyterhub" {
       singleuser = {
         image = var.jupyterlab-image
         nodeSelector = {
-          #          "dedicated" = var.user-node-group.value
-          # "${var.user-node-group.key}" = var.user-node-group.value
           "${local.singleuser_nodeselector_key}" = var.user-node-group.value
         }
       }
@@ -186,8 +184,6 @@ resource "helm_release" "jupyterhub" {
       scheduling = {
         userScheduler = {
           nodeSelector = {
-            # "dedicated" = var.general-node-group.value
-            # "${var.user-node-group.key}" = var.user-node-group.value
             "${local.userscheduler_nodeselector_key}" = local.userscheduler_nodeselector_value
           }
         }