From d053784a80fb4d461a9c3f5dd324dce75e884ca2 Mon Sep 17 00:00:00 2001 From: Adam Lewis <23342526+Adam-D-Lewis@users.noreply.github.com> Date: Tue, 23 May 2023 09:19:16 -0500 Subject: [PATCH] Add Argo Workflow Admission controller (#1741) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- nebari/constants.py | 2 + nebari/deploy.py | 2 +- nebari/schema.py | 6 + nebari/stages/input_vars.py | 13 + .../modules/kubernetes/keycloak-helm/main.tf | 2 +- .../main.tf | 28 ++ .../outputs.tf | 11 + .../permissions.tf | 15 + .../07-kubernetes-services/argo-workflows.tf | 25 +- .../services/argo-workflows/main.tf | 303 +++++++++++++++++- .../services/argo-workflows/variables.tf | 17 + .../files/jupyterhub/03-profiles.py | 14 +- 12 files changed, 430 insertions(+), 8 deletions(-) diff --git a/nebari/constants.py b/nebari/constants.py index 1865188b3c..f76549ef88 100644 --- a/nebari/constants.py +++ b/nebari/constants.py @@ -11,4 +11,6 @@ DEFAULT_CONDA_STORE_IMAGE_TAG = "v0.4.14" +DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "update_nwc-05c3b99-20230512" + LATEST_SUPPORTED_PYTHON_VERSION = "3.10" diff --git a/nebari/deploy.py b/nebari/deploy.py index 3f7361c494..eae6b20a32 100644 --- a/nebari/deploy.py +++ b/nebari/deploy.py @@ -235,7 +235,7 @@ def guided_install( provision_07_kubernetes_services(stage_outputs, config, disable_checks) provision_08_nebari_tf_extensions(stage_outputs, config, disable_checks) - print("Nebari deployed successfully") + print("Nebari deployed successfully") print("Services:") for service_name, service in stage_outputs["stages/07-kubernetes-services"][ diff --git a/nebari/schema.py b/nebari/schema.py index a8cef633bc..d160e679dc 100644 --- a/nebari/schema.py +++ b/nebari/schema.py @@ -86,9 +86,15 @@ class HelmExtension(Base): # ============== Argo-Workflows ========= +class NebariWorkflowController(Base): + enabled: bool + image_tag: typing.Optional[str] + + class ArgoWorkflows(Base): enabled: bool overrides: typing.Optional[typing.Dict] + nebari_workflow_controller: typing.Optional[NebariWorkflowController] # ============== kbatch ============= diff --git a/nebari/stages/input_vars.py b/nebari/stages/input_vars.py index 0eafdb1cb8..c3e25c010c 100644 --- a/nebari/stages/input_vars.py +++ b/nebari/stages/input_vars.py @@ -6,6 +6,7 @@ from nebari.constants import ( DEFAULT_CONDA_STORE_IMAGE_TAG, DEFAULT_GKE_RELEASE_CHANNEL, + DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG, DEFAULT_TRAEFIK_IMAGE_TAG, ) @@ -349,6 +350,18 @@ def stage_07_kubernetes_services(stage_outputs, config): "argo-workflows-overrides": [ json.dumps(config.get("argo_workflows", {}).get("overrides", {})) ], + "nebari-workflow-controller": config["argo_workflows"] + .get("nebari_workflow_controller", {}) + .get("enabled", True), + "keycloak-read-only-user-credentials": stage_outputs[ + "stages/06-kubernetes-keycloak-configuration" + ]["keycloak-read-only-user-credentials"]["value"], + "workflow-controller-image-tag": config.get("argo_workflows", {}) + .get("nebari_workflow_controller", {}) + .get( + "image_tag", + DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG, + ), # kbatch "kbatch-enabled": config["kbatch"]["enabled"], # prefect diff --git a/nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf b/nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf index 700664e283..d9e804ee98 100644 --- a/nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf +++ b/nebari/template/stages/05-kubernetes-keycloak/modules/kubernetes/keycloak-helm/main.tf @@ -23,7 +23,7 @@ resource "helm_release" "keycloak" { }) ], var.overrides) - set { + set_sensitive { name = "nebari_bot_password" value = var.nebari-bot-password } diff --git a/nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf b/nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf index 25320b556f..fc7175ff58 100644 --- a/nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf +++ b/nebari/template/stages/06-kubernetes-keycloak-configuration/main.tf @@ -47,6 +47,34 @@ resource "keycloak_default_groups" "default" { ] } +data "keycloak_realm" "master" { + realm = "master" +} + +resource "random_password" "keycloak-view-only-user-password" { + length = 32 + special = false +} + +resource "keycloak_user" "read-only-user" { + realm_id = data.keycloak_realm.master.id + username = "read-only-user" + initial_password { + value = random_password.keycloak-view-only-user-password.result + temporary = false + } +} + +resource "keycloak_user_roles" "user_roles" { + realm_id = data.keycloak_realm.master.id + user_id = keycloak_user.read-only-user.id + + role_ids = [ + data.keycloak_role.view-users.id, + ] + exhaustive = true +} + # needed for keycloak monitoring to function resource "keycloak_realm_events" "realm_events" { realm_id = keycloak_realm.main.id diff --git a/nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf b/nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf index ed7daabf16..f70f6cbb6f 100644 --- a/nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf +++ b/nebari/template/stages/06-kubernetes-keycloak-configuration/outputs.tf @@ -2,3 +2,14 @@ output "realm_id" { description = "Realm id used for nebari resources" value = keycloak_realm.main.id } + +output "keycloak-read-only-user-credentials" { + description = "Credentials for user that can read users/groups, but not modify them" + sensitive = true + value = { + username = keycloak_user.read-only-user.username + password = random_password.keycloak-view-only-user-password.result + client_id = "admin-cli" + realm = data.keycloak_realm.master.realm + } +} diff --git a/nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf b/nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf index cce54d0720..c866bea5f0 100644 --- a/nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf +++ b/nebari/template/stages/06-kubernetes-keycloak-configuration/permissions.tf @@ -9,6 +9,21 @@ data "keycloak_role" "manage-users" { name = "manage-users" } +data "keycloak_openid_client" "nebari-realm" { + depends_on = [ + keycloak_realm.main, + ] + realm_id = data.keycloak_realm.master.id + client_id = "${var.realm}-realm" +} + +data "keycloak_role" "view-users" { + realm_id = data.keycloak_realm.master.id + client_id = data.keycloak_openid_client.nebari-realm.id + name = "view-users" +} + + data "keycloak_role" "query-users" { realm_id = keycloak_realm.main.id client_id = data.keycloak_openid_client.realm_management.id diff --git a/nebari/template/stages/07-kubernetes-services/argo-workflows.tf b/nebari/template/stages/07-kubernetes-services/argo-workflows.tf index 22b448207b..34927f2b59 100644 --- a/nebari/template/stages/07-kubernetes-services/argo-workflows.tf +++ b/nebari/template/stages/07-kubernetes-services/argo-workflows.tf @@ -11,6 +11,24 @@ variable "argo-workflows-overrides" { default = [] } +variable "nebari-workflow-controller" { + description = "Nebari Workflow Controller enabled" + type = bool + default = true +} + + +variable "keycloak-read-only-user-credentials" { + description = "Keycloak password for nebari-bot" + type = map(string) + default = {} +} + +variable "workflow-controller-image-tag" { + description = "Image tag for nebari-workflow-controller" + type = string +} + # ====================== RESOURCES ======================= module "argo-workflows" { @@ -21,6 +39,9 @@ module "argo-workflows" { external-url = var.endpoint realm_id = var.realm_id - node-group = var.node_groups.general - overrides = var.argo-workflows-overrides + node-group = var.node_groups.general + overrides = var.argo-workflows-overrides + keycloak-read-only-user-credentials = var.keycloak-read-only-user-credentials + workflow-controller-image-tag = var.workflow-controller-image-tag + nebari-workflow-controller = var.nebari-workflow-controller } diff --git a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf index f374df02c6..a7182e6a37 100644 --- a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf +++ b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/main.tf @@ -123,6 +123,40 @@ resource "kubernetes_manifest" "argo-workflows-ingress-route" { spec = { entryPoints = ["websecure"] routes = [ + { + kind = "Rule" + match = "Host(`${var.external-url}`) && Path(`/${local.argo-workflows-prefix}/validate`)" + middlewares = concat( + [{ + name = kubernetes_manifest.argo-workflows-middleware-stripprefix.manifest.metadata.name + namespace = var.namespace + }] + ) + services = [ + { + name = "wf-admission-controller" + port = 8080 + namespace = var.namespace + } + ] + }, + { + kind = "Rule" + match = "Host(`${var.external-url}`) && Path(`/${local.argo-workflows-prefix}/mutate`)" + middlewares = concat( + [{ + name = kubernetes_manifest.argo-workflows-middleware-stripprefix.manifest.metadata.name + namespace = var.namespace + }] + ) + services = [ + { + name = "wf-admission-controller" + port = 8080 + namespace = var.namespace + } + ] + }, { kind = "Rule" match = "Host(`${var.external-url}`) && PathPrefix(`/${local.argo-workflows-prefix}`)" @@ -141,7 +175,7 @@ resource "kubernetes_manifest" "argo-workflows-ingress-route" { namespace = var.namespace } ] - } + }, ] } } @@ -265,3 +299,270 @@ resource "kubernetes_cluster_role_binding" "argo-view-rb" { namespace = var.namespace } } + +# Workflow Admission Controller +resource "kubernetes_role" "pod_viewer" { + + metadata { + name = "nebari-pod-viewer" + namespace = var.namespace + } + + rule { + api_groups = [""] + resources = ["pods"] + verbs = ["get", "list"] + } +} + +resource "kubernetes_role" "workflow_viewer" { + + metadata { + name = "nebari-workflow-viewer" + namespace = var.namespace + } + + rule { + api_groups = ["argoproj.io"] + resources = ["workflows", "cronworkflows"] + verbs = ["get", "list"] + } +} + +resource "kubernetes_service_account" "wf-admission-controller" { + metadata { + name = "wf-admission-controller-sa" + namespace = var.namespace + } +} + +resource "kubernetes_role_binding" "wf-admission-controller-pod-viewer" { + metadata { + name = "wf-admission-controller-pod-viewer" + namespace = var.namespace + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role.pod_viewer.metadata.0.name + } + + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.wf-admission-controller.metadata.0.name + namespace = var.namespace + } +} + +resource "kubernetes_role_binding" "wf-admission-controller-wf-viewer" { + metadata { + name = "wf-admission-controller-wf-viewer" + namespace = var.namespace + } + + role_ref { + api_group = "rbac.authorization.k8s.io" + kind = "Role" + name = kubernetes_role.workflow_viewer.metadata.0.name + } + + subject { + kind = "ServiceAccount" + name = kubernetes_service_account.wf-admission-controller.metadata.0.name + namespace = var.namespace + } +} + + +resource "kubernetes_secret" "keycloak-read-only-user-credentials" { + metadata { + name = "keycloak-read-only-user-credentials" + namespace = var.namespace + } + + data = { + username = var.keycloak-read-only-user-credentials["username"] + password = var.keycloak-read-only-user-credentials["password"] + client_id = var.keycloak-read-only-user-credentials["client_id"] + realm = var.keycloak-read-only-user-credentials["realm"] + } + + type = "Opaque" +} + + +resource "kubernetes_manifest" "mutatingwebhookconfiguration_admission_controller" { + count = var.nebari-workflow-controller ? 1 : 0 + + manifest = { + "apiVersion" = "admissionregistration.k8s.io/v1" + "kind" = "MutatingWebhookConfiguration" + "metadata" = { + "name" = "wf-admission-controller" + } + "webhooks" = [ + { + "admissionReviewVersions" = [ + "v1", + "v1beta1", + ] + + "clientConfig" = { + "url" = "https://${var.external-url}/${local.argo-workflows-prefix}/mutate" + } + + "name" = "wf-mutating-admission-controller.${var.namespace}.svc" + "rules" = [ + { + "apiGroups" = [ + "argoproj.io", + ] + "apiVersions" = [ + "v1alpha1", + ] + "operations" = [ + "CREATE", + ] + "resources" = [ + "workflows", + "cronworkflows", + ] + }, + ] + "sideEffects" = "None" + }, + ] + } +} + +resource "kubernetes_manifest" "validatingwebhookconfiguration_admission_controller" { + count = var.nebari-workflow-controller ? 1 : 0 + manifest = { + "apiVersion" = "admissionregistration.k8s.io/v1" + "kind" = "ValidatingWebhookConfiguration" + "metadata" = { + "name" = "wf-admission-controller" + } + "webhooks" = [ + { + "admissionReviewVersions" = [ + "v1", + "v1beta1", + ] + "clientConfig" = { + "url" = "https://${var.external-url}/${local.argo-workflows-prefix}/validate" + } + "name" = "wf-validating-admission-controller.${var.namespace}.svc" + "rules" = [ + { + "apiGroups" = [ + "argoproj.io", + ] + "apiVersions" = [ + "v1alpha1", + ] + "operations" = [ + "CREATE", + ] + "resources" = [ + "workflows", + ] + }, + ] + "sideEffects" = "None" + }, + ] + } +} + +resource "kubernetes_manifest" "deployment_admission_controller" { + count = var.nebari-workflow-controller ? 1 : 0 + manifest = { + "apiVersion" = "apps/v1" + "kind" = "Deployment" + "metadata" = { + "name" = "nebari-workflow-controller" + "namespace" = var.namespace + } + "spec" = { + "replicas" = 1 + "selector" = { + "matchLabels" = { + "app" = "nebari-workflow-controller" + } + } + "template" = { + "metadata" = { + "labels" = { + "app" = "nebari-workflow-controller" + } + } + "spec" = { + serviceAccountName = kubernetes_service_account.wf-admission-controller.metadata.0.name + automountServiceAccountToken = true + "containers" = [ + { + command = ["bash", "-c"] + args = ["python -m nebari_workflow_controller"] + + "env" = [ + { + "name" = "KEYCLOAK_USERNAME" + "valueFrom" = { + "secretKeyRef" = { + "key" = "username" + "name" = "keycloak-read-only-user-credentials" + } + } + }, + { + "name" = "KEYCLOAK_PASSWORD" + "valueFrom" = { + "secretKeyRef" = { + "key" = "password" + "name" = "keycloak-read-only-user-credentials" + } + } + }, + { + "name" = "KEYCLOAK_URL" + "value" = "https://${var.external-url}/auth/" + }, + { + "name" = "NAMESPACE" + "value" = var.namespace + }, + ] + "image" = "quay.io/nebari/nebari-workflow-controller:${var.workflow-controller-image-tag}" + "name" = "admission-controller" + }, + ] + } + } + } + } +} + +resource "kubernetes_manifest" "service_admission_controller" { + manifest = { + "apiVersion" = "v1" + "kind" = "Service" + "metadata" = { + "name" = "wf-admission-controller" + "namespace" = var.namespace + } + "spec" = { + "ports" = [ + { + "name" = "wf-admission-controller" + "port" = 8080 + "targetPort" = 8080 + }, + ] + "selector" = { + "app" = "nebari-workflow-controller" + } + } + } +} diff --git a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf index 30a0b4a2a8..05007b7cf4 100644 --- a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf +++ b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/argo-workflows/variables.tf @@ -34,3 +34,20 @@ variable "realm_id" { description = "Keycloak realm to use for deploying openid client" type = string } + +variable "keycloak-read-only-user-credentials" { + sensitive = true + description = "Keycloak password for nebari-bot" + type = map(string) + default = {} +} + +variable "workflow-controller-image-tag" { + description = "Image tag for nebari-workflow-controller" + type = string +} + +variable "nebari-workflow-controller" { + description = "Nebari Workflow Controller enabled" + type = bool +} diff --git a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py index b564647ec2..255d5e1fe0 100644 --- a/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py +++ b/nebari/template/stages/07-kubernetes-services/modules/kubernetes/services/jupyterhub/files/jupyterhub/03-profiles.py @@ -59,7 +59,11 @@ def base_profile_home_mounts(username): "command": ["sh", "-c", command], "securityContext": {"runAsUser": 0}, "volumeMounts": [ - {"mountPath": "/mnt", "name": "home"}, + { + "mountPath": f"/mnt/{pvc_home_mount_path.format(username=username)}", + "name": "home", + "subPath": pvc_home_mount_path.format(username=username), + }, {"mountPath": "/etc/skel", "name": "skel"}, ], } @@ -118,9 +122,11 @@ def base_profile_shared_mounts(groups): "securityContext": {"runAsUser": 0}, "volumeMounts": [ { - "mountPath": "/mnt", + "mountPath": f"/mnt/{pvc_shared_mount_path.format(group=group)}", "name": "shared" if home_pvc_name != shared_pvc_name else "home", + "subPath": pvc_shared_mount_path.format(group=group), } + for group in groups ], } ] @@ -181,9 +187,11 @@ def profile_conda_store_mounts(username, groups): "securityContext": {"runAsUser": 0}, "volumeMounts": [ { - "mountPath": "/mnt", + "mountPath": f"/mnt/{namespace}", "name": "conda-store", + "subPath": namespace, } + for namespace in conda_store_namespaces ], } ]