Update nginx frontend config to support additional clusters. (#1759)

* Update frontend config to support additional clusters.

Additionally updates the local devel setup to easily add an additional
cluster with oidc options pointing to dex running on the initial
cluster.

* Do not require CA data for additional clusters.

chart/kubeapps/templates/kubeapps-frontend-config.yaml

@@ -25,7 +25,11 @@ data:
     # Google Kubernetes Engine requires the access_token as the Bearer when talking to the k8s api server.
     proxy_set_header Authorization "Bearer $http_x_forwarded_access_token";
     {{- end }}
-
+  {{- range .Values.featureFlags.additionalClusters }}
+  {{- if .certificateAuthorityData }}
+  {{ .name }}-ca.pem: {{ .certificateAuthorityData }}
+   {{- end }}
+  {{- end}}
   vhost.conf: |-
     # Retain the default nginx handling of requests without a "Connection" header
     map $http_upgrade $connection_upgrade {
@@ -55,6 +59,21 @@ data:
         include "./server_blocks/k8s-api-proxy.conf";
       }
 
+      # Ensure each additional cluster can be reached (should only be
+      # used with an auth-proxy where k8s credentials never leave
+      # the cluster). See additionalClusters option.
+      {{- range .Values.featureFlags.additionalClusters }}
+      location ~* /api/clusters/{{ .name }} {
+        rewrite /api/clusters/{{ .name }}/(.*) /$1 break;
+        rewrite /api/clusters/{{ .name }} / break;
+        proxy_pass {{ .apiServiceURL }};
+        {{- if .certificateAuthorityData }}
+        proxy_ssl_trusted_certificate "./server_blocks/{{ .name }}-ca.pem";
+        {{- end }}
+        include "./server_blocks/k8s-api-proxy.conf";
+      }
+      {{- end }}
+
       # TODO: The following location is left for backwards compat but will no longer
       # be needed once clients are sending the cluster name.
       # Using regexp match instead of prefix one because the application can be

chart/kubeapps/values.yaml

@@ -694,3 +694,9 @@ featureFlags:
   reposPerNamespace: true
   invalidateCache: true
   operators: false
+  # additionalClusters is a WIP feature for multi-cluster support.
+  additionalClusters: []
+  # additionalClusters:
+  # - name: second-cluster
+  #   apiServiceURL: https://second-cluster:6443
+  #   certificateAuthorityData: LS0tLS1CRUdJ...

docs/user/manifests/kubeapps-local-dev-additional-apiserver-config.json

@@ -0,0 +1,24 @@
+{
+  "kind": "Cluster",
+  "apiVersion": "kind.sigs.k8s.io/v1alpha3",
+  "nodes": [
+    {
+      "role": "control-plane",
+      "extraMounts": [
+        {
+          "readOnly": true,
+          "hostPath": "./devel/dex.crt",
+          "containerPath": "/etc/ssl/certs/dex.crt"
+        }
+      ]
+    }
+  ],
+  "kubeadmConfigPatchesJson6902": [
+    {
+      "group": "kubeadm.k8s.io",
+      "version": "v1beta2",
+      "kind": "ClusterConfiguration",
+      "patch": "[{ \"op\": \"add\", \"path\": \"/apiServer/extraArgs\", \"value\": {}}, {\"op\": \"add\", \"path\": \"/apiServer/extraArgs/oidc-issuer-url\", \"value\": \"https://172.18.0.2:32000\"}, {\"op\": \"add\", \"path\": \"/apiServer/extraArgs/oidc-client-id\", \"value\": \"kubeapps\"}, {\"op\": \"add\", \"path\": \"/apiServer/extraArgs/oidc-ca-file\", \"value\": \"/etc/ssl/certs/dex.crt\"}, {\"op\": \"add\", \"path\": \"/apiServer/extraArgs/oidc-username-claim\", \"value\": \"email\"}, {\"op\": \"add\", \"path\": \"/apiServer/extraArgs/oidc-username-prefix\",\"value\": \"oidc:\"}]"
+    }
+  ]
+}

script/cluster-kind.mk

@@ -2,20 +2,44 @@
 # with OIDC integration for development and testing.
 KUBE ?= ${HOME}/.kube
 CLUSTER_NAME ?= kubeapps
+ADDITIONAL_CLUSTER_NAME ?= kubeapps-additional
 
 CLUSTER_CONFIG = ${KUBE}/kind-config-${CLUSTER_NAME}
+ADDITIONAL_CLUSTER_CONFIG = ${KUBE}/kind-config-${ADDITIONAL_CLUSTER_NAME}
 
 ${CLUSTER_CONFIG}:
 	kind create cluster \
 		--kubeconfig ${CLUSTER_CONFIG} \
 		--name ${CLUSTER_NAME} \
 		--config=./docs/user/manifests/kubeapps-local-dev-apiserver-config.json \
 		--retain
+	kubectl apply --kubeconfig=$CLUSTER_CONFIG -f ./docs/user/manifests/kubeapps-local-dev-users-rbac.yaml
 
 cluster-kind: ${CLUSTER_CONFIG}
 
+# dex will be running on the same node as the API server in the dev environment, so we can
+# reuse the key and cert from the apiserver, which already includes v3 extensions
+# for the correct alternative name (using the IP address).
+devel/dex.crt:
+	kubectl -n kube-system cp kube-apiserver-kubeapps-control-plane:etc/kubernetes/pki/apiserver.crt ./devel/dex.crt
+
+devel/dex.key:
+	kubectl -n kube-system cp kube-apiserver-kubeapps-control-plane:etc/kubernetes/pki/apiserver.key ./devel/dex.key
+
+${ADDITIONAL_CLUSTER_CONFIG}: devel/dex.crt
+	kind create cluster \
+		--kubeconfig ${ADDITIONAL_CLUSTER_CONFIG} \
+		--name ${ADDITIONAL_CLUSTER_NAME} \
+		--config=./docs/user/manifests/kubeapps-local-dev-additional-apiserver-config.json \
+		--retain
+	kubectl apply --kubeconfig=$ADDITIONAL_CLUSTER_CONFIG -f ./docs/user/manifests/kubeapps-local-dev-users-rbac.yaml
+
+additional-cluster-kind: ${ADDITIONAL_CLUSTER_CONFIG}
+
 delete-cluster-kind:
 	kind delete cluster --name ${CLUSTER_NAME} || true
+	kind delete cluster --name ${ADDITIONAL_CLUSTER_NAME} || true
 	rm ${CLUSTER_CONFIG}
+	rm ${ADDITIONAL_CLUSTER_CONFIG} || true
 
-.PHONY: cluster-kind cluster-kind-delete
+.PHONY: additional-cluster-kind cluster-kind cluster-kind-delete

script/deploy-dev.mk

@@ -4,16 +4,11 @@
 # Targets in this helper assume that kubectl is configured with a cluster
 # that has been setup with OIDC support (see ./cluster-kind.mk)
 
-deploy-dex:
+deploy-dex: devel/dex.crt devel/dex.key
 	kubectl create namespace dex
-	# dex is running on the same node as the API server in the dev environment, so we can
-	# reuse the key and cert from the apiserver, which already includes v3 extensions
-	# for the correct alternative name (using the IP address).
-	kubectl -n kube-system cp kube-apiserver-kubeapps-control-plane:etc/kubernetes/pki/apiserver.crt ./devel/apiserver.crt
-	kubectl -n kube-system cp kube-apiserver-kubeapps-control-plane:etc/kubernetes/pki/apiserver.key ./devel/apiserver.key
 	kubectl -n dex create secret tls dex-web-server-tls \
-		--key ./devel/apiserver.key \
-		--cert ./devel/apiserver.crt
+		--key ./devel/dex.key \
+		--cert ./devel/dex.crt
 	helm install dex stable/dex --namespace dex --version 2.4.0 \
 		--values ./docs/user/manifests/kubeapps-local-dev-dex-values.yaml
 
@@ -28,7 +23,6 @@ deploy-dev: deploy-dex deploy-openldap
 		--values ./docs/user/manifests/kubeapps-local-dev-values.yaml \
 		--values ./docs/user/manifests/kubeapps-local-dev-auth-proxy-values.yaml \
 		--set useHelm3=true
-	kubectl apply -f ./docs/user/manifests/kubeapps-local-dev-users-rbac.yaml
 	@echo "\nYou can now simply open your browser at http://172.18.0.2:30000 to access Kubeapps!"
 	@echo "When logging in, you will be redirected to dex (with a self-signed cert) and can login with email as either of"
 	@echo "  kubeapps-operator@example.com:password"