[MΜ 26623] Migrate to single NGINX deployment, enable autoscaling.

README.md

@@ -224,3 +224,19 @@ terraform apply
 Tip: a quick and reliable way to get access to a cluster's terraform files and state is to use the `cloud workbench cluster` command. This will checkout the correct files locally in the same manner that the provisioning process uses.
 
 For more information on this change and reasoning for it, check out the [kops release notes](/~https://github.com/kubernetes/kops/releases/tag/v1.17.0).
+
+#### Cluster reprovisioning steps for new NGINX deployment
+
+This is related to the changes introduced in [PR-263](/~https://github.com/mattermost/mattermost-cloud/pull/263)
+
+Please follow the steps below for the reprovisioning of existing clusters:
+- Reprovision the cluster by running ```cloud cluster provision --cluster <cluster_id> --nginx-version 2.11.0```.
+- Check that new nginx deployed both internal and public Load Balancers (nginx-ingress-nginx-controller-internal and nginx-ingress-nginx-controller).
+- Manually update Prometheus Route53 record to use the new private Load Balancer (nginx-ingress-nginx-controller-internal).
+- Manually update cluster installations Route53 records one by one to use the new public Load balancer (nginx-ingress-nginx-controller).
+  - Update clusterinstallation ingress annotation to use Nginx class nginx-controller instead of nginx.
+  - Manually update network policy to target nginx instead of public-nginx.
+- Confirm that all services are up and running.
+- Delete old NGINX helm charts.
+  - ```helm del --purge public-nginx```
+  - ```helm del --purge private-nginx```

cmd/cloud/cluster.go

@@ -32,15 +32,13 @@ func init() {
 	clusterCreateCmd.Flags().String("prometheus-version", model.PrometheusDefaultVersion, "The version of Prometheus to provision. Use 'stable' to provision the latest stable version published upstream.")
 	clusterCreateCmd.Flags().String("fluentbit-version", model.FluentbitDefaultVersion, "The version of Fluentbit to provision. Use 'stable' to provision the latest stable version published upstream.")
 	clusterCreateCmd.Flags().String("nginx-version", model.NginxDefaultVersion, "The version of Nginx to provision. Use 'stable' to provision the latest stable version published upstream.")
-	clusterCreateCmd.Flags().String("public-nginx-version", model.PublicNginxDefaultVersion, "The version of Public Nginx to provision. Use 'stable' to provision the latest stable version published upstream.")
 	clusterCreateCmd.Flags().String("teleport-version", model.TeleportDefaultVersion, "The version of Teleport to provision. Use 'stable' to provision the latest stable version published upstream.")
 	clusterCreateCmd.Flags().Bool("dry-run", false, "When set to true, only print the cluster creation request without sending it.")
 
 	clusterProvisionCmd.Flags().String("cluster", "", "The id of the cluster to be provisioned.")
 	clusterProvisionCmd.Flags().String("prometheus-version", "", "The version of Prometheus to provision, no change if omitted. Use \"stable\" as an argument to this command to indicate that you wish to remove the pinned version and return the utility to tracking the latest version.")
 	clusterProvisionCmd.Flags().String("fluentbit-version", "", "The version of Fluentbit to provision, no change if omitted. Use \"stable\" as an argument to this command to indicate that you wish to remove the pinned version and return the utility to tracking the latest version.")
 	clusterProvisionCmd.Flags().String("nginx-version", "", "The version of Nginx to provision, no change if omitted. Use \"stable\" as an argument to this command to indicate that you wish to remove the pinned version and return the utility to tracking the latest version.")
-	clusterProvisionCmd.Flags().String("public-nginx-version", "", "The version of Public Nginx to provision, no change if omitted. Use \"stable\" as an argument to this command to indicate that you wish to remove the pinned version and return the utility to tracking the latest version.")
 	clusterProvisionCmd.Flags().String("teleport-version", "", "The version of Teleport to provision, no change if omitted. Use \"stable\" as an argument to this command to indicate that you wish to remove the pinned version and return the utility to tracking the latest version.")
 	clusterProvisionCmd.MarkFlagRequired("cluster")
 
@@ -455,7 +453,6 @@ func processUtilityFlags(command *cobra.Command) map[string]string {
 	prometheusVersion, _ := command.Flags().GetString("prometheus-version")
 	fluentbitVersion, _ := command.Flags().GetString("fluentbit-version")
 	nginxVersion, _ := command.Flags().GetString("nginx-version")
-	publicNginxVersion, _ := command.Flags().GetString("public-nginx-version")
 	teleportVersion, _ := command.Flags().GetString("teleport-version")
 
 	utilityVersions := make(map[string]string)
@@ -472,10 +469,6 @@ func processUtilityFlags(command *cobra.Command) map[string]string {
 		utilityVersions[model.NginxCanonicalName] = nginxVersion
 	}
 
-	if publicNginxVersion != "" {
-		utilityVersions[model.PublicNginxCanonicalName] = publicNginxVersion
-	}
-
 	if teleportVersion != "" {
 		utilityVersions[model.TeleportCanonicalName] = teleportVersion
 	}

helm-charts/public-nginx_values.yaml → helm-charts/nginx_values.yaml

@@ -2,14 +2,30 @@
 ## Ref: /~https://github.com/kubernetes/ingress/blob/master/controllers/nginx/configuration.md
 ##
 controller:
-  ingressClass: nginx
+  ingressClass: nginx-controller
+  image:
+    repository: us.gcr.io/k8s-artifacts-prod/ingress-nginx/controller
+    digest: sha256:56633bd00dab33d92ba14c6e709126a762d54a75a6e72437adefeaaca0abb069
+    pullPolicy: IfNotPresent
+    # www-data -> uid 101
+    runAsUser: 101
+    allowPrivilegeEscalation: true
 
   service:
+    enabled: true
     annotations:
       service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
       service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
       service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"
-
+
+    internal:
+      enabled: true
+      annotations:
+        service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+        service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp
+        service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "443"
+        service.beta.kubernetes.io/aws-load-balancer-internal: 0.0.0.0/0
+
     enableHttp: true
     enableHttps: true
     targetPorts:
@@ -52,3 +68,24 @@ controller:
       if ( $server_port = 80 ) {
          return 308 https://$host$request_uri;
       }
+
+  resources: 
+   limits:
+     cpu: 1000m
+     memory: 500Mi
+   requests:
+     cpu: 500m
+     memory: 250Mi
+
+  autoscaling:
+    enabled: true
+    minReplicas: 2
+    maxReplicas: 8
+    targetCPUUtilizationPercentage: 150
+    targetMemoryUtilizationPercentage: 150
+  metrics:
+     enabled: true
+
+defaultBackend:
+  enabled: true
+  replicaCount: 2

helm-charts/prometheus_values.yaml

@@ -6,7 +6,7 @@ server:
   ingress:
     enabled: true
     annotations:
-      kubernetes.io/ingress.class: private-nginx
+      kubernetes.io/ingress.class: nginx-controller
 
 alertmanager:
   enabled: false

internal/api/request_test.go

@@ -24,7 +24,7 @@ func TestNewCreateClusterRequestFromReader(t *testing.T) {
 			NodeMinCount:           2,
 			NodeMaxCount:           2,
 			Zones:                  []string{"us-east-1a"},
-			DesiredUtilityVersions: map[string]string{"fluentbit": "2.8.7", "nginx": "1.30.0", "prometheus": "10.4.0", "public-nginx": "1.30.0", "teleport": "0.2.0"},
+			DesiredUtilityVersions: map[string]string{"fluentbit": "2.8.7", "nginx": "2.11.0", "prometheus": "10.4.0", "teleport": "0.2.0"},
 		}
 	}
 
@@ -78,11 +78,10 @@ func TestNewCreateClusterRequestFromReader(t *testing.T) {
 			NodeMaxCount:       2,
 			Zones:              []string{"zone1", "zone2"},
 			DesiredUtilityVersions: map[string]string{
-				"fluentbit":    "2.8.7",
-				"nginx":        "1.30.0",
-				"prometheus":   "10.4.0",
-				"public-nginx": "1.30.0",
-				"teleport":     "0.2.0"},
+				"fluentbit":  "2.8.7",
+				"nginx":      "2.11.0",
+				"prometheus": "10.4.0",
+				"teleport":   "0.2.0"},
 		}, clusterRequest)
 	})
 }

internal/provisioner/kops_provisioner_cluster_installation.go

@@ -83,7 +83,7 @@ func (provisioner *KopsProvisioner) CreateClusterInstallation(cluster *model.Clu
 			MattermostEnv: installation.MattermostEnv.ToEnvList(),
 			UseIngressTLS: false,
 			IngressAnnotations: map[string]string{
-				"kubernetes.io/ingress.class":                          "nginx",
+				"kubernetes.io/ingress.class":                          "nginx-controller",
 				"kubernetes.io/tls-acme":                               "true",
 				"nginx.ingress.kubernetes.io/proxy-buffering":          "on",
 				"nginx.ingress.kubernetes.io/proxy-body-size":          "100m",

internal/provisioner/kops_utils.go

@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"path"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/mattermost/mattermost-cloud/internal/tools/k8s"
@@ -89,8 +90,8 @@ func waitForNamespacesDeleted(ctx context.Context, namespaces []string, k8sClien
 	}
 }
 
-// getLoadBalancerEndpoint is used to get the endpoint of the internal ingress.
-func getLoadBalancerEndpoint(ctx context.Context, namespace string, logger log.FieldLogger, configPath string) (string, error) {
+// getPrivateLoadBalancerEndpoint returns the private load balancer endpoint of the NGINX service.
+func getPrivateLoadBalancerEndpoint(ctx context.Context, namespace string, logger log.FieldLogger, configPath string) (string, error) {
 	k8sClient, err := k8s.New(configPath, logger)
 	if err != nil {
 		return "", err
@@ -101,26 +102,27 @@ func getLoadBalancerEndpoint(ctx context.Context, namespace string, logger log.F
 			return "", err
 		}
 		for _, service := range services.Items {
-			if service.Status.LoadBalancer.Ingress != nil {
-				endpoint := service.Status.LoadBalancer.Ingress[0].Hostname
-				if endpoint == "" {
-					return "", errors.New("loadbalancer endpoint value is empty")
+			if strings.HasSuffix(service.Name, "internal") {
+				if service.Status.LoadBalancer.Ingress != nil {
+					endpoint := service.Status.LoadBalancer.Ingress[0].Hostname
+					if endpoint == "" {
+						return "", errors.New("loadbalancer endpoint value is empty")
+					}
+
+					return endpoint, nil
 				}
-
-				return endpoint, nil
 			}
 		}
-
 		select {
 		case <-ctx.Done():
-			return "", errors.Wrap(ctx.Err(), "timed out waiting for helm to become ready")
+			return "", errors.Wrap(ctx.Err(), "timed out waiting for internal load balancer to become ready")
 		case <-time.After(5 * time.Second):
 		}
 	}
 }
 
-// GetNGINXLoadBalancerEndpoint returns the load balancer endpoint of the NGINX service.
-func (provisioner *KopsProvisioner) GetNGINXLoadBalancerEndpoint(cluster *model.Cluster, namespace string) (string, error) {
+// GetPublicLoadBalancerEndpoint returns the public load balancer endpoint of the NGINX service.
+func (provisioner *KopsProvisioner) GetPublicLoadBalancerEndpoint(cluster *model.Cluster, namespace string) (string, error) {
 	logger := provisioner.logger.WithFields(log.Fields{
 		"cluster":         cluster.ID,
 		"nginx-namespace": namespace,
@@ -146,13 +148,15 @@ func (provisioner *KopsProvisioner) GetNGINXLoadBalancerEndpoint(cluster *model.
 		return "", err
 	}
 	for _, service := range services.Items {
-		if service.Status.LoadBalancer.Ingress != nil {
-			endpoint := service.Status.LoadBalancer.Ingress[0].Hostname
-			if endpoint == "" {
-				return "", errors.New("loadbalancer endpoint value is empty")
-			}
+		if !strings.HasSuffix(service.Name, "internal") {
+			if service.Status.LoadBalancer.Ingress != nil {
+				endpoint := service.Status.LoadBalancer.Ingress[0].Hostname
+				if endpoint == "" {
+					return "", errors.New("loadbalancer endpoint value is empty")
+				}
 
-			return endpoint, nil
+				return endpoint, nil
+			}
 		}
 	}
 	return "", errors.New("failed to get NGINX load balancer endpoint")

internal/provisioner/nginx.go

@@ -5,23 +5,26 @@
 package provisioner
 
 import (
+	"fmt"
 	"strings"
 
+	"github.com/mattermost/mattermost-cloud/internal/tools/aws"
 	"github.com/mattermost/mattermost-cloud/internal/tools/kops"
 	"github.com/mattermost/mattermost-cloud/model"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
 type nginx struct {
+	awsClient      aws.AWS
 	provisioner    *KopsProvisioner
 	kops           *kops.Cmd
 	logger         log.FieldLogger
 	desiredVersion string
 	actualVersion  string
 }
 
-func newNginxHandle(desiredVersion string, provisioner *KopsProvisioner, kops *kops.Cmd, logger log.FieldLogger) (*nginx, error) {
+func newNginxHandle(desiredVersion string, provisioner *KopsProvisioner, awsClient aws.AWS, kops *kops.Cmd, logger log.FieldLogger) (*nginx, error) {
 	if logger == nil {
 		return nil, errors.New("cannot instantiate NGINX handle with nil logger")
 	}
@@ -30,11 +33,16 @@ func newNginxHandle(desiredVersion string, provisioner *KopsProvisioner, kops *k
 		return nil, errors.New("cannot create a connection to Nginx if the provisioner provided is nil")
 	}
 
+	if awsClient == nil {
+		return nil, errors.New("cannot create a connection to Prometheus if the awsClient provided is nil")
+	}
+
 	if kops == nil {
 		return nil, errors.New("cannot create a connection to Nginx if the Kops command provided is nil")
 	}
 
 	return &nginx{
+		awsClient:      awsClient,
 		provisioner:    provisioner,
 		kops:           kops,
 		logger:         logger.WithField("cluster-utility", model.NginxCanonicalName),
@@ -69,20 +77,32 @@ func (n *nginx) DesiredVersion() string {
 }
 
 func (n *nginx) ActualVersion() string {
-	return strings.TrimPrefix(n.actualVersion, "nginx-ingress-")
+	return strings.TrimPrefix(n.actualVersion, "ingress-nginx-")
 }
 
 func (n *nginx) Destroy() error {
 	return nil
 }
 
 func (n *nginx) NewHelmDeployment() *helmDeployment {
+	awsACMCert, err := n.awsClient.GetCertificateSummaryByTag(aws.DefaultInstallCertificatesTagKey, aws.DefaultInstallCertificatesTagValue, n.logger)
+	if err != nil {
+		n.logger.WithError(err).Error("unable to retrive the AWS ACM")
+		return nil
+	}
+
+	awsACMPrivateCert, err := n.awsClient.GetCertificateSummaryByTag(aws.DefaultInstallPrivateCertificatesTagKey, aws.DefaultInstallPrivateCertificatesTagValue, n.logger)
+	if err != nil {
+		n.logger.WithError(err).Error("unable to retrive the AWS Private ACM")
+		return nil
+	}
+
 	return &helmDeployment{
-		chartDeploymentName: "private-nginx",
-		chartName:           "stable/nginx-ingress",
-		namespace:           "internal-nginx",
-		setArgument:         "",
-		valuesPath:          "helm-charts/private-nginx_values.yaml",
+		chartDeploymentName: "nginx",
+		chartName:           "ingress-nginx/ingress-nginx",
+		namespace:           "nginx",
+		setArgument:         fmt.Sprintf("controller.service.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-cert=%s,controller.service.internal.annotations.service\\.beta\\.kubernetes\\.io/aws-load-balancer-ssl-cert=%s", *awsACMCert.CertificateArn, *awsACMPrivateCert.CertificateArn),
+		valuesPath:          "helm-charts/nginx_values.yaml",
 		kopsProvisioner:     n.provisioner,
 		kops:                n.kops,
 		logger:              n.logger,

internal/provisioner/prometheus.go

@@ -91,7 +91,7 @@ func (p *prometheus) CreateOrUpgrade() error {
 	ctx, cancel := context.WithTimeout(context.Background(), 120)
 	defer cancel()
 
-	endpoint, err := getLoadBalancerEndpoint(ctx, "internal-nginx", logger.WithField("prometheus-action", "create"), p.kops.GetKubeConfigPath())
+	endpoint, err := getPrivateLoadBalancerEndpoint(ctx, "nginx", logger.WithField("prometheus-action", "create"), p.kops.GetKubeConfigPath())
 	if err != nil {
 		return errors.Wrap(err, "couldn't get the load balancer endpoint (nginx) for Prometheus")
 	}