Extract from public-api all the Prometheus dependencies, and moves th…

…ings into a new viz component 'linkerd-metrics-api' (linkerd#5560)

* Protobuf changes:
- Moved `healthcheck.proto` back from viz to `proto/common` as it remains being used by the main `healthcheck.go` library (it was moved to viz by linkerd#5510).
- Extracted from `viz.proto` the IP-related types and put them in `/controller/gen/common/net` to be used by both the public and the viz APIs.

* Added chart templates for new viz linkerd-metrics-api pod

* Spin-off viz healthcheck:
- Created `viz/pkg/healthcheck/healthcheck.go` that wraps the original `pkg/healthcheck/healthcheck.go` while adding the `vizNamespace` and `vizAPIClient` fields which were removed from the core `healthcheck`. That way the core healthcheck doesn't have any dependencies on viz, and viz' healthcheck can now be used to retrieve viz api clients.
- The core and viz healthcheck libs are now abstracted out via the new `healthcheck.Runner` interface.
- Refactored the data plane checks so they don't rely on calling `ListPods`
- The checks in `viz/cmd/check.go` have been moved to `viz/pkg/healthcheck/healthcheck.go` as well, so `check.go`'s sole responsibility is dealing with command business. This command also now retrieves its viz api client through viz' healthcheck.

* Removed linkerd-controller dependency on Prometheus:
- Removed the `global.prometheusUrl` config in the core values.yml.
- Leave the Heartbeat's `-prometheus` flag hard-coded temporarily. TO-DO: have it automatically discover viz and pull Prometheus' endpoint (linkerd#5352).

* Moved observability gRPC from linkerd-controller to viz:
- Created a new gRPC server under `viz/metrics-api` moving prometheus-dependent functions out of the core gRPC server and into it (same thing for the accompaigning http server).
- Did the same for the `PublicAPIClient` (now called just `Client`) interface. The `VizAPIClient` interface disappears as it's enough to just rely on the viz `ApiClient` protobuf type.
- Moved the other files implementing the rest of the gRPC functions from `controller/api/public` to `viz/metrics-api` (`edge.go`, `stat_summary.go`, etc.).
- Also simplified some type names to avoid stuttering.

* Added linkerd-metrics-api bootstrap files. At the same time, we strip out of the public-api's `main.go` file the prometheus parameters and other no longer relevant bits.

* linkerd-web updates: it requires connecting with both the public-api and the viz api, so both addresses (and the viz namespace) are now provided as parameters to the container.

* CLI updates and other minor things:
- Changes to command files under `cli/cmd`:
  - Updated `endpoints.go` according to new API interface name.
  - Updated `version.go`, `dashboard` and `uninstall.go` to pull the viz namespace dynamically.
- Changes to command files under `viz/cmd`:
  - `edges.go`, `routes.go`, `stat.go` and `top.go`: point to dependencies that were moved from public-api to viz.
- Other changes to have tests pass:
  - Added `metrics-api` to list of docker images to build in actions workflows.
  - In `bin/fmt` exclude protobuf generated files instead of entire directories because directories could contain both generated and non-generated code (case in point: `viz/metrics-api`).

* Add retry to 'tap API service is running' check

* mc check shouldn't err when viz is not available. Also properly set the log in multicluster/cmd/root.go so that it properly displays messages when --verbose is used

Signed-off-by: Jijeesh <jijeesh.ka@gmail.com>

.github/workflows/integration_tests.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       matrix:
         # Keep in sync with release.yaml matrix build
-        target: [proxy, controller, web, cni-plugin, debug, cli-bin, grafana, jaeger-webhook]
+        target: [proxy, controller, metrics-api, web, cni-plugin, debug, cli-bin, grafana, jaeger-webhook]
     name: Docker build (${{ matrix.target }})
     timeout-minutes: 30
     steps:

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
     strategy:
       matrix:
         # Keep in sync with integration_tests.yaml matrix build
-        target: [proxy, controller, web, cni-plugin, debug, cli-bin, grafana, jaeger-webhook]
+        target: [proxy, controller, metrics-api, web, cni-plugin, debug, cli-bin, grafana, jaeger-webhook]
     name: Docker build (${{ matrix.target }})
     timeout-minutes: 30
     steps:

bin/docker-build

@@ -21,3 +21,5 @@ else
 fi
 "$bindir"/docker-build-grafana
 "$bindir"/docker-build-jaeger-webhook
+"$bindir"/docker-build-metrics-api
+

bin/docker-build-metrics-api

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -eu
+
+if [ $# -ne 0 ]; then
+    echo "no arguments allowed for ${0##*/}, given: $*" >&2
+    exit 64
+fi
+
+bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
+rootdir=$( cd "$bindir"/.. && pwd )
+
+# shellcheck source=_docker.sh
+. "$bindir"/_docker.sh
+# shellcheck source=_tag.sh
+. "$bindir"/_tag.sh
+
+dockerfile=$rootdir/viz/metrics-api/Dockerfile
+tag=$(head_root_tag)
+docker_build metrics-api "$tag" "$dockerfile"
+

bin/docker-push

@@ -14,6 +14,6 @@ bindir=$( cd "${BASH_SOURCE[0]%/*}" && pwd )
 # shellcheck source=_docker.sh
 . "$bindir"/_docker.sh
 
-for img in cli-bin cni-plugin controller debug grafana proxy web ; do
+for img in cli-bin cni-plugin controller metrics-api debug grafana proxy web ; do
     docker_push $img "$tag"
 done

bin/fmt

@@ -1,15 +1,13 @@
 #!/usr/bin/env bash
 
-set -eu
+set -u
 
-# go list will exclude the vendor directory.  We use grep to exclude generated
-# code.
-while IFS= read -r line; do dirs+=("$line"); done <<< "$(go list -f \{\{.Dir\}\} ./... | grep -v controller/gen | grep -v viz/metrics-api/gen )"
+while IFS= read -r line; do dirs+=("$line"); done <<< "$(go list -f \{\{.Dir\}\} ./...)"
 
 # go list will list all subdirectories and goimports acts recursively.  This
 # results in certain files being reported multiple time.  Therefore, we must
-# dedup them.
-files=$(bin/goimports -l "${dirs[@]}" | sort -u)
+# dedup them. We also ignore protobuf auto-generated code.
+files=$(bin/goimports -l "${dirs[@]}" | sort -u | grep -v pb.go)
 
 if [ -n "$files" ]; then
   for file in "${files[@]}"; do

bin/image-load

@@ -10,7 +10,7 @@ while :
 do
   case $1 in
     -h|--help)
-      echo "Load into KinD/k3d the images for Linkerd's proxy, controller, web, grafana, cli-bin, debug and cni-plugin."
+      echo "Load into KinD/k3d the images for Linkerd's proxy, controller, metrics-api, web, grafana, cli-bin, debug and cni-plugin."
       echo ""
       echo "Usage:"
       echo "    bin/image-load [--kind] [--k3d] [--archive]"
@@ -79,7 +79,7 @@ fi
 "$bin" version
 
 rm -f load_fail
-for img in proxy controller web grafana cli-bin debug cni-plugin jaeger-webhook; do
+for img in proxy controller web metrics-api grafana cli-bin debug cni-plugin jaeger-webhook; do
   printf 'Importing %s...\n' $img
   if [ $archive ]; then
     param="image-archives/$img.tar"

bin/protoc-go.sh

@@ -6,15 +6,17 @@ bindir=$( cd "${0%/*}" && pwd )
 
 go install -mod=readonly github.com/golang/protobuf/protoc-gen-go
 
-rm -rf controller/gen/public controller/gen/config viz/metrics-api/gen
-mkdir -p controller/gen/public viz/metrics-api/gen/viz/healthcheck
+rm -rf controller/gen/common controller/gen/public controller/gen/config viz/metrics-api/gen
+mkdir -p controller/gen/common/healthcheck controller/gen/common/net controller/gen/public viz/metrics-api/gen/viz
 
+"$bindir"/protoc -I proto --go_out=plugins=grpc,paths=source_relative:controller/gen proto/common/net.proto
 "$bindir"/protoc -I proto --go_out=plugins=grpc,paths=source_relative:controller/gen proto/public.proto
 "$bindir"/protoc -I proto --go_out=plugins=grpc,paths=source_relative:controller/gen proto/config/config.proto
-"$bindir"/protoc -I viz/metrics-api/proto --go_out=plugins=grpc,paths=source_relative:viz/metrics-api/gen viz/metrics-api/proto/healthcheck.proto
+"$bindir"/protoc -I proto --go_out=plugins=grpc,paths=source_relative:controller/gen proto/common/healthcheck.proto
 "$bindir"/protoc -I proto -I viz/metrics-api/proto --go_out=plugins=grpc,paths=source_relative:viz/metrics-api/gen viz/metrics-api/proto/viz.proto
 
+mv controller/gen/common/net.pb.go   controller/gen/common/net/
 mv controller/gen/public.pb.go controller/gen/public/
-mv viz/metrics-api/gen/healthcheck.pb.go viz/metrics-api/gen/viz/healthcheck/healthcheck.pb.go
+mv controller/gen/common/healthcheck.pb.go   controller/gen/common/healthcheck/
 mv viz/metrics-api/gen/viz.pb.go viz/metrics-api/gen/viz/viz.pb.go
 

charts/linkerd2/README.md

@@ -153,7 +153,6 @@ Kubernetes: `>=1.13.0-0`
 | global.namespace | string | `"linkerd"` | Control plane namespace |
 | global.podAnnotations | object | `{}` | Additional annotations to add to all pods |
 | global.podLabels | object | `{}` | Additional labels to add to all pods |
-| global.prometheusUrl | string | `""` | url of existing prometheus |
 | global.proxy.cores | int | `0` | The `cpu.limit` and `cores` should be kept in sync. The value of `cores` must be an integer and should typically be set by rounding up from the limit. E.g. if cpu.limit is '1500m', cores should be 2. |
 | global.proxy.enableExternalProfiles | bool | `false` | Enable service profiles for non-Kubernetes services |
 | global.proxy.image.name | string | `"ghcr.io/linkerd/proxy"` | Docker image for the proxy |

charts/linkerd2/templates/controller.yaml

@@ -98,11 +98,6 @@ spec:
         - -log-level={{.Values.global.controllerLogLevel}}
         - -log-format={{.Values.global.controllerLogFormat}}
         - -cluster-domain={{.Values.global.clusterDomain}}
-        {{- if .Values.global.prometheusUrl }}
-        - -prometheus-url={{.Values.global.prometheusUrl}}
-        {{- else }}
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.{{.Values.global.clusterDomain}}:9090
-        {{- end }}
         {{- include "partials.linkerd.trace" . | nindent 8 -}}
         image: {{.Values.controllerImage}}:{{default .Values.global.linkerdVersion .Values.global.controllerImageVersion}}
         imagePullPolicy: {{.Values.global.imagePullPolicy}}

charts/linkerd2/templates/heartbeat.yaml

@@ -47,11 +47,7 @@ spec:
             - "-controller-namespace={{.Values.global.namespace}}"
             - "-log-level={{.Values.global.controllerLogLevel}}"
             - "-log-format={{.Values.global.controllerLogFormat}}"
-            {{- if .Values.global.prometheusUrl }}
-            - "-prometheus-url={{.Values.global.prometheusUrl}}"
-            {{- else }}
             - "-prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.{{.Values.global.clusterDomain}}:9090"
-            {{- end }}
             {{- if .Values.heartbeatResources -}}
             {{- include "partials.resources" .Values.heartbeatResources | nindent 12 }}
             {{- end }}

charts/linkerd2/values.yaml

@@ -47,8 +47,6 @@ global:
   # -- Trust domain used for identity
   identityTrustDomain: *cluster_domain
 
-  # -- url of existing prometheus
-  prometheusUrl: ""
   # -- url of external grafana instance with reverse proxy configured.
   grafanaUrl: ""
 

cli/cmd/endpoints.go

@@ -113,7 +113,7 @@ destination.`,
 	return cmd
 }
 
-func requestEndpointsFromAPI(client public.PublicAPIClient, authorities []string) (endpointsInfo, error) {
+func requestEndpointsFromAPI(client public.Client, authorities []string) (endpointsInfo, error) {
 	info := make(endpointsInfo)
 	// buffered channels to avoid blocking
 	events := make(chan *destinationPb.Update, len(authorities))

cli/cmd/testdata/install_controlplane_tracing_output.golden

@@ -1227,7 +1227,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         - -trace-collector=collector.linkerd-jaeger.svc.cluster.local:55678
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent

cli/cmd/testdata/install_custom_domain.golden

@@ -1226,7 +1226,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_custom_registry.golden

@@ -1226,7 +1226,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: my.custom.registry/linkerd-io/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_default.golden

@@ -1226,7 +1226,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_default_override_dst_get_nets.golden

@@ -1226,7 +1226,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_ha_output.golden

@@ -1351,7 +1351,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_ha_with_overrides_output.golden

@@ -1351,7 +1351,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_heartbeat_disabled_output.golden

@@ -1182,7 +1182,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_helm_output.golden

@@ -1221,7 +1221,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:linkerd-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_helm_output_ha.golden

@@ -1346,7 +1346,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:linkerd-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_helm_output_ha_labels.golden

@@ -1358,7 +1358,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:linkerd-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_helm_output_ha_namespace_selector.golden

@@ -1346,7 +1346,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:linkerd-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_no_init_container.golden

@@ -1185,7 +1185,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_output.golden

@@ -1228,7 +1228,6 @@ spec:
         - -log-level=ControllerLogLevel
         - -log-format=ControllerLogFormat
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ControllerImage:ControllerImageVersion
         imagePullPolicy: ImagePullPolicy
         livenessProbe:

cli/cmd/testdata/install_proxy_ignores.golden

@@ -1226,7 +1226,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=cluster.local
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.cluster.local:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/testdata/install_values_file.golden

@@ -1211,7 +1211,6 @@ spec:
         - -log-level=info
         - -log-format=plain
         - -cluster-domain=example.com
-        - -prometheus-url=http://linkerd-prometheus.linkerd-viz.svc.example.com:9090
         image: ghcr.io/linkerd/controller:install-control-plane-version
         imagePullPolicy: IfNotPresent
         livenessProbe:

cli/cmd/version.go

@@ -12,8 +12,8 @@ import (
 	"github.com/linkerd/linkerd2/pkg/k8s"
 	api "github.com/linkerd/linkerd2/pkg/public"
 	"github.com/linkerd/linkerd2/pkg/version"
-	pb "github.com/linkerd/linkerd2/viz/metrics-api/gen/viz"
 	"github.com/spf13/cobra"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const defaultVersionString = "unavailable"
@@ -51,7 +51,7 @@ func newCmdVersion() *cobra.Command {
 				}
 			}
 
-			configureAndRunVersion(cmd.Context(), k8sAPI, options, os.Stdout, api.RawPublicAPIClient, api.RawVizAPIClient)
+			configureAndRunVersion(cmd.Context(), k8sAPI, options, os.Stdout, api.RawPublicAPIClient)
 			return nil
 		},
 	}
@@ -70,7 +70,6 @@ func configureAndRunVersion(
 	options *versionOptions,
 	stdout io.Writer,
 	mkPublicClient func(ctx context.Context, k8sAPI *k8s.KubernetesAPI, controlPlaneNamespace, apiAddr string) (publicPb.ApiClient, error),
-	mkVizClient func(ctx context.Context, k8sAPI *k8s.KubernetesAPI, controlPlaneNamespace, apiAddr string) (pb.ApiClient, error),
 ) {
 	clientVersion := version.Version
 	if options.shortVersion {
@@ -99,38 +98,23 @@ func configureAndRunVersion(
 		}
 
 		if options.proxy {
-			vizClient, clientErr := mkVizClient(ctx, k8sAPI, controlPlaneNamespace, apiAddr)
-			if clientErr != nil {
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			defer cancel()
+			selector := fmt.Sprintf("%s=%s", k8s.ControllerNSLabel, controlPlaneNamespace)
+			podList, err := k8sAPI.CoreV1().Pods(options.namespace).List(ctx, metav1.ListOptions{LabelSelector: selector})
+			if err != nil {
 				fmt.Fprintln(stdout, "Proxy versions: unavailable")
 			} else {
-				req := &pb.ListPodsRequest{}
-				if options.namespace != "" {
-					req.Selector = &pb.ResourceSelection{
-						Resource: &pb.Resource{
-							Namespace: options.namespace,
-						},
-					}
+				counts := make(map[string]int)
+				for _, pod := range podList.Items {
+					counts[k8s.GetProxyVersion(pod)]++
 				}
-
-				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-				defer cancel()
-				resp, err := vizClient.ListPods(ctx, req)
-				if err != nil {
+				if len(counts) == 0 {
 					fmt.Fprintln(stdout, "Proxy versions: unavailable")
 				} else {
-					counts := make(map[string]int)
-					for _, pod := range resp.GetPods() {
-						if pod.ControllerNamespace == controlPlaneNamespace {
-							counts[pod.GetProxyVersion()]++
-						}
-					}
-					if len(counts) == 0 {
-						fmt.Fprintln(stdout, "Proxy versions: unavailable")
-					} else {
-						fmt.Fprintln(stdout, "Proxy versions:")
-						for version, count := range counts {
-							fmt.Fprintf(stdout, "\t%s (%d pods)\n", version, count)
-						}
+					fmt.Fprintln(stdout, "Proxy versions:")
+					for version, count := range counts {
+						fmt.Fprintf(stdout, "\t%s (%d pods)\n", version, count)
 					}
 				}
 			}

cli/cmd/version_test.go

@@ -67,7 +67,7 @@ func TestConfigureAndRunVersion(t *testing.T) {
 		t.Run(fmt.Sprintf("test %d TestConfigureAndRunVersion()", i), func(t *testing.T) {
 			wout := bytes.NewBufferString("")
 
-			configureAndRunVersion(context.Background(), nil, tc.options, wout, tc.mkClient, nil)
+			configureAndRunVersion(context.Background(), nil, tc.options, wout, tc.mkClient)
 
 			if tc.out != wout.String() {
 				t.Fatalf("Expected output: \"%s\", got: \"%s\"", tc.out, wout)

controller/Dockerfile

@@ -13,7 +13,7 @@ RUN ./bin/install-deps $TARGETARCH
 FROM go-deps as golang
 WORKDIR /linkerd-build
 COPY controller/gen controller/gen
-# TODO: remove when viz API code gets moved to /viz
+# TODO: remove when tap code gets moved to /viz
 COPY viz/metrics-api/gen/viz viz/metrics-api/gen/viz
 COPY pkg pkg
 COPY controller controller