Add skaffold for 1 click debugging + update k8s gitlab versions

.gitlab-ci.yml

@@ -1,8 +1,8 @@
 ---
 include:
-  - project: "hc/backend/deploy-tools"
+  - project: "cloud/backend/deploy-tools"
     file: "ci/build/image.yml"
-  - project: "hc/backend/deploy-tools"
+  - project: "cloud/backend/deploy-tools"
     file: "ci/release/image.yml"
 
 stages:
@@ -23,7 +23,7 @@ variables:
 
 test:lint:
   stage: test
-  image: golangci/golangci-lint:v1.27.0
+  image: golangci/golangci-lint:v1.49.0
   allow_failure: true
   script:
     - golangci-lint run -v
@@ -80,7 +80,7 @@ build:image:
   stage: e2e-test
   image: docker:git
   variables:
-    K8S_VERSION: k8s-1.19.10
+    K8S_VERSION: k8s-1.25.0
     CCM_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
   before_script:
     - apk add --no-cache git make musl-dev go openssh-client
@@ -89,50 +89,50 @@ build:image:
   tags:
     - hc-bladerunner-build
 
-k8s-1.20:
+k8s-1.23:
   <<: *testk8se2e
   stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.20.12
+    K8S_VERSION: k8s-1.23.8
 
-k8s-1.20-networks:
+k8s-1.23-networks:
   <<: *testk8se2e
   stage: e2e-test-k8s-networks
   variables:
-    K8S_VERSION: k8s-1.20.12
+    K8S_VERSION: k8s-1.23.8
     USE_NETWORKS: "yes"
 
-k8s-1.21:
+k8s-1.24:
   <<: *testk8se2e
   stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.21.6
+    K8S_VERSION: k8s-1.24.2
 
-k8s-1.21-networks:
+k8s-1.24-networks:
   <<: *testk8se2e
   stage: e2e-test-k8s-networks
   variables:
-    K8S_VERSION: k8s-1.21.6
+    K8S_VERSION: k8s-1.24.2
     USE_NETWORKS: "yes"
 
-k8s-1.22:
+k8s-1.25:
   <<: *testk8se2e
   stage: e2e-test-k8s
   variables:
-    K8S_VERSION: k8s-1.22.3
+    K8S_VERSION: k8s-1.25.0
 
-k8s-1.22-networks:
+k8s-1.25-networks:
   <<: *testk8se2e
   stage: e2e-test-k8s-networks
   variables:
-    K8S_VERSION: k8s-1.22.3
+    K8S_VERSION: k8s-1.25.0
     USE_NETWORKS: "yes"
 
 .e2e:k3s: &testk3se2e
   stage: e2e-test
   image: docker:git
   variables:
-    K8S_VERSION: k3s-1.20.12+k3s1
+    K8S_VERSION: k3s-1.25.0+k3s1
     CCM_IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
   before_script:
     - apk add --no-cache git make musl-dev go openssh-client
@@ -141,43 +141,43 @@ k8s-1.22-networks:
   tags:
     - hc-bladerunner-build
 
-k3s-1.20:
+k3s-1.23:
   <<: *testk3se2e
   stage: e2e-test-k3s
   variables:
-    K8S_VERSION: k3s-1.20.12+k3s1
+    K8S_VERSION: k3s-1.23.8+k3s1
 
-k3s-1.20-networks:
+k3s-1.23-networks:
   <<: *testk3se2e
   stage: e2e-test-k3s-networks
   variables:
-    K8S_VERSION: k3s-1.20.12+k3s1
+    K8S_VERSION: k3s-1.23.8+k3s1
     USE_NETWORKS: "yes"
 
-k3s-1.21:
+k3s-1.24:
   <<: *testk3se2e
   stage: e2e-test-k3s
   variables:
-    K8S_VERSION: k3s-1.21.6+k3s1
+    K8S_VERSION: k3s-1.24.2+k3s1
 
-k3s-1.21-networks:
+k3s-1.24-networks:
   <<: *testk3se2e
   stage: e2e-test-k3s-networks
   variables:
-    K8S_VERSION: k3s-1.21.6+k3s1
+    K8S_VERSION: k3s-1.24.2+k3s1
     USE_NETWORKS: "yes"
 
-k3s-1.22:
+k3s-1.25:
   <<: *testk3se2e
   stage: e2e-test-k3s
   variables:
-    K8S_VERSION: k3s-1.22.3+k3s1
+    K8S_VERSION: k3s-1.25.0+k3s1
 
-k3s-1.22-networks:
+k3s-1.25-networks:
   <<: *testk3se2e
   stage: e2e-test-k3s-networks
   variables:
-    K8S_VERSION: k3s-1.22.3+k3s1
+    K8S_VERSION: k3s-1.25.0+k3s1
     USE_NETWORKS: "yes"
 
 release:image:

Dockerfile

@@ -1,4 +1,15 @@
+FROM golang:1.18 as builder
+WORKDIR /hccm
+ADD go.mod go.sum /hccm/
+RUN go mod download
+ADD . /hccm/
+RUN ls -al
+# `skaffold debug` sets SKAFFOLD_GO_GCFLAGS to disable compiler optimizations
+ARG SKAFFOLD_GO_GCFLAGS
+RUN CGO_ENABLED=0 go build -gcflags="${SKAFFOLD_GO_GCFLAGS}" -o hcloud-cloud-controller-manager.bin github.com/hetznercloud/hcloud-cloud-controller-manager
+
+
 FROM alpine:3.12
 RUN apk add --no-cache ca-certificates bash
-COPY hcloud-cloud-controller-manager /bin/hcloud-cloud-controller-manager
+COPY --from=builder /hccm/hcloud-cloud-controller-manager.bin /bin/hcloud-cloud-controller-manager
 ENTRYPOINT ["/bin/hcloud-cloud-controller-manager"]

README.md

@@ -259,6 +259,69 @@ alias kgp="kubectl get pods"
 alias kgs="kubectl get services"
 ```
 
+## Local test setup
+This repository provides [skaffold](https://skaffold.dev/) to easily deploy / debug this controller on demand
+
+### Requirements
+1. Install [hcloud-cli](/~https://github.com/hetznercloud/cli)
+2. Install [k3sup](/~https://github.com/alexellis/k3sup)
+3. Install [cilium](/~https://github.com/cilium/cilium-cli)
+4. Install [docker](https://www.docker.com/)
+
+You will also need to set a `HCLOUD_TOKEN` in your shell session
+### Manual Installation guide
+1. Create an SSH key
+
+Assuming you already have created an ssh key via `ssh-keygen`
+```
+hcloud ssh-key create --name ssh-key-ccm-test --public-key-from-file ~/.ssh/id_rsa.pub 
+```
+
+2. Create a server
+```
+hcloud server create --name ccm-test-server --image ubuntu-20.04 --ssh-key ssh-key-ccm-test --type cx11 
+```
+
+3. Setup k3s on this server
+```
+k3sup install --ip $(hcloud server ip ccm-test-server) --local-path=/tmp/kubeconfig --cluster --k3s-channel=v1.23 --k3s-extra-args='--no-flannel --no-deploy=servicelb --no-deploy=traefik --disable-cloud-controller --disable-network-policy --kubelet-arg=cloud-provider=external'
+```
+- The kubeconfig will be created under `/tmp/kubeconfig`
+- Kubernetes version can be configured via `--k3s-channel`
+
+4. Switch your kubeconfig to the test cluster. Very important: exporting this like 
+```
+export KUBECONFIG=/tmp/kubeconfig
+```
+
+5. Install cilium + test your cluster
+```
+cilium install
+```
+
+6. Add your secret to the cluster
+```
+kubectl -n kube-system create secret generic hcloud --from-literal="token=$HCLOUD_TOKEN"
+```
+
+7. Install hcloud-cloud-controller-manager + test your cluster
+```
+kubectl apply -f  /~https://github.com/hetznercloud/hcloud-cloud-controller-manager/releases/latest/download/ccm.yaml
+kubectl config set-context default
+kubectl get node -o wide
+```
+
+8. Deploy your CSI driver
+```
+SKAFFOLD_DEFAULT_REPO=naokiii skaffold dev
+```
+- `docker login` required
+- Skaffold is using your own dockerhub repo to push the CSI image.
+
+On code change, skaffold will repack the image & deploy it to your test cluster again. Also, it is printing all logs from csi components.
+
+*After setting this up, only the command from step 8 is required!*
+
 ## License
 
 Apache License, Version 2.0

deploy/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+# Toggle between ccm and ccm-networks for skaffold testing
+- ccm.yaml
+#- ccm-networks.yaml

e2etests/setup.go

@@ -374,7 +374,7 @@ func (s *hcloudK8sSetup) deployFlannel() error {
 		return fmt.Errorf("%s: apply flannel deployment: %s", op, err)
 	}
 	fmt.Printf("%s: patch flannel deployment\n", op)
-	err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl -n kube-system patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'")
+	err = RunCommandOnServer(s.privKey, s.ClusterNode, "KUBECONFIG=/root/.kube/config kubectl -n kube-flannel patch ds kube-flannel-ds --type json -p '[{\"op\":\"add\",\"path\":\"/spec/template/spec/tolerations/-\",\"value\":{\"key\":\"node.cloudprovider.kubernetes.io/uninitialized\",\"value\":\"true\",\"effect\":\"NoSchedule\"}}]'")
 	if err != nil {
 		return fmt.Errorf("%s: patch flannel deployment: %s", op, err)
 	}

hack/kustomization.yaml

@@ -0,0 +1,17 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../deploy
+patches:
+  - target:
+      kind: Deployment
+      name: hcloud-cloud-controller-manager
+    patch: |-
+      - op: add
+        path: /spec/template/spec/containers/0/env/-
+        value:
+          name: LOG_LEVEL
+          value: info
+      - op: replace
+        path: /spec/template/spec/containers/0/env/1/valueFrom/secretKeyRef/name
+        value: hcloud

skaffold.yaml

@@ -0,0 +1,16 @@
+apiVersion: skaffold/v2beta19
+kind: Config
+metadata:
+  name: cloud-controller-manager
+build:
+  artifacts:
+    - image: hetznercloud/hcloud-cloud-controller-manager
+      docker:
+        dockerfile: Dockerfile
+        cacheFrom:
+          - hetznercloud/hcloud-cloud-controller-manager:buildcache
+  local:
+    useBuildkit: true
+deploy:
+  kustomize:
+    paths: [hack/]