Merge pull request #2543 from 2opremio/improve-e2e-test

Run e2e test with bats

.gitignore

@@ -50,6 +50,7 @@ test/bin/kubectl
 test/bin/kustomize
 test/bin/helm
 test/bin/kind
+test/e2e/bats
 
 # Docs
 docs/_build

Makefile

@@ -5,6 +5,8 @@ SUDO := $(shell docker info > /dev/null 2> /dev/null || echo "sudo")
 
 TEST_FLAGS?=
 
+BATS_VERSION := 1.1.0
+
 include docker/kubectl.version
 include docker/kustomize.version
 include docker/helm.version
@@ -56,8 +58,8 @@ realclean: clean
 test: test/bin/helm test/bin/kubectl test/bin/kustomize $(GENERATED_TEMPLATES_FILE)
 	PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/fluxcd/flux/vendor" | sort -u)
 
-e2e: test/bin/helm test/bin/kubectl build/.flux.done
-	PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh
+e2e: test/bin/helm test/bin/kubectl test/e2e/bats build/.flux.done
+	PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.bash
 
 build/.%.done: docker/Dockerfile.%
 	mkdir -p ./build/docker/$*
@@ -106,6 +108,14 @@ cache/%/helm-$(HELM_VERSION): docker/helm.version
 	tar -m -C ./cache -xzf cache/$*/helm-$(HELM_VERSION).tar.gz $*/helm
 	mv cache/$*/helm $@
 
+test/e2e/bats: cache/bats_v$(BATS_VERSION).tar.gz
+	mkdir -p $@
+	tar -C $@ --strip-components 1 -xzf $< 
+
+cache/bats_v$(BATS_VERSION).tar.gz:
+	curl --fail -L -o $@ /~https://github.com/bats-core/bats-core/archive/v$(BATS_VERSION).tar.gz
+
+
 $(GOBIN)/fluxctl: $(FLUXCTL_DEPS) $(GENERATED_TEMPLATES_FILE)
 	go install ./cmd/fluxctl
 

test/e2e/helm_chart_smoke_test.bats

@@ -0,0 +1,104 @@
+#!/usr/bin/env bats
+
+load lib/install
+
+function setup() {
+  install_git_srv
+  install_tiller
+  install_flux_with_helm
+}
+
+
+@test "Helm chart installation smoke test" {
+
+  echo -n '>>> Waiting for gitconfig secret ' >&3
+  retries=24
+  count=0
+  ok=false
+  until ${ok}; do
+    actual=$(kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig -ojsonpath={..data.gitconfig} | base64 --decode)
+    if [ "${actual}" = "${GITCONFIG}" ]; then
+        kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig
+        ok=true
+    else
+        echo -n '.' >&3
+        ok=false
+        sleep 5
+    fi
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+        echo ' No more retries left' >&3
+        kubectl -n "${FLUX_NAMESPACE}" get secrets
+        false
+    fi
+  done
+  echo ' done' >&3
+
+  echo -n ">>> Waiting for namespace ${DEMO_NAMESPACE} " >&3
+  retries=24
+  count=1
+  ok=false
+  until ${ok}; do
+    kubectl describe "ns/${DEMO_NAMESPACE}" && ok=true || ok=false
+    echo -n '.' >&3
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+      echo ' No more retries left'
+      false
+    fi
+  done
+  echo ' done' >&3
+
+  echo -n '>>> Waiting for workload podinfo ' >&3
+  retries=24
+  count=0
+  ok=false
+  until ${ok}; do
+    kubectl -n "${DEMO_NAMESPACE}" describe deployment/podinfo && ok=true || ok=false
+    echo -n '.' >&3
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+      echo ' No more retries left' >&3
+      false
+    fi
+  done
+  echo ' done' >&3
+
+  echo -n '>>> Waiting for mongodb HelmRelease ' >&3
+  retries=24
+  count=0
+  ok=false
+  until ${ok}; do
+    kubectl -n $DEMO_NAMESPACE describe helmrelease/mongodb && ok=true || ok=false
+    echo -n '.' >&3
+    sleep 5
+    count=$(($count + 1))
+    if [[ ${count} -eq ${retries} ]]; then
+      echo ' No more retries left' >&3
+      false
+    fi
+  done
+  echo ' done' >&3
+
+  echo '>>> List pods' >&3
+  kubectl -n "${DEMO_NAMESPACE}" get pods
+
+  echo '>>> Check workload' >&3
+  kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo
+
+}
+
+function teardown() {
+  # For debugging purposes (in case the test fails)
+  echo '>>> Flux logs'
+  kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux
+  echo '>>> Helm Operator logs'
+  kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator
+
+  uninstall_flux_with_helm
+  uninstall_tiller
+  uninstall_git_srv
+  kubectl delete namespace demo
+}

test/e2e/lib/defer.bash

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+declare -a on_exit_items
+
+function on_exit() {
+  if [ "${#on_exit_items[@]}" -gt 0 ]; then
+    echo -e '\nRunning deferred items, please do not interrupt until they are done:'
+  fi
+  for I in "${on_exit_items[@]}"; do
+      echo "deferred: ${I}"
+      eval "${I}"
+  done
+}
+
+trap on_exit EXIT
+
+function defer() {
+  on_exit_items=("$*" "${on_exit_items[@]}")
+}

test/e2e/lib/install.bash

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+function install_tiller() {
+if ! helm version > /dev/null 2>&1; then # only if helm isn't already installed
+    kubectl --namespace kube-system create sa tiller
+    kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
+    helm init --service-account tiller --upgrade --wait
+fi
+}
+
+function uninstall_tiller() {
+  helm reset --force
+  kubectl delete clusterrolebinding tiller-cluster-rule
+  kubectl --namespace kube-system delete sa tiller
+}
+
+function install_flux_with_helm() {
+ local create_crds='true' 
+ if kubectl get crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works > /dev/null 2>&1; then
+   # CRDs existed, don't try to create them
+   echo 'CRDs existed, setting helmOperator.createCRD=false'
+   create_crds='false'
+ fi
+
+  helm install --name flux --wait \
+--namespace "${FLUX_NAMESPACE}" \
+--set image.repository=docker.io/fluxcd/flux \
+--set image.tag=latest \
+--set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \
+--set git.secretName=ssh-git \
+--set git.pollInterval=30s \
+--set git.config.secretName=gitconfig \
+--set git.config.enabled=true \
+--set-string git.config.data="${GITCONFIG}" \
+--set helmOperator.create=true `# just needed to add the HelmRelease CRD`\
+--set helmOperator.git.secretName=ssh-git \
+--set helmOperator.createCRD="${create_crds}" \
+--set registry.excludeImage=* \
+--set-string ssh.known_hosts="${KNOWN_HOSTS}" \
+"${FLUX_ROOT_DIR}/chart/flux"
+
+}
+
+function uninstall_flux_with_helm() {
+  helm delete --purge flux > /dev/null 2>&1
+  kubectl delete crd helmreleases.flux.weave.works > /dev/null 2>&1
+}
+
+function install_git_srv() {
+  kubectl apply -n "${FLUX_NAMESPACE}" -f "${E2E_DIR}/fixtures/gitsrv.yaml"
+  kubectl -n "${FLUX_NAMESPACE}" rollout status deployment/gitsrv
+}
+
+function uninstall_git_srv() {
+  kubectl delete -n "${FLUX_NAMESPACE}" -f "${E2E_DIR}/fixtures/gitsrv.yaml"
+}

test/e2e/run.bash

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+set -o errexit
+
+# This script runs the bats tests, first ensuring there is a kubernetes cluster available, 
+# with a flux namespace and a git secret ready to use
+
+# Global variables to be used in the libraries/tests
+export FLUX_NAMESPACE=flux-e2e
+export FLUX_ROOT_DIR=$(git rev-parse --show-toplevel)
+export E2E_DIR="${FLUX_ROOT_DIR}/test/e2e"
+export FIXTURES_DIR="${E2E_DIR}/fixtures"
+export KNOWN_HOSTS=$(cat "${FIXTURES_DIR}/known_hosts")
+export GITCONFIG=$(cat "${FIXTURES_DIR}/gitconfig")
+export DEMO_NAMESPACE=demo
+
+KIND_VERSION="v0.5.1"
+CACHE_DIR="${FLUX_ROOT_DIR}/cache/$CURRENT_OS_ARCH"
+KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION"
+KIND_CLUSTER=flux-e2e
+USING_KIND=false
+
+
+source "${E2E_DIR}/lib/defer.bash"
+
+# Check if there is a kubernetes cluster running, otherwise use Kind
+if ! kubectl version > /dev/null 2>&1 ; then
+  if [ ! -f "${KIND_CACHE_PATH}" ]; then
+    echo '>>> Downloading Kind'
+    mkdir -p "${CACHE_DIR}"
+    curl -sL "/~https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}"
+  fi
+  echo '>>> Creating Kind Kubernetes cluster'
+  cp "${KIND_CACHE_PATH}" "${FLUX_ROOT_DIR}/test/bin/kind"
+  chmod +x "${FLUX_ROOT_DIR}/test/bin/kind"
+  kind create cluster --name "${KIND_CLUSTER}" --wait 5m
+  defer kind --name "${KIND_CLUSTER}" delete cluster > /dev/null 2>&1 || true
+  export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)"
+  USING_KIND=true
+  kubectl get pods --all-namespaces
+fi
+
+kubectl create namespace "$FLUX_NAMESPACE"
+defer kubectl delete namespace "$FLUX_NAMESPACE"
+
+echo '>>> Creating ssh key and Git access secret'
+ssh-keygen -t rsa -N "" -f "${FIXTURES_DIR}/id_rsa"
+defer rm -f "${FIXTURES_DIR}/id_rsa" "${FIXTURES_DIR}/id_rsa.pub"
+kubectl create secret generic ssh-git --namespace="${FLUX_NAMESPACE}" --from-file="${FIXTURES_DIR}/known_hosts" --from-file="${FIXTURES_DIR}/id_rsa" --from-file=identity="${FIXTURES_DIR}/id_rsa" --from-file="${FIXTURES_DIR}/id_rsa.pub"
+
+if [ "${USING_KIND}" = 'true' ]; then
+    echo '>>> Loading images into the Kind cluster'
+    kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/fluxcd/flux:latest'
+fi
+
+# Run the tests
+echo '>>> Running the tests'
+(cd "${E2E_DIR}"; "${E2E_DIR}/bats/bin/bats" -t .)