diff --git a/.circleci/config.yml b/.circleci/config.yml index d47399a96f..b97819061f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,9 +2,8 @@ version: 2 jobs: build: working_directory: /go/src/github.com/weaveworks/flux - docker: + machine: - image: circleci/golang:1.10 - - image: memcached steps: - checkout - setup_remote_docker @@ -21,7 +20,7 @@ jobs: - run: make check-generated - run: make test TEST_FLAGS="-race -tags integration -timeout 60s" - run: make all - + - run: make e2e - deploy: name: Maybe push prerelease images command: | @@ -34,7 +33,6 @@ jobs: docker tag "docker.io/weaveworks/helm-operator:$(docker/image-tag)" "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" docker push "docker.io/weaveworks/helm-operator-prerelease:$(docker/image-tag)" fi - - deploy: name: Maybe push release image and upload binaries command: | @@ -50,17 +48,6 @@ jobs: RELEASE_TAG=$(echo "$CIRCLE_TAG" | cut -c 6-) docker push "docker.io/weaveworks/helm-operator:${RELEASE_TAG}" fi - e2e-testing: - machine: true - working_directory: ~/go/src/github.com/weaveworks/flux - steps: - - checkout - - run: test/e2e/e2e-golang.sh - - run: test/e2e/e2e-flux-build.sh - - run: test/e2e/e2e-kind.sh - - run: test/e2e/e2e-helm.sh - - run: test/e2e/e2e-git.sh - - run: test/e2e/e2e-flux-chart.sh workflows: version: 2 @@ -70,6 +57,4 @@ workflows: filters: tags: only: /(helm-)?[0-9]+(\.[0-9]+)*(-[a-z]+)?/ - - e2e-testing: - requires: - - build + diff --git a/.gitignore b/.gitignore index 7e72e279e3..f0927ceda6 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,4 @@ docker/fluxy-dumbconf.priv test/profiles test/bin/kubectl test/bin/helm +test/bin/kind diff --git a/Makefile b/Makefile index 1df81e4bb6..9e16730860 100644 --- a/Makefile +++ b/Makefile @@ -47,14 +47,17 @@ release-bins: clean: go clean rm -rf ./build - rm -f test/bin/kubectl test/bin/helm + rm -f test/bin/kubectl test/bin/helm test/bin/kind realclean: clean rm -rf ./cache -test: test/bin/helm test/bin/kubectl +test: test/bin/helm test/bin/kubectl build/.flux.done build/.helm-operator.done PATH="${PWD}/bin:${PWD}/test/bin:${PATH}" go test ${TEST_FLAGS} $(shell go list ./... | grep -v "^github.com/weaveworks/flux/vendor" | sort -u) +e2e: test/bin/helm test/bin/kubectl + PATH="${PWD}/test/bin:${PATH}" CURRENT_OS_ARCH=$(CURRENT_OS_ARCH) test/e2e/run.sh + build/.%.done: docker/Dockerfile.% mkdir -p ./build/docker/$* cp $^ ./build/docker/$*/ diff --git a/test/e2e/e2e-flux-build.sh b/test/e2e/e2e-flux-build.sh deleted file mode 100755 index 709500e438..0000000000 --- a/test/e2e/e2e-flux-build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -echo ">>> Installing go dep" -curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh -dep ensure -vendor-only - -echo ">>> Building docker images" -make all diff --git a/test/e2e/e2e-flux-chart.sh b/test/e2e/e2e-flux-chart.sh deleted file mode 100755 index 42713e3403..0000000000 --- a/test/e2e/e2e-flux-chart.sh +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -REPO_ROOT=$(git rev-parse --show-toplevel) -KNOWN_HOSTS=$(cat ${REPO_ROOT}/test/e2e/known_hosts) -GITCONFIG=$(cat ${REPO_ROOT}/test/e2e/gitconfig) - -echo ">>> Loading $(docker/image-tag) into the cluster" -kind load docker-image "docker.io/weaveworks/flux:$(docker/image-tag)" -kind load docker-image "docker.io/weaveworks/helm-operator:$(docker/image-tag)" - -echo ">>> Installing Flux with Helm" -helm install --name flux --wait \ ---namespace flux \ ---set image.tag=$(docker/image-tag) \ ---set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ ---set git.secretName=ssh-git \ ---set git.pollInterval=30s \ ---set git.config.secretName=gitconfig \ ---set git.config.enabled=true \ ---set-string git.config.data="${GITCONFIG}" \ ---set helmOperator.tag=$(docker/image-tag) \ ---set helmOperator.create=true \ ---set helmOperator.createCRD=true \ ---set helmOperator.git.secretName=ssh-git \ ---set registry.excludeImage=* \ ---set-string ssh.known_hosts="${KNOWN_HOSTS}" \ -${REPO_ROOT}/chart/flux - -echo '>>> Waiting for gitconfig secret' -retries=12 -count=0 -ok=false -until ${ok}; do - actual=$(kubectl get secrets -n flux gitconfig -ojsonpath={..data.gitconfig} | base64 -d) - if [ "${actual}" == "${GITCONFIG}" ]; then - echo -e "Expected Git configuration deployed\n" - kubectl get secrets -n flux gitconfig && echo - ok=true - else - ok=false - sleep 10 - fi - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux get secrets - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for namespace demo' -retries=12 -count=1 -ok=false -until ${ok}; do - kubectl describe ns/demo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for workload podinfo' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/podinfo && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - echo "No more retries left" - exit 1 - fi -done - -echo '>>> Waiting for Helm release mongodb' -retries=12 -count=0 -ok=false -until ${ok}; do - kubectl -n demo describe deployment/mongodb && ok=true || ok=false - sleep 10 - count=$(($count + 1)) - if [[ ${count} -eq ${retries} ]]; then - kubectl -n flux logs deployment/flux - kubectl -n flux logs deployment/flux-helm-operator - echo "No more retries left" - exit 1 - fi -done - -echo ">>> Flux logs" -kubectl -n flux logs deployment/flux - -echo ">>> Helm Operator logs" -kubectl -n flux logs deployment/flux-helm-operator - -echo ">>> List pods" -kubectl -n demo get pods - -echo ">>> Check workload" -kubectl -n demo rollout status deployment/podinfo - -echo ">>> Check Helm release" -kubectl -n demo rollout status deployment/mongodb diff --git a/test/e2e/e2e-git.sh b/test/e2e/e2e-git.sh deleted file mode 100755 index 725c50742c..0000000000 --- a/test/e2e/e2e-git.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -SCRIPT_DIR="${REPO_ROOT}/test/e2e" -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing git" -kubectl create namespace flux -ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" -kubectl create secret generic ssh-git --namespace=flux --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" -rm "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" -kubectl apply -f "${SCRIPT_DIR}/git-dep.yaml" - -kubectl -n flux rollout status deployment/gitsrv - diff --git a/test/e2e/e2e-golang.sh b/test/e2e/e2e-golang.sh deleted file mode 100755 index dad56278a0..0000000000 --- a/test/e2e/e2e-golang.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -GO_VERSION=1.12.5 - -echo ">>> Installing go ${GO_VERSION}" -curl -O https://storage.googleapis.com/golang/go${GO_VERSION}.linux-amd64.tar.gz -tar -xf go${GO_VERSION}.linux-amd64.tar.gz -sudo rm -rf /usr/local/go -sudo mv go /usr/local - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin - -mkdir -p $HOME/go/bin -mkdir -p $HOME/go/src - -go version diff --git a/test/e2e/e2e-helm.sh b/test/e2e/e2e-helm.sh deleted file mode 100755 index 85f12a898d..0000000000 --- a/test/e2e/e2e-helm.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -REPO_ROOT=$(git rev-parse --show-toplevel) -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" - -echo ">>> Installing Helm" -curl https://raw.githubusercontent.com/kubernetes/helm/master/scripts/get | bash - -echo '>>> Installing Tiller' -kubectl --namespace kube-system create sa tiller -kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller -helm init --service-account tiller --upgrade --wait - diff --git a/test/e2e/e2e-kind.sh b/test/e2e/e2e-kind.sh deleted file mode 100755 index 41ba82362e..0000000000 --- a/test/e2e/e2e-kind.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -set -o errexit - -export GOPATH=$HOME/go -export PATH=$PATH:/usr/local/go/bin:$GOPATH/bin -REPO_ROOT=$(git rev-parse --show-toplevel) - -echo ">>> Installing kubectl" -curl -sLO https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl && \ -chmod +x kubectl && \ -sudo mv kubectl /usr/local/bin/ - -echo ">>> Building sigs.k8s.io/kind" -# Hairy way to clone and build version 0.2.1 of Kind since it doesn't support Go Modules: -mkdir -p $GOPATH/src/sigs.k8s.io -git clone https://github.com/kubernetes-sigs/kind.git $GOPATH/src/sigs.k8s.io/kind -git -C $GOPATH/src/sigs.k8s.io/kind checkout tags/0.2.1 -go install sigs.k8s.io/kind - -echo ">>> Installing kind" -sudo cp $GOPATH/bin/kind /usr/local/bin/ -kind create cluster --wait 5m - -export KUBECONFIG="$(kind get kubeconfig-path --name="kind")" -kubectl get pods --all-namespaces diff --git a/test/e2e/git-dep.yaml b/test/e2e/gitsrv.yaml similarity index 96% rename from test/e2e/git-dep.yaml rename to test/e2e/gitsrv.yaml index 0b0da2041f..812bd4721e 100644 --- a/test/e2e/git-dep.yaml +++ b/test/e2e/gitsrv.yaml @@ -5,7 +5,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: replicas: 1 selector: @@ -46,7 +45,6 @@ metadata: labels: name: gitsrv name: gitsrv - namespace: flux spec: ports: - name: ssh diff --git a/test/e2e/run.sh b/test/e2e/run.sh new file mode 100755 index 0000000000..33a11c2d7e --- /dev/null +++ b/test/e2e/run.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash + +set -o errexit + +declare -a on_exit_items + +function on_exit() { + if [ "${#on_exit_items[@]}" -gt 0 ]; then + echo -e '\nRunning deferred items, please do not interrupt until they are done:' + fi + for I in "${on_exit_items[@]}"; do + echo "deferred: ${I}" + eval "${I}" + done +} + +# Cleaning up only makes sense in a local environment +# it just wastes time in CircleCI +if [ "${CI}" != 'true' ]; then + trap on_exit EXIT +fi + +function defer() { + on_exit_items=("$*" "${on_exit_items[@]}") +} + +REPO_ROOT=$(git rev-parse --show-toplevel) +SCRIPT_DIR="${REPO_ROOT}/test/e2e" +KIND_VERSION=0.2.1 +CACHE_DIR="${REPO_ROOT}/cache/$CURRENT_OS_ARCH" +KIND_CACHE_PATH="${CACHE_DIR}/kind-$KIND_VERSION" +KIND_CLUSTER=flux-e2e +USING_KIND=false +FLUX_NAMESPACE=flux-e2e +DEMO_NAMESPACE=demo + + +# Check if there is a kubernetes cluster running, otherwise use Kind +if ! kubectl version > /dev/null 2>&1 ; then + if [ ! -f "${KIND_CACHE_PATH}" ]; then + echo '>>> Downloading Kind' + mkdir -p "${CACHE_DIR}" + curl -sL "https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-${CURRENT_OS_ARCH}" -o "${KIND_CACHE_PATH}" + fi + echo '>>> Creating Kind Kubernetes cluster' + cp "${KIND_CACHE_PATH}" "${REPO_ROOT}/test/bin/kind" + chmod +x "${REPO_ROOT}/test/bin/kind" + kind create cluster --name "${KIND_CLUSTER}" --wait 5m + defer kind --name "${KIND_CLUSTER}" delete cluster + export KUBECONFIG="$(kind --name="${KIND_CLUSTER}" get kubeconfig-path)" + USING_KIND=true + kubectl get pods --all-namespaces +fi + + +if ! helm version > /dev/null 2>&1; then + echo '>>> Installing Tiller' + kubectl --namespace kube-system create sa tiller + defer kubectl --namespace kube-system delete sa tiller + kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller + defer kubectl delete clusterrolebinding tiller-cluster-rule + helm init --service-account tiller --upgrade --wait + defer helm reset --force +fi + +kubectl create namespace "$FLUX_NAMESPACE" +defer kubectl delete namespace "$FLUX_NAMESPACE" + +echo '>>> Installing mock git server' +ssh-keygen -t rsa -N "" -f "${SCRIPT_DIR}/id_rsa" +defer rm -f "${SCRIPT_DIR}/id_rsa" "${SCRIPT_DIR}/id_rsa.pub" +kubectl create secret generic ssh-git --namespace="${FLUX_NAMESPACE}" --from-file="${SCRIPT_DIR}/known_hosts" --from-file="${SCRIPT_DIR}/id_rsa" --from-file=identity="${SCRIPT_DIR}/id_rsa" --from-file="${SCRIPT_DIR}/id_rsa.pub" +kubectl apply -n "${FLUX_NAMESPACE}" -f "${SCRIPT_DIR}/gitsrv.yaml" +kubectl -n "${FLUX_NAMESPACE}" rollout status deployment/gitsrv + + +if [ "${USING_KIND}" = 'true' ]; then + echo '>>> Loading images into the Kind cluster' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/flux:latest' + kind --name "${KIND_CLUSTER}" load docker-image 'docker.io/weaveworks/helm-operator:latest' +fi + +echo '>>> Installing Flux with Helm' + +KNOWN_HOSTS=$(cat "${REPO_ROOT}/test/e2e/known_hosts") +GITCONFIG=$(cat "${REPO_ROOT}/test/e2e/gitconfig") + +helm install --name flux --wait \ +--namespace "${FLUX_NAMESPACE}" \ +--set image.tag=latest \ +--set git.url=ssh://git@gitsrv/git-server/repos/cluster.git \ +--set git.secretName=ssh-git \ +--set git.pollInterval=30s \ +--set git.config.secretName=gitconfig \ +--set git.config.enabled=true \ +--set-string git.config.data="$GITCONFIG" \ +--set helmOperator.tag=latest \ +--set helmOperator.create=true \ +--set helmOperator.createCRD=true \ +--set helmOperator.git.secretName=ssh-git \ +--set registry.excludeImage=* \ +--set-string ssh.known_hosts="$KNOWN_HOSTS" \ +"${REPO_ROOT}/chart/flux" + +defer helm delete --purge flux +# These CRDs are have a keep policy and need to be deleted manually +defer kubectl delete crd fluxhelmreleases.helm.integrations.flux.weave.works helmreleases.flux.weave.works + +echo -n '>>> Waiting for gitconfig secret ' +retries=24 +count=0 +ok=false +until ${ok}; do + actual=$(kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig -ojsonpath={..data.gitconfig} | base64 --decode) + if [ "${actual}" = "${GITCONFIG}" ]; then + echo ' Expected Git configuration deployed' + kubectl get secrets -n "${FLUX_NAMESPACE}" gitconfig && echo + ok=true + else + echo -n '.' + ok=false + sleep 5 + fi + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + echo ' No more retries left' + kubectl -n "${FLUX_NAMESPACE}" get secrets + exit 1 + fi +done + +echo -n ">>> Waiting for namespace ${DEMO_NAMESPACE} " +retries=24 +count=1 +ok=false +until ${ok}; do + kubectl describe "ns/${DEMO_NAMESPACE}" && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for workload podinfo ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n "${DEMO_NAMESPACE}" describe deployment/podinfo && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo -n '>>> Waiting for Helm release mongodb ' +retries=24 +count=0 +ok=false +until ${ok}; do + kubectl -n $DEMO_NAMESPACE describe deployment/mongodb && ok=true || ok=false + echo -n '.' + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + echo ' No more retries left' + exit 1 + fi +done +echo ' done' + +echo '>>> Flux logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux + +echo '>>> Helm Operator logs' +kubectl -n "${FLUX_NAMESPACE}" logs deployment/flux-helm-operator + +echo '>>> List pods' +kubectl -n "${DEMO_NAMESPACE}" get pods + +echo '>>> Check workload' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/podinfo + +echo '>>> Check Helm release' +kubectl -n "${DEMO_NAMESPACE}" rollout status deployment/mongodb + +echo -e '\nEnd to end test was successful!!\n' \ No newline at end of file