From 09d1baa9ff86d6948f231b6e9e9d565cfb3be383 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 16:34:33 -0500 Subject: [PATCH 1/8] Build: Don't check manifests at Docker Hub unless absolutely necessary --- .cicd/generate-base-images.sh | 75 ++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 36 deletions(-) diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index e2c1f0337d8..2041c0d744c 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -5,55 +5,58 @@ set -eo pipefail # search for base image in docker registries echo '--- :docker: Build or Pull Base Image :minidisc:' echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." -EXISTS_ALL='true' -EXISTS_DOCKER_HUB='false' -EXISTS_ECR='false' -for REGISTRY in ${CI_REGISTRIES[*]}; do - if [[ ! -z "$REGISTRY" ]]; then - MANIFEST_COMMAND="docker manifest inspect '$REGISTRY:$HASHED_IMAGE_TAG'" - echo "$ $MANIFEST_COMMAND" - set +e - eval $MANIFEST_COMMAND - MANIFEST_INSPECT_EXIT_STATUS="$?" - set -eo pipefail - if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then - if [[ "$(echo "$REGISTRY" | grep -icP '[.]amazonaws[.]com/')" != '0' ]]; then - EXISTS_ECR='true' - elif [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then - EXISTS_DOCKER_HUB='true' - fi - else - EXISTS_ALL='false' - fi +export EXISTS_DOCKER_HUB='false' +export EXISTS_ECR='false' +MANIFEST_QUERY_REGISTRY="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}" +MANIFEST_COMMAND="docker manifest inspect '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" +echo "$ $MANIFEST_COMMAND" +set +e +eval $MANIFEST_COMMAND +MANIFEST_INSPECT_EXIT_STATUS="$?" +set -eo pipefail +if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then + if [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then + export EXISTS_DOCKER_HUB='true' + else + export EXISTS_ECR='true' + fi +fi +# pull and copy as-necessary +if [[ "$EXISTS_ECR" == 'true' ]]; then + DOCKER_PULL_COMMAND="docker pull '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_PULL_COMMAND" + eval $DOCKER_PULL_COMMAND + # copy, if necessary + if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then + # tag + DOCKER_TAG_COMMAND="docker tag '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_TAG_COMMAND" + eval $DOCKER_TAG_COMMAND + # push + DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_PUSH_COMMAND" + eval $DOCKER_PUSH_COMMAND + export EXISTS_DOCKER_HUB='true' fi -done -# copy, if possible, since it is so much faster -if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_BASE_IMAGE" != 'true' && ! -z "$MIRROR_REGISTRY" ]]; then - echo 'Attempting copy from Docker Hub to the mirror instead of a new base image build.' +elif [[ "$EXISTS_DOCKER_HUB" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" - set +e eval $DOCKER_PULL_COMMAND - DOCKER_PULL_EXIT_STATUS="$?" - set -eo pipefail - if [[ "$DOCKER_PULL_EXIT_STATUS" == '0' ]]; then - echo 'Pull from Docker Hub worked! Pushing to mirror.' + # copy, if necessary + if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then # tag - DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND # push DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - EXISTS_ALL='true' - EXISTS_ECR='true' - else - echo 'Pull from Docker Hub failed, rebuilding base image from scratch.' + export EXISTS_ECR='true' fi fi # esplain yerself -if [[ "$EXISTS_ALL" == 'false' ]]; then +if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_ECR" == 'false' ]]; then echo 'Building base image from scratch.' elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries." @@ -61,7 +64,7 @@ elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries." fi # build, if neccessary -if [[ "$EXISTS_ALL" == 'false' || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first +if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_ECR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." echo "$ $DOCKER_BUILD_COMMAND" eval $DOCKER_BUILD_COMMAND From 47733e5c781cde4e6a6d6fa1954be34060554b65 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 16:50:16 -0500 Subject: [PATCH 2/8] Simplify variables --- .cicd/generate-base-images.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index 2041c0d744c..6aa219eb349 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -7,8 +7,7 @@ echo '--- :docker: Build or Pull Base Image :minidisc:' echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." export EXISTS_DOCKER_HUB='false' export EXISTS_ECR='false' -MANIFEST_QUERY_REGISTRY="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}" -MANIFEST_COMMAND="docker manifest inspect '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" +MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'" echo "$ $MANIFEST_COMMAND" set +e eval $MANIFEST_COMMAND @@ -23,13 +22,13 @@ if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then fi # pull and copy as-necessary if [[ "$EXISTS_ECR" == 'true' ]]; then - DOCKER_PULL_COMMAND="docker pull '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND # copy, if necessary if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then # tag - DOCKER_TAG_COMMAND="docker tag '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND # push @@ -45,7 +44,7 @@ elif [[ "$EXISTS_DOCKER_HUB" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then # copy, if necessary if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then # tag - DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MANIFEST_QUERY_REGISTRY:$HASHED_IMAGE_TAG'" + DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND # push From 81eba82ba8c700fb543b974ff6f0a96bb31f8c35 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 16:53:41 -0500 Subject: [PATCH 3/8] Bug fixes --- .cicd/generate-base-images.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index 6aa219eb349..747f2045340 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -21,7 +21,7 @@ if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then fi fi # pull and copy as-necessary -if [[ "$EXISTS_ECR" == 'true' ]]; then +if [[ "$EXISTS_ECR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND @@ -37,12 +37,12 @@ if [[ "$EXISTS_ECR" == 'true' ]]; then eval $DOCKER_PUSH_COMMAND export EXISTS_DOCKER_HUB='true' fi -elif [[ "$EXISTS_DOCKER_HUB" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then +elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND # copy, if necessary - if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then + if [[ "$EXISTS_ECR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then # tag DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" From a52fc1a40072b0bab324d195453ab317d32f82d2 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 16:54:24 -0500 Subject: [PATCH 4/8] Generic names --- .cicd/generate-base-images.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index 747f2045340..b6731a51f6f 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -6,7 +6,7 @@ set -eo pipefail echo '--- :docker: Build or Pull Base Image :minidisc:' echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." export EXISTS_DOCKER_HUB='false' -export EXISTS_ECR='false' +export EXISTS_MIRROR='false' MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'" echo "$ $MANIFEST_COMMAND" set +e @@ -17,11 +17,11 @@ if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then if [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then export EXISTS_DOCKER_HUB='true' else - export EXISTS_ECR='true' + export EXISTS_MIRROR='true' fi fi # pull and copy as-necessary -if [[ "$EXISTS_ECR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then +if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND @@ -42,7 +42,7 @@ elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then echo "$ $DOCKER_PULL_COMMAND" eval $DOCKER_PULL_COMMAND # copy, if necessary - if [[ "$EXISTS_ECR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then + if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then # tag DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" @@ -51,11 +51,11 @@ elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - export EXISTS_ECR='true' + export EXISTS_MIRROR='true' fi fi # esplain yerself -if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_ECR" == 'false' ]]; then +if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false' ]]; then echo 'Building base image from scratch.' elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries." @@ -63,7 +63,7 @@ elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries." fi # build, if neccessary -if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_ECR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first +if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." echo "$ $DOCKER_BUILD_COMMAND" eval $DOCKER_BUILD_COMMAND From f5af2df2a4c4c3a36588e4ab8c0775472ee65f0e Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 17:12:53 -0500 Subject: [PATCH 5/8] Avoid hitting Docker Hub in docker tag/label step, where possible --- .cicd/docker-tag.sh | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 2e922d1a9a5..ea71d20bc20 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -9,23 +9,17 @@ echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' echo ${CONTRACT_REGISTRIES[*]} +export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" # pull echo '+++ :arrow_down: Pulling Container(s)' -for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do - if [[ ! -z "$REGISTRY" ]]; then - echo "Pulling from '$REGISTRY'." - IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" - DOCKER_PULL_COMMAND="docker pull '$IMAGE'" - echo "$ $DOCKER_PULL_COMMAND" - eval $DOCKER_PULL_COMMAND - fi -done +DOCKER_PULL_COMMAND="docker pull '$IMAGE'" +echo "$ $DOCKER_PULL_COMMAND" +eval $DOCKER_PULL_COMMAND # tag echo '+++ :label: Tagging Container(s)' for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then echo "Tagging for registry $REGISTRY." - IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND From e287137eaf744c11f31143986f49aa452bed8459 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 18:13:25 -0500 Subject: [PATCH 6/8] Don't ever fail while cleaning up images --- .cicd/docker-tag.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index ea71d20bc20..87d09bf0f08 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -50,15 +50,15 @@ echo '--- :put_litter_in_its_place: Cleaning Up' for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then echo "Cleaning up from $REGISTRY." - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND fi - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND fi From a33e3ae173ad9dd7d390c651910996e305811baa Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 18:14:14 -0500 Subject: [PATCH 7/8] Clean up container by commit without platform type, should it be present --- .cicd/docker-tag.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 87d09bf0f08..85142e60b52 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -53,6 +53,9 @@ for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :" + echo "$ $DOCKER_RMI_COMMAND" + eval $DOCKER_RMI_COMMAND if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" From 63d31325e17e32521251be8f588cd2a24a6f22d3 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Fri, 12 Feb 2021 20:32:40 -0500 Subject: [PATCH 8/8] Support skipping public docker build step --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 67b549e2b22..ba9ed3e21b2 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -606,7 +606,7 @@ cat <