diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh index 2e922d1a9a5..85142e60b52 100755 --- a/.cicd/docker-tag.sh +++ b/.cicd/docker-tag.sh @@ -9,23 +9,17 @@ echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length' echo ${#CONTRACT_REGISTRIES[*]} echo '$ echo ${CONTRACT_REGISTRIES[*]} # array' echo ${CONTRACT_REGISTRIES[*]} +export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" # pull echo '+++ :arrow_down: Pulling Container(s)' -for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do - if [[ ! -z "$REGISTRY" ]]; then - echo "Pulling from '$REGISTRY'." - IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" - DOCKER_PULL_COMMAND="docker pull '$IMAGE'" - echo "$ $DOCKER_PULL_COMMAND" - eval $DOCKER_PULL_COMMAND - fi -done +DOCKER_PULL_COMMAND="docker pull '$IMAGE'" +echo "$ $DOCKER_PULL_COMMAND" +eval $DOCKER_PULL_COMMAND # tag echo '+++ :label: Tagging Container(s)' for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then echo "Tagging for registry $REGISTRY." - IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE" DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" echo "$ $DOCKER_TAG_COMMAND" eval $DOCKER_TAG_COMMAND @@ -56,15 +50,18 @@ echo '--- :put_litter_in_its_place: Cleaning Up' for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do if [[ ! -z "$REGISTRY" ]]; then echo "Cleaning up from $REGISTRY." - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH' || :" + echo "$ $DOCKER_RMI_COMMAND" + eval $DOCKER_RMI_COMMAND + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND fi - DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE'" + DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE' || :" echo "$ $DOCKER_RMI_COMMAND" eval $DOCKER_RMI_COMMAND fi diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index e2c1f0337d8..b6731a51f6f 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -5,39 +5,44 @@ set -eo pipefail # search for base image in docker registries echo '--- :docker: Build or Pull Base Image :minidisc:' echo "Looking for '$HASHED_IMAGE_TAG' container in our registries." -EXISTS_ALL='true' -EXISTS_DOCKER_HUB='false' -EXISTS_ECR='false' -for REGISTRY in ${CI_REGISTRIES[*]}; do - if [[ ! -z "$REGISTRY" ]]; then - MANIFEST_COMMAND="docker manifest inspect '$REGISTRY:$HASHED_IMAGE_TAG'" - echo "$ $MANIFEST_COMMAND" - set +e - eval $MANIFEST_COMMAND - MANIFEST_INSPECT_EXIT_STATUS="$?" - set -eo pipefail - if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then - if [[ "$(echo "$REGISTRY" | grep -icP '[.]amazonaws[.]com/')" != '0' ]]; then - EXISTS_ECR='true' - elif [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then - EXISTS_DOCKER_HUB='true' - fi - else - EXISTS_ALL='false' - fi +export EXISTS_DOCKER_HUB='false' +export EXISTS_MIRROR='false' +MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'" +echo "$ $MANIFEST_COMMAND" +set +e +eval $MANIFEST_COMMAND +MANIFEST_INSPECT_EXIT_STATUS="$?" +set -eo pipefail +if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then + if [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then + export EXISTS_DOCKER_HUB='true' + else + export EXISTS_MIRROR='true' + fi +fi +# pull and copy as-necessary +if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then + DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_PULL_COMMAND" + eval $DOCKER_PULL_COMMAND + # copy, if necessary + if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then + # tag + DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_TAG_COMMAND" + eval $DOCKER_TAG_COMMAND + # push + DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" + echo "$ $DOCKER_PUSH_COMMAND" + eval $DOCKER_PUSH_COMMAND + export EXISTS_DOCKER_HUB='true' fi -done -# copy, if possible, since it is so much faster -if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_BASE_IMAGE" != 'true' && ! -z "$MIRROR_REGISTRY" ]]; then - echo 'Attempting copy from Docker Hub to the mirror instead of a new base image build.' +elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PULL_COMMAND" - set +e eval $DOCKER_PULL_COMMAND - DOCKER_PULL_EXIT_STATUS="$?" - set -eo pipefail - if [[ "$DOCKER_PULL_EXIT_STATUS" == '0' ]]; then - echo 'Pull from Docker Hub worked! Pushing to mirror.' + # copy, if necessary + if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then # tag DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_TAG_COMMAND" @@ -46,14 +51,11 @@ if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_ DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'" echo "$ $DOCKER_PUSH_COMMAND" eval $DOCKER_PUSH_COMMAND - EXISTS_ALL='true' - EXISTS_ECR='true' - else - echo 'Pull from Docker Hub failed, rebuilding base image from scratch.' + export EXISTS_MIRROR='true' fi fi # esplain yerself -if [[ "$EXISTS_ALL" == 'false' ]]; then +if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false' ]]; then echo 'Building base image from scratch.' elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries." @@ -61,7 +63,7 @@ elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries." fi # build, if neccessary -if [[ "$EXISTS_ALL" == 'false' || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first +if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ." echo "$ $DOCKER_BUILD_COMMAND" eval $DOCKER_BUILD_COMMAND diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 67b549e2b22..ba9ed3e21b2 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -606,7 +606,7 @@ cat <