Merge pull request #10047 from EOSIO/zach-docker-2.0

Reduce Docker Hub Manifest Queries
EOSIO · Feb 13, 2021 · ee4b0db · ee4b0db
2 parents 252635d + 63d3132
commit ee4b0db
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 49 deletions.
diff --git a/.cicd/docker-tag.sh b/.cicd/docker-tag.sh
@@ -9,23 +9,17 @@ echo '$ echo ${#CONTRACT_REGISTRIES[*]} # array length'
 echo ${#CONTRACT_REGISTRIES[*]}
 echo '$ echo ${CONTRACT_REGISTRIES[*]} # array'
 echo ${CONTRACT_REGISTRIES[*]}
+export IMAGE="${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
 # pull
 echo '+++ :arrow_down: Pulling Container(s)'
-for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
-    if [[ ! -z "$REGISTRY" ]]; then
-        echo "Pulling from '$REGISTRY'."
-        IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
-        DOCKER_PULL_COMMAND="docker pull '$IMAGE'"
-        echo "$ $DOCKER_PULL_COMMAND"
-        eval $DOCKER_PULL_COMMAND
-    fi
-done
+DOCKER_PULL_COMMAND="docker pull '$IMAGE'"
+echo "$ $DOCKER_PULL_COMMAND"
+eval $DOCKER_PULL_COMMAND
 # tag
 echo '+++ :label: Tagging Container(s)'
 for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
     if [[ ! -z "$REGISTRY" ]]; then
         echo "Tagging for registry $REGISTRY."
-        IMAGE="$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE"
         DOCKER_TAG_COMMAND="docker tag '$IMAGE' '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'"
         echo "$ $DOCKER_TAG_COMMAND"
         eval $DOCKER_TAG_COMMAND
@@ -56,15 +50,18 @@ echo '--- :put_litter_in_its_place: Cleaning Up'
 for REGISTRY in ${CONTRACT_REGISTRIES[*]}; do
     if [[ ! -z "$REGISTRY" ]]; then
         echo "Cleaning up from $REGISTRY."
-        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH'"
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_BRANCH' || :"
+        echo "$ $DOCKER_RMI_COMMAND"
+        eval $DOCKER_RMI_COMMAND
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT' || :"
         echo "$ $DOCKER_RMI_COMMAND"
         eval $DOCKER_RMI_COMMAND
         if [[ ! -z "$BUILDKITE_TAG" && "$SANITIZED_BRANCH" != "$SANITIZED_TAG" ]]; then
-            DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG'"
+            DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$SANITIZED_TAG' || :"
             echo "$ $DOCKER_RMI_COMMAND"
             eval $DOCKER_RMI_COMMAND
         fi
-        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE'"
+        DOCKER_RMI_COMMAND="docker rmi '$REGISTRY:$PREFIX-$BUILDKITE_COMMIT-$PLATFORM_TYPE' || :"
         echo "$ $DOCKER_RMI_COMMAND"
         eval $DOCKER_RMI_COMMAND
     fi

diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh
@@ -5,39 +5,44 @@ set -eo pipefail
 # search for base image in docker registries
 echo '--- :docker: Build or Pull Base Image :minidisc:'
 echo "Looking for '$HASHED_IMAGE_TAG' container in our registries."
-EXISTS_ALL='true'
-EXISTS_DOCKER_HUB='false'
-EXISTS_ECR='false'
-for REGISTRY in ${CI_REGISTRIES[*]}; do
-    if [[ ! -z "$REGISTRY" ]]; then
-        MANIFEST_COMMAND="docker manifest inspect '$REGISTRY:$HASHED_IMAGE_TAG'"
-        echo "$ $MANIFEST_COMMAND"
-        set +e
-        eval $MANIFEST_COMMAND
-        MANIFEST_INSPECT_EXIT_STATUS="$?"
-        set -eo pipefail
-        if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then
-            if [[ "$(echo "$REGISTRY" | grep -icP '[.]amazonaws[.]com/')" != '0' ]]; then
-                EXISTS_ECR='true'
-            elif [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then
-                EXISTS_DOCKER_HUB='true'
-            fi
-        else
-            EXISTS_ALL='false'
-        fi
+export EXISTS_DOCKER_HUB='false'
+export EXISTS_MIRROR='false'
+MANIFEST_COMMAND="docker manifest inspect '${MIRROR_REGISTRY:-$DOCKERHUB_CI_REGISTRY}:$HASHED_IMAGE_TAG'"
+echo "$ $MANIFEST_COMMAND"
+set +e
+eval $MANIFEST_COMMAND
+MANIFEST_INSPECT_EXIT_STATUS="$?"
+set -eo pipefail
+if [[ "$MANIFEST_INSPECT_EXIT_STATUS" == '0' ]]; then
+    if [[ "$(echo "$REGISTRY" | grep -icP 'docker[.]io/')" != '0' ]]; then
+        export EXISTS_DOCKER_HUB='true'
+    else
+        export EXISTS_MIRROR='true'
+    fi
+fi
+# pull and copy as-necessary
+if [[ "$EXISTS_MIRROR" == 'true' && ! -z "$MIRROR_REGISTRY" ]]; then
+    DOCKER_PULL_COMMAND="docker pull '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
+    echo "$ $DOCKER_PULL_COMMAND"
+    eval $DOCKER_PULL_COMMAND
+    # copy, if necessary
+    if [[ "$EXISTS_DOCKER_HUB" == 'false' ]]; then
+        # tag
+        DOCKER_TAG_COMMAND="docker tag '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG' '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        echo "$ $DOCKER_TAG_COMMAND"
+        eval $DOCKER_TAG_COMMAND
+        # push
+        DOCKER_PUSH_COMMAND="docker push '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
+        echo "$ $DOCKER_PUSH_COMMAND"
+        eval $DOCKER_PUSH_COMMAND
+        export EXISTS_DOCKER_HUB='true'
     fi
-done
-# copy, if possible, since it is so much faster
-if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_BASE_IMAGE" != 'true' && ! -z "$MIRROR_REGISTRY" ]]; then
-    echo 'Attempting copy from Docker Hub to the mirror instead of a new base image build.'
+elif [[ "$EXISTS_DOCKER_HUB" == 'true' ]]; then
     DOCKER_PULL_COMMAND="docker pull '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG'"
     echo "$ $DOCKER_PULL_COMMAND"
-    set +e
     eval $DOCKER_PULL_COMMAND
-    DOCKER_PULL_EXIT_STATUS="$?"
-    set -eo pipefail
-    if [[ "$DOCKER_PULL_EXIT_STATUS" == '0' ]]; then
-        echo 'Pull from Docker Hub worked! Pushing to mirror.'
+    # copy, if necessary
+    if [[ "$EXISTS_MIRROR" == 'false' && ! -z "$MIRROR_REGISTRY" ]]; then
         # tag
         DOCKER_TAG_COMMAND="docker tag '$DOCKERHUB_CI_REGISTRY:$HASHED_IMAGE_TAG' '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_TAG_COMMAND"
@@ -46,22 +51,19 @@ if [[ "$EXISTS_ECR" == 'false' && "$EXISTS_DOCKER_HUB" == 'true' && "$OVERWRITE_
         DOCKER_PUSH_COMMAND="docker push '$MIRROR_REGISTRY:$HASHED_IMAGE_TAG'"
         echo "$ $DOCKER_PUSH_COMMAND"
         eval $DOCKER_PUSH_COMMAND
-        EXISTS_ALL='true'
-        EXISTS_ECR='true'
-    else
-        echo 'Pull from Docker Hub failed, rebuilding base image from scratch.'
+        export EXISTS_MIRROR='true'
     fi
 fi
 # esplain yerself
-if [[ "$EXISTS_ALL" == 'false' ]]; then
+if [[ "$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false' ]]; then
     echo 'Building base image from scratch.'
 elif [[ "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then
     echo "OVERWRITE_BASE_IMAGE is set to 'true', building from scratch and pushing to docker registries."
 elif [[ "$FORCE_BASE_IMAGE" == 'true' ]]; then
     echo "FORCE_BASE_IMAGE is set to 'true', building from scratch and NOT pushing to docker registries."
 fi
 # build, if neccessary
-if [[ "$EXISTS_ALL" == 'false' || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first
+if [[ ("$EXISTS_DOCKER_HUB" == 'false' && "$EXISTS_MIRROR" == 'false') || "$FORCE_BASE_IMAGE" == 'true' || "$OVERWRITE_BASE_IMAGE" == 'true' ]]; then # if we cannot pull the image, we build and push it first
     export DOCKER_BUILD_COMMAND="docker build --no-cache -t 'ci:$HASHED_IMAGE_TAG' -f '$CICD_DIR/platforms/$PLATFORM_TYPE/$IMAGE_TAG.dockerfile' ."
     echo "$ $DOCKER_BUILD_COMMAND"
     eval $DOCKER_BUILD_COMMAND

diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh
@@ -606,7 +606,7 @@ cat <<EOF
     command:  "./.cicd/create-docker-from-binary.sh"
     agents:
       queue: "$BUILDKITE_BUILD_AGENT_QUEUE"
-    skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER}
+    skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER}${SKIP_PUBLIC_DOCKER}
     timeout: ${TIMEOUT:-10}
 EOF
 IFS=$oIFS