GitHub Actions: implement parallel testing

This allows for parallel execution of the echidna tests, which helps improve testing speed, especially once newer solc versions are included in the pipeline.
crytic · Nov 23, 2020 · 3aa2f15 · 3aa2f15
1 parent 46fcfc6
commit 3aa2f15
Show file tree

Hide file tree

Showing 2 changed files with 90 additions and 33 deletions.
diff --git a/.github/scripts/install-solc.sh b/.github/scripts/install-solc.sh
@@ -1,13 +1,9 @@
-#! /bin/bash
+#!/bin/bash
 
 # Adapted from https://github.com/commercialhaskell/stack
 
 set -eux
 
-if [ -f $HOME/.local/bin/solc-0.4.25 ] && [ -f $HOME/.local/bin/solc-0.5.7 ]; then
-  exit 0
-fi
-
 mkdir -p $HOME/.local/bin;
 
 travis_retry() {
@@ -16,15 +12,29 @@ travis_retry() {
 }
 
 fetch_solc_linux() {
-  rm -Rf solc-static-linux;
-  wget https://github.com/ethereum/solidity/releases/download/v0.4.25/solc-static-linux;
-  chmod +x solc-static-linux;
-  mv solc-static-linux $HOME/.local/bin/solc-0.4.25;
-  wget https://github.com/ethereum/solidity/releases/download/v0.5.7/solc-static-linux;
-  chmod +x solc-static-linux;
-  mv solc-static-linux $HOME/.local/bin/solc-0.5.7;
+  VER="$1"
+  if [ ! -f "$HOME/.local/bin/solc-$VER" ]; then
+    rm -Rf solc-static-linux
+    wget "https://github.com/ethereum/solidity/releases/download/v$VER/solc-static-linux"
+    chmod +x solc-static-linux
+    mv solc-static-linux "$HOME/.local/bin/solc-$VER"
+    echo "Downloaded solc $VER"
+  else
+    echo "Skipped solc $VER, already present"
+  fi
+}
+
+fetch_all_solc_linux() {
+  fetch_solc_linux "0.4.25"
+  fetch_solc_linux "0.5.7"
+  fetch_solc_linux "0.6.12"
+  fetch_solc_linux "0.7.5"
 }
 
 if [ "$HOST_OS" = "Linux" ]; then
-  travis_retry fetch_solc_linux
+  if [ "${SOLC_VER:-}" == "" ]; then
+    travis_retry fetch_all_solc_linux
+  else
+    travis_retry fetch_solc_linux "$SOLC_VER"
+  fi
 fi
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,7 +9,8 @@ on:
     - master
 
 jobs:
-  test:
+  build:
+    name: Build Echidna
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -61,13 +62,6 @@ jobs:
           path: ~/.cabal
           key: ${{ runner.os }}-cabal-v3
 
-      - name: Build Binaries
-        run: |
-          .github/scripts/install-solc.sh
-          .github/scripts/install-crytic-compile.sh
-        env:
-          HOST_OS: ${{ runner.os }}
-
       - name: Build Libraries
         run: |
           .github/scripts/install-libsecp256k1.sh
@@ -83,17 +77,6 @@ jobs:
         run: |
           stack install --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
 
-      - name: Test
-        if: runner.os == 'Linux'
-        run: |
-          export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$HOME/.local/lib"
-          export PATH="${PATH}:$HOME/.local/bin"
-
-          for VER in "0.4.25" "0.5.7" ; do
-            cp "$HOME/.local/bin/solc-$VER" "$HOME/.local/bin/solc"
-            stack test --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
-          done
-
       - name: Amend and compress binaries (macOS)
         if: runner.os == 'macOS'
         run: .github/scripts/build-macos-release.sh
@@ -103,7 +86,71 @@ jobs:
         run: GZIP=-9 tar -czf echidna-test.tar.gz -C $HOME/.local/bin/ echidna-test
 
       - name: Upload artifact
-        uses: actions/upload-artifact@v1
+        uses: actions/upload-artifact@v2
         with:
           name: echidna-test-${{ runner.os }}
           path: echidna-test.tar.gz
+
+      - name: Build and copy test suite
+        if: runner.os == 'Linux'
+        run: |
+          stack build --test --no-run-tests --ghc-options="-Werror" --extra-include-dirs=$HOME/.local/include --extra-lib-dirs=$HOME/.local/lib
+          cp "$(find "$PWD" -name echidna-testsuite -type f)" .
+
+      - name: Upload testsuite
+        if: runner.os == 'Linux'
+        uses: actions/upload-artifact@v2
+        with:
+          name: echidna-testsuite
+          path: echidna-testsuite
+
+
+  test:
+    name: Test Echidna with solc ${{ matrix.solc }}
+    runs-on: ubuntu-latest
+    needs: build
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        solc:
+          - "0.4.25"
+          - "0.5.7"
+        experimental: [false]
+        include:
+          - solc: "0.6.12"
+            experimental: true
+          - solc: "0.7.5"
+            experimental: true
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v1
+        with:
+          python-version: '3.6'
+
+      - name: Install dependencies
+        run: |
+          .github/scripts/install-solc.sh
+          .github/scripts/install-crytic-compile.sh
+        env:
+          HOST_OS: ${{ runner.os }}
+          # there are some tests hardcoded for specific solc
+          # versions, so install everything for the time being
+          # instead of a precise version
+          #SOLC_VER: ${{ matrix.solc }}
+
+      - name: Download testsuite
+        uses: actions/download-artifact@v2
+        with:
+         name: echidna-testsuite
+
+      - name: Test
+        run: |
+          export PATH="${PATH}:$HOME/.local/bin"
+          cp "$HOME/.local/bin/solc-${{ matrix.solc }}" "$HOME/.local/bin/solc"
+          chmod +x echidna-testsuite
+          ./echidna-testsuite