Schedule based benchmark of pallet-contracts and pallet-evm #124
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Schedule based benchmark of pallet-contracts and pallet-evm | |
| on: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| env: | |
| MOONBEAM_ARTIFACT: moonbeam | |
| MOONBEAM_BIN: moonbeam | |
| MOONBEAM_VERSION: version | |
| jobs: | |
| build_dev_moonbeam: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| shell: bash | |
| container: | |
| image: paritytech/ci-unified:bullseye-1.73.0 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| sparse-checkout: | | |
| launch/moonbeam.patch | |
| sparse-checkout-cone-mode: false | |
| - name: list | |
| run: find . | |
| - name: Download Moonbeam Release | |
| run: | | |
| API_URL="https://api.github.com/repos/moonbeam-foundation/moonbeam/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Remove control characters causing error while parsing | |
| RESPONSE=$(echo $RESPONSE | tr -cd '[:print:]') | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.tarball_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| echo $RELEASE_TAG > ${{ env.MOONBEAM_VERSION }} | |
| mkdir moonbeam_release | |
| curl -L ${DOWNLOAD_URL} | tar -xzv -C moonbeam_release | |
| - name: Build | |
| run: | | |
| cd moonbeam_release/*/ | |
| patch -p1 < ../../launch/moonbeam.patch | |
| cargo build --release | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| path: | | |
| ./moonbeam_release/*/target/release/${{ env.MOONBEAM_BIN }} | |
| ./${{ env.MOONBEAM_VERSION }} | |
| retention-days: 1 | |
| smart_contract_benchmark: | |
| strategy: | |
| matrix: | |
| type: [ink-wasm, sol-wasm, evm] | |
| contract: [erc20] | |
| env: | |
| BENCHMARK_FILE: benchmark-${{ matrix.type }}.csv | |
| BENCHMARK_PATH: results | |
| needs: build_dev_moonbeam | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Cache | |
| uses: Swatinem/rust-cache@v2 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: ${{ env.MOONBEAM_ARTIFACT }} | |
| - name: Download Moonbeam Release | |
| id: moonbeam_release | |
| run: | | |
| mkdir bin | |
| cp $${{ env.MOONBEAM_ARTIFACT }}/{{ env.MOONBEAM_BIN }} bin/ && chmod +x bin/${{ env.MOONBEAM_BIN }} | |
| RELEASE_TAG=$(cat ${{ env.MOONBEAM_ARTIFACT }}/${{ env.MOONBEAM_VERSION }}) | |
| # API_URL="https://api.github.com/repos/karolk91/moonbeam/releases/latest" | |
| # RESPONSE=$(curl -s "${API_URL}") | |
| # # Get the download URL of the release binary from Repo | |
| # DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "moonbeam.gz")) | .[0].browser_download_url') | |
| # RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| # mkdir -p bin | |
| # curl -L ${DOWNLOAD_URL} | tar -xzv -C bin && chmod +x bin/moonbeam | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Download Polkadot-Parachain Release | |
| id: polkadot_parachain_release | |
| run: | | |
| API_URL="https://api.github.com/repos/paritytech/cumulus/releases/latest" | |
| RESPONSE=$(curl -s "${API_URL}") | |
| # Get the download URL of the release binary from Repo | |
| DOWNLOAD_URL=$(echo $RESPONSE | jq -r '.assets | map(select(.name == "polkadot-parachain")) | .[0].browser_download_url') | |
| RELEASE_TAG=$(echo $RESPONSE | jq -r '.tag_name') | |
| mkdir -p bin | |
| curl -L -o bin/polkadot-parachain ${DOWNLOAD_URL} && chmod +x bin/polkadot-parachain | |
| echo "tag=$(echo ${RELEASE_TAG})" >> $GITHUB_OUTPUT | |
| - name: Get Ink contract language verision | |
| id: ink_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/ink' | |
| - name: Get Solang contract language verision | |
| id: solang_version | |
| uses: ./.github/actions/get-contract-language | |
| with: | |
| contracts-directory: './contracts/solidity/wasm' | |
| - name: Get Solc contract language verision | |
| id: solc_version | |
| run: | | |
| language="" | |
| # Iterate over each contract in the directory | |
| for contract in ./contracts/solidity/evm/contracts/*/*.dbg.json; do | |
| build_info=$(jq -r '.buildInfo' "$contract") | |
| build_info_path="./contracts/solidity/evm/contracts/${build_info#../}" | |
| solc_version=$(jq -r '.solcVersion' "$build_info_path") | |
| current_language="solc $solc_version" | |
| # Check if the current language is different from the previous one | |
| if [ -n "$language" ] && [ "$current_language" != "$language" ]; then | |
| echo "Error: Different language detected in contract $contract: $language != $current_language" | |
| exit 1 | |
| fi | |
| language="$current_language" | |
| echo "Contract: $contract, Language: $current_language" | |
| done | |
| echo "language=$language" >> $GITHUB_OUTPUT | |
| - name: Display variables | |
| run: | | |
| echo moonbeam_tag: ${{ steps.moonbeam_release.outputs.tag }} | |
| echo polkadot_parachain_tag: ${{ steps.polkadot_parachain_release.outputs.tag }} | |
| - name: Execute tests | |
| id: run_smart_bench | |
| env: | |
| TEST_PARAMS: --instance-count 1 --call-count 50 | |
| run: | | |
| cd launch | |
| ./download-bins.sh | |
| # overwrite parachains | |
| cp ../bin/* ./bin | |
| ./build.sh | |
| TPS=$(./run.sh -- ${{ matrix.type }} ${{ matrix.contract }} ${TEST_PARAMS} | grep -o 'TPS: [0-9]*' | awk '{print $2}') | |
| echo "tps=$(echo ${TPS})" >> $GITHUB_OUTPUT | |
| - name: Extract Ink benchmark stats | |
| if: matrix.type == 'ink-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.ink_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - name: Extract Solang benchmark stats | |
| if: matrix.type == 'sol-wasm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.polkadot_parachain_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solang_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - name: Extract Solidity benchmark stats | |
| if: matrix.type == 'evm' | |
| run: | | |
| CURRENT_DATE=$(date +"%s") | |
| # date, parachain tag, contract, contract language, TPS | |
| echo "${CURRENT_DATE}, ${{ matrix.type }}, ${{ steps.moonbeam_release.outputs.tag }}, ${{ matrix.contract }}, \ | |
| ${{ steps.run_smart_bench.outputs.tps }}, ${{ steps.solc_version.outputs.language }}" > ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ${{ env.BENCHMARK_PATH }}/${{ env.BENCHMARK_FILE }} | |
| retention-days: 1 | |
| collect: | |
| runs-on: ubuntu-latest | |
| needs: [smart_contract_benchmark] | |
| env: | |
| BENCHMARK_FILE: benchmark-result.csv | |
| steps: | |
| - name: Download artifact | |
| uses: actions/download-artifact@v3 | |
| - name: Merge CSV | |
| run: | | |
| cat ./*/*.csv > ${{ env.BENCHMARK_FILE }} | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: ${{ env.BENCHMARK_FILE }} | |
| path: ./${{ env.BENCHMARK_FILE }} | |
| retention-days: 14 | |
| # - name: Commit benchmark stats | |
| # run: | | |
| # CURRENT_DATE=$(date +"%Y%m%d") | |
| # git config --global user.email "actions@example.com" | |
| # git config --global user.name "GitHub Action" | |
| # git add ${BENCHMARK_PATH}/${BENCHMARK_FILE} | |
| # git commit -m "Add benchmark results on ${CURRENT_DATE}" | |
| # git push | |