From 96dbfb1437489ab45a3b3ab104270eac09096fd1 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Tue, 16 Aug 2022 17:56:31 +0200 Subject: [PATCH 1/6] [WIP] Playground for check-runtime-migration --- .gitlab-ci.yml | 1710 ++++++++++++++++++++++++------------------------ 1 file changed, 862 insertions(+), 848 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d73a90cb3a63..21604efe7a99 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,858 +181,872 @@ default: #### stage: stage1 -check-runtime: - stage: stage1 - image: paritytech/tools:latest - <<: *kubernetes-env - rules: - - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 - variables: - GITLAB_API: "https://gitlab.parity.io/api/v4" - GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" - script: - - ./scripts/ci/gitlab/check_runtime.sh - allow_failure: true - -cargo-fmt: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo +nightly --version - - cargo +nightly fmt --all -- --check - allow_failure: true - -build-linux-stable: - stage: stage1 - <<: *docker-env - <<: *compiler-info - <<: *collect-artifacts - <<: *common-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - # Ensure we run the UI tests. - RUN_UI_TESTS: 1 - script: - - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name - - mv ./target/testnet/polkadot ./artifacts/. - - pushd artifacts - - sha256sum polkadot | tee polkadot.sha256 - - shasum -c polkadot.sha256 - - popd - - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" - - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" - - echo -n ${VERSION} > ./artifacts/VERSION - - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG - - cp -r scripts/* ./artifacts - -test-linux-stable: - stage: stage1 - <<: *docker-env - <<: *common-refs - <<: *pipeline-stopper-artifacts - before_script: - - rustup show - - cargo --version - - *pipeline-stopper-vars - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics - - - -spellcheck: - stage: stage1 - <<: *docker-env - <<: *test-refs - script: - - cargo spellcheck --version - # compare with the commit parent to the PR, given it's from a default branch - - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} - - echo "___Spellcheck is going to check your diff___" - - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 - $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) - allow_failure: true - -build-test-collators: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p test-parachain-adder-collator - - time cargo build --profile testnet --verbose -p test-parachain-undying-collator - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/adder-collator ./artifacts/. - - mv ./target/testnet/undying-collator ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-malus: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *test-refs - script: - - time cargo build --profile testnet --verbose -p polkadot-test-malus - - sccache -s - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/testnet/malus ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -build-staking-miner: - stage: stage1 - <<: *collect-artifacts - <<: *docker-env - <<: *compiler-info - <<: *common-refs - script: - - time cargo build --locked --release --package staking-miner - # pack artifacts - - mkdir -p ./artifacts - - mv ./target/release/staking-miner ./artifacts/. - - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION - - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG - - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" - - cp -r ./scripts/* ./artifacts - -#### stage: stage2 - -.check-dependent-project: &check-dependent-project - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *docker-env - <<: *test-pr-refs - script: - - git clone - --depth=1 - "--branch=$PIPELINE_SCRIPTS_TAG" - https://github.com/paritytech/pipeline-scripts - - ./pipeline-scripts/check_dependent_project.sh - --org paritytech - --dependent-repo "$DEPENDENT_REPO" - --github-api-token "$GITHUB_PR_TOKEN" - --extra-dependencies "$EXTRA_DEPENDENCIES" - --companion-overrides "$COMPANION_OVERRIDES" - -check-dependent-cumulus: - <<: *check-dependent-project - variables: - DEPENDENT_REPO: cumulus - EXTRA_DEPENDENCIES: substrate - COMPANION_OVERRIDES: | - polkadot: release-v* - cumulus: polkadot-v* - -test-node-metrics: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *docker-env - <<: *compiler-info - <<: *test-refs - variables: - RUST_TOOLCHAIN: stable - # Enable debug assertions since we are running optimized builds for testing - # but still want to have debug assertions. - RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" - script: - - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics - -test-deterministic-wasm: - stage: stage2 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-fmt - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - - ./scripts/ci/gitlab/test_deterministic_wasm.sh - -check-transaction-versions: - stage: stage2 - <<: *test-refs - <<: *docker-env - image: node:15 - needs: - - job: build-linux-stable - artifacts: true - before_script: - - apt-get -y update; apt-get -y install jq lsof - - npm install --ignore-scripts -g @polkadot/metadata-cmp - # Set git config - - git config remote.origin.url "https://github.com/paritytech/polkadot.git" - - git fetch origin release - script: - - ./scripts/ci/gitlab/check_extrinsics_ordering.sh - - - -# This image is used in testnets -# Release image is handled by the Github Action here: -# .github/workflows/publish-docker-release.yml -publish-polkadot-debug-image: - stage: stage2 - <<: *build-push-image - rules: - # Don't run when triggered from another pipeline - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" - - if: $CI_PIPELINE_SOURCE == "schedule" - - if: $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 - variables: - <<: *image-variables - # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile - DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile - IMAGE_NAME: docker.io/paritypr/polkadot-debug - needs: - - job: build-linux-stable - artifacts: true - after_script: - # pass artifacts to the zombienet-tests job - # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance - - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env - - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/parachains.env - expire_in: 1 days - -publish-test-collators-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/collator_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/colander - needs: - - job: build-test-collators - artifacts: true - after_script: - - buildah logout --all - # pass artifacts to the zombienet-tests job - - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env - - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/collator.env - -publish-malus-image: - # service image for Simnet - stage: stage2 - <<: *build-push-image - <<: *zombienet-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/malus_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile - IMAGE_NAME: docker.io/paritypr/malus - needs: - - job: build-malus - artifacts: true - after_script: - - buildah logout "$IMAGE_NAME" - # pass artifacts to the zombienet-tests job - - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env - - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env - artifacts: - reports: - # this artifact is used in zombienet-tests job - dotenv: ./artifacts/malus.env - -publish-staking-miner-image: - stage: stage2 - <<: *build-push-image - <<: *publish-refs - variables: - <<: *image-variables - # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile - IMAGE_NAME: docker.io/paritytech/staking-miner - GIT_STRATEGY: none - DOCKER_USER: ${Docker_Hub_User_Parity} - DOCKER_PASS: ${Docker_Hub_Pass_Parity} - needs: - - job: build-staking-miner - artifacts: true - - -publish-s3-release: &publish-s3 - stage: stage3 - needs: - - job: build-linux-stable - artifacts: true - <<: *kubernetes-env - image: paritytech/awscli:latest - variables: - GIT_STRATEGY: none - PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - # publishing binaries nightly - - if: $CI_PIPELINE_SOURCE == "schedule" - before_script: - - *check-versions - script: - - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" - - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ - - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" - - find ./artifacts -type f | while read file; do - name="${file#./artifacts/}"; - aws s3api copy-object - --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} - --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; - done - - | - cat <<-EOM - | - | polkadot binary paths: - | - | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot - | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot - | - EOM - after_script: - - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ - --recursive --human-readable --summarize - -update_polkadot_weights: &update-weights - stage: stage2 - timeout: 1d - when: manual - variables: - RUNTIME: polkadot - artifacts: - paths: - - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - script: - - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME - - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch - # uses the "shell" executors - tags: - - weights - -update_kusama_weights: - <<: *update-weights - variables: - RUNTIME: kusama - -update_westend_weights: - <<: *update-weights - variables: - RUNTIME: westend - -update_rococo_weights: - <<: *update-weights - variables: - RUNTIME: rococo - -#### stage: stage3 - -build-rustdoc: +# check-runtime: +# stage: stage1 +# image: paritytech/tools:latest +# <<: *kubernetes-env +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 +# variables: +# GITLAB_API: "https://gitlab.parity.io/api/v4" +# GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" +# script: +# - ./scripts/ci/gitlab/check_runtime.sh +# allow_failure: true + +# cargo-fmt: +# stage: stage1 +# <<: *docker-env +# <<: *test-refs +# script: +# - cargo +nightly --version +# - cargo +nightly fmt --all -- --check +# allow_failure: true + +# build-linux-stable: +# stage: stage1 +# <<: *docker-env +# <<: *compiler-info +# <<: *collect-artifacts +# <<: *common-refs +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# # Ensure we run the UI tests. +# RUN_UI_TESTS: 1 +# script: +# - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name +# - mv ./target/testnet/polkadot ./artifacts/. +# - pushd artifacts +# - sha256sum polkadot | tee polkadot.sha256 +# - shasum -c polkadot.sha256 +# - popd +# - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" +# - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" +# - echo -n ${VERSION} > ./artifacts/VERSION +# - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG +# - cp -r scripts/* ./artifacts + +# test-linux-stable: +# stage: stage1 +# <<: *docker-env +# <<: *common-refs +# <<: *pipeline-stopper-artifacts +# before_script: +# - rustup show +# - cargo --version +# - *pipeline-stopper-vars +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# script: +# - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics + + + +# spellcheck: +# stage: stage1 +# <<: *docker-env +# <<: *test-refs +# script: +# - cargo spellcheck --version +# # compare with the commit parent to the PR, given it's from a default branch +# - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} +# - echo "___Spellcheck is going to check your diff___" +# - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) +# - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 +# $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) +# allow_failure: true + +# build-test-collators: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# script: +# - time cargo build --profile testnet --verbose -p test-parachain-adder-collator +# - time cargo build --profile testnet --verbose -p test-parachain-undying-collator +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/testnet/adder-collator ./artifacts/. +# - mv ./target/testnet/undying-collator ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# build-malus: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# script: +# - time cargo build --profile testnet --verbose -p polkadot-test-malus +# - sccache -s +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/testnet/malus ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# build-staking-miner: +# stage: stage1 +# <<: *collect-artifacts +# <<: *docker-env +# <<: *compiler-info +# <<: *common-refs +# script: +# - time cargo build --locked --release --package staking-miner +# # pack artifacts +# - mkdir -p ./artifacts +# - mv ./target/release/staking-miner ./artifacts/. +# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION +# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG +# - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" +# - cp -r ./scripts/* ./artifacts + +# #### stage: stage2 + +# .check-dependent-project: &check-dependent-project +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: cargo-fmt +# artifacts: false +# <<: *docker-env +# <<: *test-pr-refs +# script: +# - git clone +# --depth=1 +# "--branch=$PIPELINE_SCRIPTS_TAG" +# https://github.com/paritytech/pipeline-scripts +# - ./pipeline-scripts/check_dependent_project.sh +# --org paritytech +# --dependent-repo "$DEPENDENT_REPO" +# --github-api-token "$GITHUB_PR_TOKEN" +# --extra-dependencies "$EXTRA_DEPENDENCIES" +# --companion-overrides "$COMPANION_OVERRIDES" + +# check-dependent-cumulus: +# <<: *check-dependent-project +# variables: +# DEPENDENT_REPO: cumulus +# EXTRA_DEPENDENCIES: substrate +# COMPANION_OVERRIDES: | +# polkadot: release-v* +# cumulus: polkadot-v* + +# test-node-metrics: +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: cargo-fmt +# artifacts: false +# <<: *docker-env +# <<: *compiler-info +# <<: *test-refs +# variables: +# RUST_TOOLCHAIN: stable +# # Enable debug assertions since we are running optimized builds for testing +# # but still want to have debug assertions. +# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" +# script: +# - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics + +# test-deterministic-wasm: +# stage: stage2 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: cargo-fmt +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# - ./scripts/ci/gitlab/test_deterministic_wasm.sh + +# check-transaction-versions: +# stage: stage2 +# <<: *test-refs +# <<: *docker-env +# image: node:15 +# needs: +# - job: build-linux-stable +# artifacts: true +# before_script: +# - apt-get -y update; apt-get -y install jq lsof +# - npm install --ignore-scripts -g @polkadot/metadata-cmp +# # Set git config +# - git config remote.origin.url "https://github.com/paritytech/polkadot.git" +# - git fetch origin release +# script: +# - ./scripts/ci/gitlab/check_extrinsics_ordering.sh + + + +# # This image is used in testnets +# # Release image is handled by the Github Action here: +# # .github/workflows/publish-docker-release.yml +# publish-polkadot-debug-image: +# stage: stage2 +# <<: *build-push-image +# rules: +# # Don't run when triggered from another pipeline +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# - if: $CI_PIPELINE_SOURCE == "web" +# - if: $CI_PIPELINE_SOURCE == "schedule" +# - if: $CI_COMMIT_REF_NAME == "master" +# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile +# DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile +# IMAGE_NAME: docker.io/paritypr/polkadot-debug +# needs: +# - job: build-linux-stable +# artifacts: true +# after_script: +# # pass artifacts to the zombienet-tests job +# # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance +# - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env +# - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/parachains.env +# expire_in: 1 days + +# publish-test-collators-image: +# # service image for Simnet +# stage: stage2 +# <<: *build-push-image +# <<: *zombienet-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/collator_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile +# IMAGE_NAME: docker.io/paritypr/colander +# needs: +# - job: build-test-collators +# artifacts: true +# after_script: +# - buildah logout --all +# # pass artifacts to the zombienet-tests job +# - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env +# - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/collator.env + +# publish-malus-image: +# # service image for Simnet +# stage: stage2 +# <<: *build-push-image +# <<: *zombienet-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/malus_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile +# IMAGE_NAME: docker.io/paritypr/malus +# needs: +# - job: build-malus +# artifacts: true +# after_script: +# - buildah logout "$IMAGE_NAME" +# # pass artifacts to the zombienet-tests job +# - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env +# - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env +# artifacts: +# reports: +# # this artifact is used in zombienet-tests job +# dotenv: ./artifacts/malus.env + +# publish-staking-miner-image: +# stage: stage2 +# <<: *build-push-image +# <<: *publish-refs +# variables: +# <<: *image-variables +# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile +# IMAGE_NAME: docker.io/paritytech/staking-miner +# GIT_STRATEGY: none +# DOCKER_USER: ${Docker_Hub_User_Parity} +# DOCKER_PASS: ${Docker_Hub_Pass_Parity} +# needs: +# - job: build-staking-miner +# artifacts: true + + +# publish-s3-release: &publish-s3 +# stage: stage3 +# needs: +# - job: build-linux-stable +# artifacts: true +# <<: *kubernetes-env +# image: paritytech/awscli:latest +# variables: +# GIT_STRATEGY: none +# PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" +# rules: +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# # publishing binaries nightly +# - if: $CI_PIPELINE_SOURCE == "schedule" +# before_script: +# - *check-versions +# script: +# - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" +# - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ +# - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" +# - find ./artifacts -type f | while read file; do +# name="${file#./artifacts/}"; +# aws s3api copy-object +# --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} +# --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; +# done +# - | +# cat <<-EOM +# | +# | polkadot binary paths: +# | +# | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot +# | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot +# | +# EOM +# after_script: +# - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ +# --recursive --human-readable --summarize + +# update_polkadot_weights: &update-weights +# stage: stage2 +# timeout: 1d +# when: manual +# variables: +# RUNTIME: polkadot +# artifacts: +# paths: +# - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch +# script: +# - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME +# - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch +# # uses the "shell" executors +# tags: +# - weights + +# update_kusama_weights: +# <<: *update-weights +# variables: +# RUNTIME: kusama + +# update_westend_weights: +# <<: *update-weights +# variables: +# RUNTIME: westend + +# update_rococo_weights: +# <<: *update-weights +# variables: +# RUNTIME: rococo + +# #### stage: stage3 + +# build-rustdoc: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *docker-env +# <<: *test-refs +# variables: +# SKIP_WASM_BUILD: 1 +# artifacts: +# name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" +# when: on_success +# expire_in: 1 days +# paths: +# - ./crate-docs/ +# script: +# # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` +# # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable +# - time cargo +nightly doc --workspace --verbose --no-deps +# - rm -f ./target/doc/.lock +# - mv ./target/doc ./crate-docs +# # FIXME: remove me after CI image gets nonroot +# - chown -R nonroot:nonroot ./crate-docs +# - echo "" > ./crate-docs/index.html + +# build-implementers-guide: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *collect-artifacts-short +# script: +# - cargo install mdbook mdbook-mermaid mdbook-linkcheck +# - mdbook build ./roadmap/implementers-guide +# - mkdir -p artifacts +# - mv roadmap/implementers-guide/book artifacts/ +# # FIXME: remove me after CI image gets nonroot +# - chown -R nonroot:nonroot artifacts/ + +# check-try-runtime: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-node-metrics +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# # Check that everything compiles with `try-runtime` feature flag. +# - cargo check --features try-runtime --all + +# More info can be found here: https://github.com/paritytech/polkadot/pull/5865 +check-runtime-migration: stage: stage3 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *docker-env - <<: *test-refs - variables: - SKIP_WASM_BUILD: 1 - artifacts: - name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" - when: on_success - expire_in: 1 days - paths: - - ./crate-docs/ - script: - # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` - # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable - - time cargo +nightly doc --workspace --verbose --no-deps - - rm -f ./target/doc/.lock - - mv ./target/doc ./crate-docs - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot ./crate-docs - - echo "" > ./crate-docs/index.html - -build-implementers-guide: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *collect-artifacts-short - script: - - cargo install mdbook mdbook-mermaid mdbook-linkcheck - - mdbook build ./roadmap/implementers-guide - - mkdir -p artifacts - - mv roadmap/implementers-guide/book artifacts/ - # FIXME: remove me after CI image gets nonroot - - chown -R nonroot:nonroot artifacts/ - -check-try-runtime: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - <<: *test-refs - <<: *docker-env - <<: *compiler-info - script: - # Check that everything compiles with `try-runtime` feature flag. - - cargo check --features try-runtime --all - -check-no-default-features: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false + # needs: + # - job: test-node-metrics + # artifacts: false <<: *test-refs <<: *docker-env <<: *compiler-info script: - # Check that polkadot-cli will compile no default features. - - pushd ./node/service && cargo check --no-default-features && popd - - pushd ./cli && cargo check --no-default-features --features "service" && popd - - sccache -s - -build-short-benchmark: - stage: stage3 - <<: *test-refs - <<: *docker-env - <<: *collect-artifacts - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-node-metrics - artifacts: false - script: - - cargo +nightly build --profile release --locked --features=runtime-benchmarks - - mkdir artifacts - - cp ./target/release/polkadot ./artifacts/ - -deploy-parity-testnet: - stage: stage3 - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: test-deterministic-wasm - artifacts: false - <<: *deploy-testnet-refs - variables: - POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" - POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" - allow_failure: false - trigger: "parity/infrastructure/parity-testnet" - -zombienet-tests-parachains-smoke-test: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-smoke-test.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-pvf: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-parachains-pvf.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-disputes.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-parachains-disputes-garbage-candidate: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - - job: publish-malus-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-parachains-garbage-candidate.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-test-parachains-upgrade-smoke-test: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' - before_script: - - echo "ZombieNet Tests Config" - - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" - - echo "docker.io/parity/polkadot-collator:latest" - - echo "${ZOMBIENET_IMAGE}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0002-parachains-upgrade-smoke-test.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-misc-paritydb: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-test-collators-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie,zombie::network-node - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-paritydb.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-malus-dispute-valid: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - - job: publish-malus-image - - job: publish-test-collators-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0001-dispute-valid-block.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-deregister-register-validator: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - artifacts: true - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-deregister-register-validator-smoke.feature" - allow_failure: false - retry: 2 - tags: - - zombienet-polkadot-integration-test - -zombienet-tests-beefy-and-mmr: - stage: stage3 - image: "${ZOMBIENET_IMAGE}" - <<: *kubernetes-env - <<: *zombienet-refs - needs: - - job: publish-polkadot-debug-image - variables: - GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" - before_script: - - echo "Zombie-net Tests Config" - - echo "${ZOMBIENET_IMAGE_NAME}" - - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" - - echo "${GH_DIR}" - - export DEBUG=zombie* - - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} - script: - - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh - --github-remote-dir="${GH_DIR}" - --test="0003-beefy-and-mmr.feature" - allow_failure: true - retry: 2 - tags: - - zombienet-polkadot-integration-test - -#### stage: stage4 - -publish-rustdoc: - stage: stage4 - <<: *kubernetes-env - image: paritytech/tools:latest - variables: - GIT_DEPTH: 100 - rules: - - if: $CI_PIPELINE_SOURCE == "pipeline" - when: never - - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" - - if: $CI_COMMIT_REF_NAME == "master" - # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other - # artifacts from being dowloaded by this job. - needs: - - job: build-rustdoc - artifacts: true - - job: build-implementers-guide - artifacts: true - script: - # Save README and docs - - cp -r ./crate-docs/ /tmp/doc/ - - cp -r ./artifacts/book/ /tmp/ - # setup ssh - - eval $(ssh-agent) - - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} - - mkdir ~/.ssh && touch ~/.ssh/known_hosts - - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts - # Set git config - - git config user.email "devops-team@parity.io" - - git config user.name "${GITHUB_USER}" - - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" - - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" - - git fetch origin gh-pages - - git checkout gh-pages - # Remove everything and restore generated docs and README - - cp index.html /tmp - - cp README.md /tmp - - rm -rf ./* - # dir for rustdoc - - mkdir -p doc - # dir for implementors guide - - mkdir -p book - - mv /tmp/doc/* doc/ - - mv /tmp/book/html/* book/ - - mv /tmp/index.html . - - mv /tmp/README.md . - # Upload files - - git add --all --force - # `git commit` has an exit code of > 0 if there is nothing to commit. - # This causes GitLab to exit immediately and marks this job failed. - # We don't want to mark the entire job failed if there's nothing to - # publish though, hence the `|| true`. - - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || - echo "___Nothing to commit___" - - git push origin gh-pages --force - - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" - after_script: - - rm -rf .git/ ./* - -# Run all pallet benchmarks only once to check if there are any errors -short-benchmark-polkadot: &short-bench - stage: stage4 - <<: *test-pr-refs - <<: *docker-env - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: build-short-benchmark - artifacts: true - variables: - RUNTIME: polkadot - script: - - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - -short-benchmark-kusama: - <<: *short-bench - variables: - RUNTIME: kusama - -short-benchmark-westend: - <<: *short-bench - variables: - RUNTIME: westend - -#### stage: .post - -# This job cancels the whole pipeline if any of provided jobs fail. -# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests -# to fail the pipeline as soon as possible to shorten the feedback loop. -.cancel-pipeline-template: - stage: .post - rules: - - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - when: on_failure - variables: - PROJECT_ID: "${CI_PROJECT_ID}" - PROJECT_NAME: "${CI_PROJECT_NAME}" - PIPELINE_ID: "${CI_PIPELINE_ID}" - FAILED_JOB_URL: "${FAILED_JOB_URL}" - FAILED_JOB_NAME: "${FAILED_JOB_NAME}" - PR_NUM: "${PR_NUM}" - trigger: - project: "parity/infrastructure/ci_cd/pipeline-stopper" - # remove branch, when pipeline-stopper for polakdot is updated to the same branch - branch: "as-improve" - -cancel-pipeline-test-linux-stable: - extends: .cancel-pipeline-template - needs: - - job: test-linux-stable + - export RUST_LOG=remote-ext=debug,runtime=debug + - time cargo test -- --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime + +# check-no-default-features: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *test-refs +# <<: *docker-env +# <<: *compiler-info +# script: +# # Check that polkadot-cli will compile no default features. +# - pushd ./node/service && cargo check --no-default-features && popd +# - pushd ./cli && cargo check --no-default-features --features "service" && popd +# - sccache -s + +# build-short-benchmark: +# stage: stage3 +# <<: *test-refs +# <<: *docker-env +# <<: *collect-artifacts +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-node-metrics +# artifacts: false +# script: +# - cargo +nightly build --profile release --locked --features=runtime-benchmarks +# - mkdir artifacts +# - cp ./target/release/polkadot ./artifacts/ + +# deploy-parity-testnet: +# stage: stage3 +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: test-deterministic-wasm +# artifacts: false +# <<: *deploy-testnet-refs +# variables: +# POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" +# POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" +# allow_failure: false +# trigger: "parity/infrastructure/parity-testnet" + +# zombienet-tests-parachains-smoke-test: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-parachains-smoke-test.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-parachains-pvf: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-parachains-pvf.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-parachains-disputes: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# - job: publish-malus-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0002-parachains-disputes.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-parachains-disputes-garbage-candidate: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# - job: publish-malus-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0003-parachains-garbage-candidate.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-test-parachains-upgrade-smoke-test: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' +# before_script: +# - echo "ZombieNet Tests Config" +# - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" +# - echo "docker.io/parity/polkadot-collator:latest" +# - echo "${ZOMBIENET_IMAGE}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0002-parachains-upgrade-smoke-test.feature" +# allow_failure: true +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-misc-paritydb: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-test-collators-image +# artifacts: true +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie,zombie::network-node +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-paritydb.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-malus-dispute-valid: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# - job: publish-malus-image +# - job: publish-test-collators-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie* +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0001-dispute-valid-block.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-deregister-register-validator: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# artifacts: true +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie* +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0003-deregister-register-validator-smoke.feature" +# allow_failure: false +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# zombienet-tests-beefy-and-mmr: +# stage: stage3 +# image: "${ZOMBIENET_IMAGE}" +# <<: *kubernetes-env +# <<: *zombienet-refs +# needs: +# - job: publish-polkadot-debug-image +# variables: +# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" +# before_script: +# - echo "Zombie-net Tests Config" +# - echo "${ZOMBIENET_IMAGE_NAME}" +# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" +# - echo "${GH_DIR}" +# - export DEBUG=zombie* +# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} +# script: +# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh +# --github-remote-dir="${GH_DIR}" +# --test="0003-beefy-and-mmr.feature" +# allow_failure: true +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +# #### stage: stage4 + +# publish-rustdoc: +# stage: stage4 +# <<: *kubernetes-env +# image: paritytech/tools:latest +# variables: +# GIT_DEPTH: 100 +# rules: +# - if: $CI_PIPELINE_SOURCE == "pipeline" +# when: never +# - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" +# - if: $CI_COMMIT_REF_NAME == "master" +# # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other +# # artifacts from being dowloaded by this job. +# needs: +# - job: build-rustdoc +# artifacts: true +# - job: build-implementers-guide +# artifacts: true +# script: +# # Save README and docs +# - cp -r ./crate-docs/ /tmp/doc/ +# - cp -r ./artifacts/book/ /tmp/ +# # setup ssh +# - eval $(ssh-agent) +# - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} +# - mkdir ~/.ssh && touch ~/.ssh/known_hosts +# - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts +# # Set git config +# - git config user.email "devops-team@parity.io" +# - git config user.name "${GITHUB_USER}" +# - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" +# - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" +# - git fetch origin gh-pages +# - git checkout gh-pages +# # Remove everything and restore generated docs and README +# - cp index.html /tmp +# - cp README.md /tmp +# - rm -rf ./* +# # dir for rustdoc +# - mkdir -p doc +# # dir for implementors guide +# - mkdir -p book +# - mv /tmp/doc/* doc/ +# - mv /tmp/book/html/* book/ +# - mv /tmp/index.html . +# - mv /tmp/README.md . +# # Upload files +# - git add --all --force +# # `git commit` has an exit code of > 0 if there is nothing to commit. +# # This causes GitLab to exit immediately and marks this job failed. +# # We don't want to mark the entire job failed if there's nothing to +# # publish though, hence the `|| true`. +# - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || +# echo "___Nothing to commit___" +# - git push origin gh-pages --force +# - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" +# after_script: +# - rm -rf .git/ ./* + +# # Run all pallet benchmarks only once to check if there are any errors +# short-benchmark-polkadot: &short-bench +# stage: stage4 +# <<: *test-pr-refs +# <<: *docker-env +# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs +# needs: +# - job: build-short-benchmark +# artifacts: true +# variables: +# RUNTIME: polkadot +# script: +# - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + +# short-benchmark-kusama: +# <<: *short-bench +# variables: +# RUNTIME: kusama + +# short-benchmark-westend: +# <<: *short-bench +# variables: +# RUNTIME: westend + +# #### stage: .post + +# # This job cancels the whole pipeline if any of provided jobs fail. +# # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# # to fail the pipeline as soon as possible to shorten the feedback loop. +# .cancel-pipeline-template: +# stage: .post +# rules: +# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs +# when: on_failure +# variables: +# PROJECT_ID: "${CI_PROJECT_ID}" +# PROJECT_NAME: "${CI_PROJECT_NAME}" +# PIPELINE_ID: "${CI_PIPELINE_ID}" +# FAILED_JOB_URL: "${FAILED_JOB_URL}" +# FAILED_JOB_NAME: "${FAILED_JOB_NAME}" +# PR_NUM: "${PR_NUM}" +# trigger: +# project: "parity/infrastructure/ci_cd/pipeline-stopper" +# # remove branch, when pipeline-stopper for polakdot is updated to the same branch +# branch: "as-improve" + +# cancel-pipeline-test-linux-stable: +# extends: .cancel-pipeline-template +# needs: +# - job: test-linux-stable From 8240455ee3f47388495c7719d20b126b5f0d6ec9 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Tue, 16 Aug 2022 18:16:16 +0200 Subject: [PATCH 2/6] add condition --- .gitlab-ci.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 21604efe7a99..1b55988f115a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -625,18 +625,28 @@ default: # - cargo check --features try-runtime --all # More info can be found here: https://github.com/paritytech/polkadot/pull/5865 +# Works only in PRs check-runtime-migration: stage: stage3 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs # needs: # - job: test-node-metrics # artifacts: false - <<: *test-refs + <<: *test-pr-refs <<: *docker-env <<: *compiler-info script: - - export RUST_LOG=remote-ext=debug,runtime=debug - - time cargo test -- --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime + - export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" + -H "Authorization: token $GITHUB_PR_TOKEN" + https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "runtimemigration" | wc -l) + - if [[ $has_runtimemigration_label != 0 ]]; then + echo "Found label runtimemigration. Running tests" + export RUST_LOG=remote-ext=debug,runtime=debug; + time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime; + else + echo "runtimemigration label not found. Skipping" + fi + # check-no-default-features: # stage: stage3 From 83395a389821897eb7c2acba05a28f89018d143d Mon Sep 17 00:00:00 2001 From: alvicsam Date: Tue, 16 Aug 2022 18:19:35 +0200 Subject: [PATCH 3/6] fix syntax --- .gitlab-ci.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1b55988f115a..fc9708180bcb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -636,13 +636,15 @@ check-runtime-migration: <<: *docker-env <<: *compiler-info script: - - export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" - -H "Authorization: token $GITHUB_PR_TOKEN" + - | + export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" \ + -H "Authorization: token $GITHUB_PR_TOKEN" \ https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "runtimemigration" | wc -l) - - if [[ $has_runtimemigration_label != 0 ]]; then + - | + if [[ $has_runtimemigration_label != 0 ]]; then echo "Found label runtimemigration. Running tests" - export RUST_LOG=remote-ext=debug,runtime=debug; - time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime; + export RUST_LOG=remote-ext=debug,runtime=debug + time cargo test --release -p westend-runtime -p polkadot-runtime -p kusama-runtime --features try-runtime else echo "runtimemigration label not found. Skipping" fi From 4d5a08b92f80d960cb8ef00d8d39780a9ba30428 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 17 Aug 2022 11:04:47 +0200 Subject: [PATCH 4/6] enable pipeline --- .gitlab-ci.yml | 1706 ++++++++++++++++++++++++------------------------ 1 file changed, 853 insertions(+), 853 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fc9708180bcb..d7eb92acde9a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,448 +181,448 @@ default: #### stage: stage1 -# check-runtime: -# stage: stage1 -# image: paritytech/tools:latest -# <<: *kubernetes-env -# rules: -# - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 -# variables: -# GITLAB_API: "https://gitlab.parity.io/api/v4" -# GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" -# script: -# - ./scripts/ci/gitlab/check_runtime.sh -# allow_failure: true - -# cargo-fmt: -# stage: stage1 -# <<: *docker-env -# <<: *test-refs -# script: -# - cargo +nightly --version -# - cargo +nightly fmt --all -- --check -# allow_failure: true - -# build-linux-stable: -# stage: stage1 -# <<: *docker-env -# <<: *compiler-info -# <<: *collect-artifacts -# <<: *common-refs -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# # Ensure we run the UI tests. -# RUN_UI_TESTS: 1 -# script: -# - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name -# - mv ./target/testnet/polkadot ./artifacts/. -# - pushd artifacts -# - sha256sum polkadot | tee polkadot.sha256 -# - shasum -c polkadot.sha256 -# - popd -# - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" -# - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" -# - echo -n ${VERSION} > ./artifacts/VERSION -# - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG -# - cp -r scripts/* ./artifacts - -# test-linux-stable: -# stage: stage1 -# <<: *docker-env -# <<: *common-refs -# <<: *pipeline-stopper-artifacts -# before_script: -# - rustup show -# - cargo --version -# - *pipeline-stopper-vars -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# script: -# - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics - - - -# spellcheck: -# stage: stage1 -# <<: *docker-env -# <<: *test-refs -# script: -# - cargo spellcheck --version -# # compare with the commit parent to the PR, given it's from a default branch -# - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} -# - echo "___Spellcheck is going to check your diff___" -# - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) -# - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 -# $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) -# allow_failure: true - -# build-test-collators: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# script: -# - time cargo build --profile testnet --verbose -p test-parachain-adder-collator -# - time cargo build --profile testnet --verbose -p test-parachain-undying-collator -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/testnet/adder-collator ./artifacts/. -# - mv ./target/testnet/undying-collator ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# build-malus: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# script: -# - time cargo build --profile testnet --verbose -p polkadot-test-malus -# - sccache -s -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/testnet/malus ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# build-staking-miner: -# stage: stage1 -# <<: *collect-artifacts -# <<: *docker-env -# <<: *compiler-info -# <<: *common-refs -# script: -# - time cargo build --locked --release --package staking-miner -# # pack artifacts -# - mkdir -p ./artifacts -# - mv ./target/release/staking-miner ./artifacts/. -# - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION -# - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG -# - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" -# - cp -r ./scripts/* ./artifacts - -# #### stage: stage2 - -# .check-dependent-project: &check-dependent-project -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: cargo-fmt -# artifacts: false -# <<: *docker-env -# <<: *test-pr-refs -# script: -# - git clone -# --depth=1 -# "--branch=$PIPELINE_SCRIPTS_TAG" -# https://github.com/paritytech/pipeline-scripts -# - ./pipeline-scripts/check_dependent_project.sh -# --org paritytech -# --dependent-repo "$DEPENDENT_REPO" -# --github-api-token "$GITHUB_PR_TOKEN" -# --extra-dependencies "$EXTRA_DEPENDENCIES" -# --companion-overrides "$COMPANION_OVERRIDES" - -# check-dependent-cumulus: -# <<: *check-dependent-project -# variables: -# DEPENDENT_REPO: cumulus -# EXTRA_DEPENDENCIES: substrate -# COMPANION_OVERRIDES: | -# polkadot: release-v* -# cumulus: polkadot-v* - -# test-node-metrics: -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: cargo-fmt -# artifacts: false -# <<: *docker-env -# <<: *compiler-info -# <<: *test-refs -# variables: -# RUST_TOOLCHAIN: stable -# # Enable debug assertions since we are running optimized builds for testing -# # but still want to have debug assertions. -# RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" -# script: -# - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics - -# test-deterministic-wasm: -# stage: stage2 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: cargo-fmt -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# - ./scripts/ci/gitlab/test_deterministic_wasm.sh - -# check-transaction-versions: -# stage: stage2 -# <<: *test-refs -# <<: *docker-env -# image: node:15 -# needs: -# - job: build-linux-stable -# artifacts: true -# before_script: -# - apt-get -y update; apt-get -y install jq lsof -# - npm install --ignore-scripts -g @polkadot/metadata-cmp -# # Set git config -# - git config remote.origin.url "https://github.com/paritytech/polkadot.git" -# - git fetch origin release -# script: -# - ./scripts/ci/gitlab/check_extrinsics_ordering.sh - - - -# # This image is used in testnets -# # Release image is handled by the Github Action here: -# # .github/workflows/publish-docker-release.yml -# publish-polkadot-debug-image: -# stage: stage2 -# <<: *build-push-image -# rules: -# # Don't run when triggered from another pipeline -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# - if: $CI_PIPELINE_SOURCE == "web" -# - if: $CI_PIPELINE_SOURCE == "schedule" -# - if: $CI_COMMIT_REF_NAME == "master" -# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile -# DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile -# IMAGE_NAME: docker.io/paritypr/polkadot-debug -# needs: -# - job: build-linux-stable -# artifacts: true -# after_script: -# # pass artifacts to the zombienet-tests job -# # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance -# - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env -# - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/parachains.env -# expire_in: 1 days - -# publish-test-collators-image: -# # service image for Simnet -# stage: stage2 -# <<: *build-push-image -# <<: *zombienet-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/collator_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/colander -# needs: -# - job: build-test-collators -# artifacts: true -# after_script: -# - buildah logout --all -# # pass artifacts to the zombienet-tests job -# - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env -# - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/collator.env - -# publish-malus-image: -# # service image for Simnet -# stage: stage2 -# <<: *build-push-image -# <<: *zombienet-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/malus_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/malus -# needs: -# - job: build-malus -# artifacts: true -# after_script: -# - buildah logout "$IMAGE_NAME" -# # pass artifacts to the zombienet-tests job -# - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env -# - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/malus.env - -# publish-staking-miner-image: -# stage: stage2 -# <<: *build-push-image -# <<: *publish-refs -# variables: -# <<: *image-variables -# # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile -# IMAGE_NAME: docker.io/paritytech/staking-miner -# GIT_STRATEGY: none -# DOCKER_USER: ${Docker_Hub_User_Parity} -# DOCKER_PASS: ${Docker_Hub_Pass_Parity} -# needs: -# - job: build-staking-miner -# artifacts: true - - -# publish-s3-release: &publish-s3 -# stage: stage3 -# needs: -# - job: build-linux-stable -# artifacts: true -# <<: *kubernetes-env -# image: paritytech/awscli:latest -# variables: -# GIT_STRATEGY: none -# PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" -# rules: -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# # publishing binaries nightly -# - if: $CI_PIPELINE_SOURCE == "schedule" -# before_script: -# - *check-versions -# script: -# - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" -# - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ -# - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" -# - find ./artifacts -type f | while read file; do -# name="${file#./artifacts/}"; -# aws s3api copy-object -# --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} -# --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; -# done -# - | -# cat <<-EOM -# | -# | polkadot binary paths: -# | -# | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot -# | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot -# | -# EOM -# after_script: -# - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ -# --recursive --human-readable --summarize - -# update_polkadot_weights: &update-weights -# stage: stage2 -# timeout: 1d -# when: manual -# variables: -# RUNTIME: polkadot -# artifacts: -# paths: -# - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch -# script: -# - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME -# - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch -# # uses the "shell" executors -# tags: -# - weights - -# update_kusama_weights: -# <<: *update-weights -# variables: -# RUNTIME: kusama - -# update_westend_weights: -# <<: *update-weights -# variables: -# RUNTIME: westend - -# update_rococo_weights: -# <<: *update-weights -# variables: -# RUNTIME: rococo - -# #### stage: stage3 - -# build-rustdoc: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *docker-env -# <<: *test-refs -# variables: -# SKIP_WASM_BUILD: 1 -# artifacts: -# name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" -# when: on_success -# expire_in: 1 days -# paths: -# - ./crate-docs/ -# script: -# # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` -# # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable -# - time cargo +nightly doc --workspace --verbose --no-deps -# - rm -f ./target/doc/.lock -# - mv ./target/doc ./crate-docs -# # FIXME: remove me after CI image gets nonroot -# - chown -R nonroot:nonroot ./crate-docs -# - echo "" > ./crate-docs/index.html - -# build-implementers-guide: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *collect-artifacts-short -# script: -# - cargo install mdbook mdbook-mermaid mdbook-linkcheck -# - mdbook build ./roadmap/implementers-guide -# - mkdir -p artifacts -# - mv roadmap/implementers-guide/book artifacts/ -# # FIXME: remove me after CI image gets nonroot -# - chown -R nonroot:nonroot artifacts/ - -# check-try-runtime: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-node-metrics -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# # Check that everything compiles with `try-runtime` feature flag. -# - cargo check --features try-runtime --all +check-runtime: + stage: stage1 + image: paritytech/tools:latest + <<: *kubernetes-env + rules: + - if: $CI_COMMIT_REF_NAME =~ /^release-v[0-9]+\.[0-9]+.*$/ # i.e. release-v0.9.27 + variables: + GITLAB_API: "https://gitlab.parity.io/api/v4" + GITHUB_API_PROJECT: "parity%2Finfrastructure%2Fgithub-api" + script: + - ./scripts/ci/gitlab/check_runtime.sh + allow_failure: true + +cargo-fmt: + stage: stage1 + <<: *docker-env + <<: *test-refs + script: + - cargo +nightly --version + - cargo +nightly fmt --all -- --check + allow_failure: true + +build-linux-stable: + stage: stage1 + <<: *docker-env + <<: *compiler-info + <<: *collect-artifacts + <<: *common-refs + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + # Ensure we run the UI tests. + RUN_UI_TESTS: 1 + script: + - time cargo build --profile testnet --features pyroscope --verbose --bin polkadot + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - VERSION="${CI_COMMIT_REF_NAME}" # will be tag or branch name + - mv ./target/testnet/polkadot ./artifacts/. + - pushd artifacts + - sha256sum polkadot | tee polkadot.sha256 + - shasum -c polkadot.sha256 + - popd + - EXTRATAG="${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" + - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" + - echo -n ${VERSION} > ./artifacts/VERSION + - echo -n ${EXTRATAG} > ./artifacts/EXTRATAG + - cp -r scripts/* ./artifacts + +test-linux-stable: + stage: stage1 + <<: *docker-env + <<: *common-refs + <<: *pipeline-stopper-artifacts + before_script: + - rustup show + - cargo --version + - *pipeline-stopper-vars + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --workspace --profile testnet --verbose --locked --features=runtime-benchmarks,runtime-metrics + + + +spellcheck: + stage: stage1 + <<: *docker-env + <<: *test-refs + script: + - cargo spellcheck --version + # compare with the commit parent to the PR, given it's from a default branch + - git fetch origin +${CI_DEFAULT_BRANCH}:${CI_DEFAULT_BRANCH} + - echo "___Spellcheck is going to check your diff___" + - cargo spellcheck list-files -vvv $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + - time cargo spellcheck check -vvv --cfg=scripts/ci/gitlab/spellcheck.toml --checkers hunspell --code 1 + $(git diff --diff-filter=AM --name-only $(git merge-base ${CI_COMMIT_SHA} ${CI_DEFAULT_BRANCH} -- :^bridges)) + allow_failure: true + +build-test-collators: + stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *test-refs + script: + - time cargo build --profile testnet --verbose -p test-parachain-adder-collator + - time cargo build --profile testnet --verbose -p test-parachain-undying-collator + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/adder-collator ./artifacts/. + - mv ./target/testnet/undying-collator ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "adder-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - echo "undying-collator version = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-malus: + stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *test-refs + script: + - time cargo build --profile testnet --verbose -p polkadot-test-malus + - sccache -s + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/testnet/malus ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "polkadot-test-malus = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +build-staking-miner: + stage: stage1 + <<: *collect-artifacts + <<: *docker-env + <<: *compiler-info + <<: *common-refs + script: + - time cargo build --locked --release --package staking-miner + # pack artifacts + - mkdir -p ./artifacts + - mv ./target/release/staking-miner ./artifacts/. + - echo -n "${CI_COMMIT_REF_NAME}" > ./artifacts/VERSION + - echo -n "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" > ./artifacts/EXTRATAG + - echo "staking-miner = $(cat ./artifacts/VERSION) (EXTRATAG = $(cat ./artifacts/EXTRATAG))" + - cp -r ./scripts/* ./artifacts + +#### stage: stage2 + +.check-dependent-project: &check-dependent-project + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: cargo-fmt + artifacts: false + <<: *docker-env + <<: *test-pr-refs + script: + - git clone + --depth=1 + "--branch=$PIPELINE_SCRIPTS_TAG" + https://github.com/paritytech/pipeline-scripts + - ./pipeline-scripts/check_dependent_project.sh + --org paritytech + --dependent-repo "$DEPENDENT_REPO" + --github-api-token "$GITHUB_PR_TOKEN" + --extra-dependencies "$EXTRA_DEPENDENCIES" + --companion-overrides "$COMPANION_OVERRIDES" + +check-dependent-cumulus: + <<: *check-dependent-project + variables: + DEPENDENT_REPO: cumulus + EXTRA_DEPENDENCIES: substrate + COMPANION_OVERRIDES: | + polkadot: release-v* + cumulus: polkadot-v* + +test-node-metrics: + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: cargo-fmt + artifacts: false + <<: *docker-env + <<: *compiler-info + <<: *test-refs + variables: + RUST_TOOLCHAIN: stable + # Enable debug assertions since we are running optimized builds for testing + # but still want to have debug assertions. + RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings" + script: + - time cargo test --profile testnet --verbose --locked --features=runtime-metrics -p polkadot-node-metrics + +test-deterministic-wasm: + stage: stage2 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: cargo-fmt + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + - ./scripts/ci/gitlab/test_deterministic_wasm.sh + +check-transaction-versions: + stage: stage2 + <<: *test-refs + <<: *docker-env + image: node:15 + needs: + - job: build-linux-stable + artifacts: true + before_script: + - apt-get -y update; apt-get -y install jq lsof + - npm install --ignore-scripts -g @polkadot/metadata-cmp + # Set git config + - git config remote.origin.url "https://github.com/paritytech/polkadot.git" + - git fetch origin release + script: + - ./scripts/ci/gitlab/check_extrinsics_ordering.sh + + + +# This image is used in testnets +# Release image is handled by the Github Action here: +# .github/workflows/publish-docker-release.yml +publish-polkadot-debug-image: + stage: stage2 + <<: *build-push-image + rules: + # Don't run when triggered from another pipeline + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" + - if: $CI_PIPELINE_SOURCE == "schedule" + - if: $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 + variables: + <<: *image-variables + # scripts/ci/dockerfiles/polkadot_injected_debug.Dockerfile + DOCKERFILE: ci/dockerfiles/polkadot_injected_debug.Dockerfile + IMAGE_NAME: docker.io/paritypr/polkadot-debug + needs: + - job: build-linux-stable + artifacts: true + after_script: + # pass artifacts to the zombienet-tests job + # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance + - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env + - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/parachains.env + expire_in: 1 days + +publish-test-collators-image: + # service image for Simnet + stage: stage2 + <<: *build-push-image + <<: *zombienet-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/collator_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/collator_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/colander + needs: + - job: build-test-collators + artifacts: true + after_script: + - buildah logout --all + # pass artifacts to the zombienet-tests job + - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env + - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/collator.env + +publish-malus-image: + # service image for Simnet + stage: stage2 + <<: *build-push-image + <<: *zombienet-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/malus_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/malus_injected.Dockerfile + IMAGE_NAME: docker.io/paritypr/malus + needs: + - job: build-malus + artifacts: true + after_script: + - buildah logout "$IMAGE_NAME" + # pass artifacts to the zombienet-tests job + - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env + - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env + artifacts: + reports: + # this artifact is used in zombienet-tests job + dotenv: ./artifacts/malus.env + +publish-staking-miner-image: + stage: stage2 + <<: *build-push-image + <<: *publish-refs + variables: + <<: *image-variables + # scripts/ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + DOCKERFILE: ci/dockerfiles/staking-miner/staking-miner_injected.Dockerfile + IMAGE_NAME: docker.io/paritytech/staking-miner + GIT_STRATEGY: none + DOCKER_USER: ${Docker_Hub_User_Parity} + DOCKER_PASS: ${Docker_Hub_Pass_Parity} + needs: + - job: build-staking-miner + artifacts: true + + +publish-s3-release: &publish-s3 + stage: stage3 + needs: + - job: build-linux-stable + artifacts: true + <<: *kubernetes-env + image: paritytech/awscli:latest + variables: + GIT_STRATEGY: none + PREFIX: "builds/polkadot/${ARCH}-${DOCKER_OS}" + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + # publishing binaries nightly + - if: $CI_PIPELINE_SOURCE == "schedule" + before_script: + - *check-versions + script: + - echo "uploading objects to https://releases.parity.io/${PREFIX}/${VERSION}" + - aws s3 sync --acl public-read ./artifacts/ s3://${AWS_BUCKET}/${PREFIX}/${VERSION}/ + - echo "update objects at https://releases.parity.io/${PREFIX}/${EXTRATAG}" + - find ./artifacts -type f | while read file; do + name="${file#./artifacts/}"; + aws s3api copy-object + --copy-source ${AWS_BUCKET}/${PREFIX}/${VERSION}/${name} + --bucket ${AWS_BUCKET} --key ${PREFIX}/${EXTRATAG}/${name}; + done + - | + cat <<-EOM + | + | polkadot binary paths: + | + | - https://releases.parity.io/${PREFIX}/${EXTRATAG}/polkadot + | - https://releases.parity.io/${PREFIX}/${VERSION}/polkadot + | + EOM + after_script: + - aws s3 ls s3://${AWS_BUCKET}/${PREFIX}/${EXTRATAG}/ + --recursive --human-readable --summarize + +update_polkadot_weights: &update-weights + stage: stage2 + timeout: 1d + when: manual + variables: + RUNTIME: polkadot + artifacts: + paths: + - ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + script: + - ./scripts/ci/run_benches_for_runtime.sh $RUNTIME + - git diff -P > ${RUNTIME}_weights_${CI_COMMIT_SHORT_SHA}.patch + # uses the "shell" executors + tags: + - weights + +update_kusama_weights: + <<: *update-weights + variables: + RUNTIME: kusama + +update_westend_weights: + <<: *update-weights + variables: + RUNTIME: westend + +update_rococo_weights: + <<: *update-weights + variables: + RUNTIME: rococo + +#### stage: stage3 + +build-rustdoc: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *docker-env + <<: *test-refs + variables: + SKIP_WASM_BUILD: 1 + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc" + when: on_success + expire_in: 1 days + paths: + - ./crate-docs/ + script: + # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features` + # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable + - time cargo +nightly doc --workspace --verbose --no-deps + - rm -f ./target/doc/.lock + - mv ./target/doc ./crate-docs + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot ./crate-docs + - echo "" > ./crate-docs/index.html + +build-implementers-guide: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *collect-artifacts-short + script: + - cargo install mdbook mdbook-mermaid mdbook-linkcheck + - mdbook build ./roadmap/implementers-guide + - mkdir -p artifacts + - mv roadmap/implementers-guide/book artifacts/ + # FIXME: remove me after CI image gets nonroot + - chown -R nonroot:nonroot artifacts/ + +check-try-runtime: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-node-metrics + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + # Check that everything compiles with `try-runtime` feature flag. + - cargo check --features try-runtime --all # More info can be found here: https://github.com/paritytech/polkadot/pull/5865 # Works only in PRs @@ -650,415 +650,415 @@ check-runtime-migration: fi -# check-no-default-features: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *test-refs -# <<: *docker-env -# <<: *compiler-info -# script: -# # Check that polkadot-cli will compile no default features. -# - pushd ./node/service && cargo check --no-default-features && popd -# - pushd ./cli && cargo check --no-default-features --features "service" && popd -# - sccache -s - -# build-short-benchmark: -# stage: stage3 -# <<: *test-refs -# <<: *docker-env -# <<: *collect-artifacts -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-node-metrics -# artifacts: false -# script: -# - cargo +nightly build --profile release --locked --features=runtime-benchmarks -# - mkdir artifacts -# - cp ./target/release/polkadot ./artifacts/ - -# deploy-parity-testnet: -# stage: stage3 -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: test-deterministic-wasm -# artifacts: false -# <<: *deploy-testnet-refs -# variables: -# POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" -# POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" -# allow_failure: false -# trigger: "parity/infrastructure/parity-testnet" - -# zombienet-tests-parachains-smoke-test: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-parachains-smoke-test.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-parachains-pvf: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-parachains-pvf.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-parachains-disputes: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# - job: publish-malus-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0002-parachains-disputes.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-parachains-disputes-garbage-candidate: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# - job: publish-malus-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0003-parachains-garbage-candidate.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-test-parachains-upgrade-smoke-test: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' -# before_script: -# - echo "ZombieNet Tests Config" -# - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" -# - echo "docker.io/parity/polkadot-collator:latest" -# - echo "${ZOMBIENET_IMAGE}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0002-parachains-upgrade-smoke-test.feature" -# allow_failure: true -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-misc-paritydb: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-test-collators-image -# artifacts: true -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie,zombie::network-node -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-paritydb.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-malus-dispute-valid: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# - job: publish-malus-image -# - job: publish-test-collators-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie* -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0001-dispute-valid-block.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-deregister-register-validator: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# artifacts: true -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie* -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0003-deregister-register-validator-smoke.feature" -# allow_failure: false -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# zombienet-tests-beefy-and-mmr: -# stage: stage3 -# image: "${ZOMBIENET_IMAGE}" -# <<: *kubernetes-env -# <<: *zombienet-refs -# needs: -# - job: publish-polkadot-debug-image -# variables: -# GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" -# before_script: -# - echo "Zombie-net Tests Config" -# - echo "${ZOMBIENET_IMAGE_NAME}" -# - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" -# - echo "${GH_DIR}" -# - export DEBUG=zombie* -# - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh -# --github-remote-dir="${GH_DIR}" -# --test="0003-beefy-and-mmr.feature" -# allow_failure: true -# retry: 2 -# tags: -# - zombienet-polkadot-integration-test - -# #### stage: stage4 - -# publish-rustdoc: -# stage: stage4 -# <<: *kubernetes-env -# image: paritytech/tools:latest -# variables: -# GIT_DEPTH: 100 -# rules: -# - if: $CI_PIPELINE_SOURCE == "pipeline" -# when: never -# - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" -# - if: $CI_COMMIT_REF_NAME == "master" -# # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other -# # artifacts from being dowloaded by this job. -# needs: -# - job: build-rustdoc -# artifacts: true -# - job: build-implementers-guide -# artifacts: true -# script: -# # Save README and docs -# - cp -r ./crate-docs/ /tmp/doc/ -# - cp -r ./artifacts/book/ /tmp/ -# # setup ssh -# - eval $(ssh-agent) -# - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} -# - mkdir ~/.ssh && touch ~/.ssh/known_hosts -# - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts -# # Set git config -# - git config user.email "devops-team@parity.io" -# - git config user.name "${GITHUB_USER}" -# - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" -# - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" -# - git fetch origin gh-pages -# - git checkout gh-pages -# # Remove everything and restore generated docs and README -# - cp index.html /tmp -# - cp README.md /tmp -# - rm -rf ./* -# # dir for rustdoc -# - mkdir -p doc -# # dir for implementors guide -# - mkdir -p book -# - mv /tmp/doc/* doc/ -# - mv /tmp/book/html/* book/ -# - mv /tmp/index.html . -# - mv /tmp/README.md . -# # Upload files -# - git add --all --force -# # `git commit` has an exit code of > 0 if there is nothing to commit. -# # This causes GitLab to exit immediately and marks this job failed. -# # We don't want to mark the entire job failed if there's nothing to -# # publish though, hence the `|| true`. -# - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || -# echo "___Nothing to commit___" -# - git push origin gh-pages --force -# - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" -# after_script: -# - rm -rf .git/ ./* - -# # Run all pallet benchmarks only once to check if there are any errors -# short-benchmark-polkadot: &short-bench -# stage: stage4 -# <<: *test-pr-refs -# <<: *docker-env -# # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs -# needs: -# - job: build-short-benchmark -# artifacts: true -# variables: -# RUNTIME: polkadot -# script: -# - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 - -# short-benchmark-kusama: -# <<: *short-bench -# variables: -# RUNTIME: kusama - -# short-benchmark-westend: -# <<: *short-bench -# variables: -# RUNTIME: westend - -# #### stage: .post - -# # This job cancels the whole pipeline if any of provided jobs fail. -# # In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests -# # to fail the pipeline as soon as possible to shorten the feedback loop. -# .cancel-pipeline-template: -# stage: .post -# rules: -# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -# when: on_failure -# variables: -# PROJECT_ID: "${CI_PROJECT_ID}" -# PROJECT_NAME: "${CI_PROJECT_NAME}" -# PIPELINE_ID: "${CI_PIPELINE_ID}" -# FAILED_JOB_URL: "${FAILED_JOB_URL}" -# FAILED_JOB_NAME: "${FAILED_JOB_NAME}" -# PR_NUM: "${PR_NUM}" -# trigger: -# project: "parity/infrastructure/ci_cd/pipeline-stopper" -# # remove branch, when pipeline-stopper for polakdot is updated to the same branch -# branch: "as-improve" - -# cancel-pipeline-test-linux-stable: -# extends: .cancel-pipeline-template -# needs: -# - job: test-linux-stable +check-no-default-features: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *test-refs + <<: *docker-env + <<: *compiler-info + script: + # Check that polkadot-cli will compile no default features. + - pushd ./node/service && cargo check --no-default-features && popd + - pushd ./cli && cargo check --no-default-features --features "service" && popd + - sccache -s + +build-short-benchmark: + stage: stage3 + <<: *test-refs + <<: *docker-env + <<: *collect-artifacts + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-node-metrics + artifacts: false + script: + - cargo +nightly build --profile release --locked --features=runtime-benchmarks + - mkdir artifacts + - cp ./target/release/polkadot ./artifacts/ + +deploy-parity-testnet: + stage: stage3 + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: test-deterministic-wasm + artifacts: false + <<: *deploy-testnet-refs + variables: + POLKADOT_CI_COMMIT_NAME: "${CI_COMMIT_REF_NAME}" + POLKADOT_CI_COMMIT_REF: "${CI_COMMIT_SHORT_SHA}" + allow_failure: false + trigger: "parity/infrastructure/parity-testnet" + +zombienet-tests-parachains-smoke-test: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE="docker.io/paritypr/colander:4519" # The collator image is fixed + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-smoke-test.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-pvf: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-parachains-pvf.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-disputes: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-disputes.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-parachains-disputes-garbage-candidate: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + - job: publish-malus-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-parachains-garbage-candidate.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-test-parachains-upgrade-smoke-test: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: 'https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke' + before_script: + - echo "ZombieNet Tests Config" + - echo "${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG}" + - echo "docker.io/parity/polkadot-collator:latest" + - echo "${ZOMBIENET_IMAGE}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE="docker.io/parity/polkadot-collator:latest" # Use cumulus lastest image + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0002-parachains-upgrade-smoke-test.feature" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-misc-paritydb: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-test-collators-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/misc" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-paritydb.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-malus-dispute-valid: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + - job: publish-malus-image + - job: publish-test-collators-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/node/malus/integrationtests" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${MALUS_IMAGE_NAME} ${MALUS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + - export COL_IMAGE=${COLLATOR_IMAGE_NAME}:${COLLATOR_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0001-dispute-valid-block.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-deregister-register-validator: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + artifacts: true + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/smoke" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + - export MALUS_IMAGE=${MALUS_IMAGE_NAME}:${MALUS_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-deregister-register-validator-smoke.feature" + allow_failure: false + retry: 2 + tags: + - zombienet-polkadot-integration-test + +zombienet-tests-beefy-and-mmr: + stage: stage3 + image: "${ZOMBIENET_IMAGE}" + <<: *kubernetes-env + <<: *zombienet-refs + needs: + - job: publish-polkadot-debug-image + variables: + GH_DIR: "https://github.com/paritytech/polkadot/tree/${CI_COMMIT_SHORT_SHA}/zombienet_tests/functional" + before_script: + - echo "Zombie-net Tests Config" + - echo "${ZOMBIENET_IMAGE_NAME}" + - echo "${PARACHAINS_IMAGE_NAME} ${PARACHAINS_IMAGE_TAG}" + - echo "${GH_DIR}" + - export DEBUG=zombie* + - export ZOMBIENET_INTEGRATION_TEST_IMAGE=${PARACHAINS_IMAGE_NAME}:${PARACHAINS_IMAGE_TAG} + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-env-manager.sh + --github-remote-dir="${GH_DIR}" + --test="0003-beefy-and-mmr.feature" + allow_failure: true + retry: 2 + tags: + - zombienet-polkadot-integration-test + +#### stage: stage4 + +publish-rustdoc: + stage: stage4 + <<: *kubernetes-env + image: paritytech/tools:latest + variables: + GIT_DEPTH: 100 + rules: + - if: $CI_PIPELINE_SOURCE == "pipeline" + when: never + - if: $CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_REF_NAME == "master" + - if: $CI_COMMIT_REF_NAME == "master" + # `needs:` can be removed after CI image gets nonroot. In this case `needs:` stops other + # artifacts from being dowloaded by this job. + needs: + - job: build-rustdoc + artifacts: true + - job: build-implementers-guide + artifacts: true + script: + # Save README and docs + - cp -r ./crate-docs/ /tmp/doc/ + - cp -r ./artifacts/book/ /tmp/ + # setup ssh + - eval $(ssh-agent) + - ssh-add - <<< ${GITHUB_SSH_PRIV_KEY} + - mkdir ~/.ssh && touch ~/.ssh/known_hosts + - ssh-keyscan -t rsa github.com >> ~/.ssh/known_hosts + # Set git config + - git config user.email "devops-team@parity.io" + - git config user.name "${GITHUB_USER}" + - git config remote.origin.url "git@github.com:/paritytech/${CI_PROJECT_NAME}.git" + - git config remote.origin.fetch "+refs/heads/*:refs/remotes/origin/*" + - git fetch origin gh-pages + - git checkout gh-pages + # Remove everything and restore generated docs and README + - cp index.html /tmp + - cp README.md /tmp + - rm -rf ./* + # dir for rustdoc + - mkdir -p doc + # dir for implementors guide + - mkdir -p book + - mv /tmp/doc/* doc/ + - mv /tmp/book/html/* book/ + - mv /tmp/index.html . + - mv /tmp/README.md . + # Upload files + - git add --all --force + # `git commit` has an exit code of > 0 if there is nothing to commit. + # This causes GitLab to exit immediately and marks this job failed. + # We don't want to mark the entire job failed if there's nothing to + # publish though, hence the `|| true`. + - git commit -m "Updated docs for ${CI_COMMIT_REF_NAME}" || + echo "___Nothing to commit___" + - git push origin gh-pages --force + - echo "___Rustdoc was successfully published to https://paritytech.github.io/polkadot/___" + after_script: + - rm -rf .git/ ./* + +# Run all pallet benchmarks only once to check if there are any errors +short-benchmark-polkadot: &short-bench + stage: stage4 + <<: *test-pr-refs + <<: *docker-env + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: build-short-benchmark + artifacts: true + variables: + RUNTIME: polkadot + script: + - ./artifacts/polkadot benchmark pallet --execution wasm --wasm-execution compiled --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 + +short-benchmark-kusama: + <<: *short-bench + variables: + RUNTIME: kusama + +short-benchmark-westend: + <<: *short-bench + variables: + RUNTIME: westend + +#### stage: .post + +# This job cancels the whole pipeline if any of provided jobs fail. +# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests +# to fail the pipeline as soon as possible to shorten the feedback loop. +.cancel-pipeline-template: + stage: .post + rules: + - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs + when: on_failure + variables: + PROJECT_ID: "${CI_PROJECT_ID}" + PROJECT_NAME: "${CI_PROJECT_NAME}" + PIPELINE_ID: "${CI_PIPELINE_ID}" + FAILED_JOB_URL: "${FAILED_JOB_URL}" + FAILED_JOB_NAME: "${FAILED_JOB_NAME}" + PR_NUM: "${PR_NUM}" + trigger: + project: "parity/infrastructure/ci_cd/pipeline-stopper" + # remove branch, when pipeline-stopper for polakdot is updated to the same branch + branch: "as-improve" + +cancel-pipeline-test-linux-stable: + extends: .cancel-pipeline-template + needs: + - job: test-linux-stable From 47ff0c9703d0a40042bf24ef5b39ec23ee7911d0 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 17 Aug 2022 11:19:06 +0200 Subject: [PATCH 5/6] uncomment needs --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d7eb92acde9a..83f95d1f4497 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -629,9 +629,9 @@ check-try-runtime: check-runtime-migration: stage: stage3 # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - # needs: - # - job: test-node-metrics - # artifacts: false + needs: + - job: test-node-metrics + artifacts: false <<: *test-pr-refs <<: *docker-env <<: *compiler-info From 9d7afdb06b77267b2ef1e77b1c4bcc48211aeec5 Mon Sep 17 00:00:00 2001 From: alvicsam Date: Wed, 17 Aug 2022 11:33:56 +0200 Subject: [PATCH 6/6] change label name to code label --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 83f95d1f4497..94f700f7b985 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -639,7 +639,7 @@ check-runtime-migration: - | export has_runtimemigration_label=$(curl -sS -H "Accept: application/vnd.github+json" \ -H "Authorization: token $GITHUB_PR_TOKEN" \ - https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "runtimemigration" | wc -l) + https://api.github.com/repos/paritytech/polkadot/issues/$CI_COMMIT_REF_NAME/labels | grep "E1" | wc -l) - | if [[ $has_runtimemigration_label != 0 ]]; then echo "Found label runtimemigration. Running tests"