diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bafd74f..94fa046 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,26 @@ jobs: SPLIT_INDEX: ${{ strategy.job-index }} DEBUG: 'cypress-split' + test-random-order: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + containers: [1, 2] + steps: + - name: Checkout ๐Ÿ›Ž + uses: actions/checkout@v4 + + - name: Run random order Cypress E2E tests ๐Ÿงช + # https://github.com/cypress-io/github-action + uses: cypress-io/github-action@v6 + # using operating system process environment variables + env: + SPLIT_RANDOM_SEED: 42 + SPLIT: ${{ strategy.job-total }} + SPLIT_INDEX: ${{ strategy.job-index }} + DEBUG: 'cypress-split' + test-subfolder: runs-on: ubuntu-22.04 steps: @@ -329,6 +349,7 @@ jobs: if: github.ref == 'refs/heads/main' needs: [ + test-random-order, test-skipped-specs, test-unit, test-empty, diff --git a/README.md b/README.md index 59751e4..52ce598 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,18 @@ SPEC="cypress/e2e/**/*.cy.js" npx cypress run --spec "cypress/e2e/**/*.cy.js" npx cypress run --spec "cypress/e2e/**/*.cy.js" --env spec="cypress/e2e/**/*.cy.js" ``` +## Random shuffle + +You can shuffle the found specs before splitting using a stable seed + +``` +$ SPLIT_RANDOM_SEED=42 npx cypress run ... +``` + +This is useful to randomize the order of specs to find any dependencies between the tests. + +**Note:** all parallel machines usually compute the list of specs, thus the seed must be the same to guarantee the same list is generated and split correctly, otherwise some specs would be "lost". + ## Relative specs output If `cypress-split` has `SPLIT` and the index and finds the specs, it sets the list of specs in the `config` object diff --git a/package-lock.json b/package-lock.json index 4eabb43..0e6efb3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "arg": "^5.0.2", "console.table": "^0.10.0", "debug": "^4.3.4", + "fast-shuffle": "^6.1.0", "find-cypress-specs": "1.43.1", "globby": "^11.1.0", "humanize-duration": "^3.28.0" @@ -4272,6 +4273,14 @@ "node": ">=8.6.0" } }, + "node_modules/fast-shuffle": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/fast-shuffle/-/fast-shuffle-6.1.0.tgz", + "integrity": "sha512-3aj8oO6bvZFKYDGvXNmmEuxyOjre8trCpIbtFSM/DSKd+o3iSbQQPb5BZQeJ7SPYVivn9EeW3gKh0QdnD027MQ==", + "dependencies": { + "pcg": "1.0.0" + } + }, "node_modules/fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -5949,6 +5958,11 @@ "node": ">=8" } }, + "node_modules/long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -9551,6 +9565,15 @@ "node": ">=8" } }, + "node_modules/pcg": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/pcg/-/pcg-1.0.0.tgz", + "integrity": "sha512-6wjoSJZ4TEJhI0rLDOKd5mOu6TwS4svn9oBaRsD1PCrhlDNLWAaTimWJgBABmIGJxzkI+RbaHJYRLGVf9QFE5Q==", + "dependencies": { + "long": "5.2.3", + "ramda": "0.29.0" + } + }, "node_modules/pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -9906,6 +9929,15 @@ "resolved": "https://registry.npmjs.org/quote-unquote/-/quote-unquote-1.0.0.tgz", "integrity": "sha512-twwRO/ilhlG/FIgYeKGFqyHhoEhqgnKVkcmqMKi2r524gz3ZbDTcyFt38E9xjJI2vT+KbRNHVbnJ/e0I25Azwg==" }, + "node_modules/ramda": { + "version": "0.29.0", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.29.0.tgz", + "integrity": "sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/ramda" + } + }, "node_modules/rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", @@ -15406,6 +15438,14 @@ "micromatch": "^4.0.4" } }, + "fast-shuffle": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/fast-shuffle/-/fast-shuffle-6.1.0.tgz", + "integrity": "sha512-3aj8oO6bvZFKYDGvXNmmEuxyOjre8trCpIbtFSM/DSKd+o3iSbQQPb5BZQeJ7SPYVivn9EeW3gKh0QdnD027MQ==", + "requires": { + "pcg": "1.0.0" + } + }, "fastq": { "version": "1.15.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.15.0.tgz", @@ -16614,6 +16654,11 @@ } } }, + "long": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==" + }, "loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -19034,6 +19079,15 @@ "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz", "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==" }, + "pcg": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/pcg/-/pcg-1.0.0.tgz", + "integrity": "sha512-6wjoSJZ4TEJhI0rLDOKd5mOu6TwS4svn9oBaRsD1PCrhlDNLWAaTimWJgBABmIGJxzkI+RbaHJYRLGVf9QFE5Q==", + "requires": { + "long": "5.2.3", + "ramda": "0.29.0" + } + }, "pend": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", @@ -19275,6 +19329,11 @@ "resolved": "https://registry.npmjs.org/quote-unquote/-/quote-unquote-1.0.0.tgz", "integrity": "sha512-twwRO/ilhlG/FIgYeKGFqyHhoEhqgnKVkcmqMKi2r524gz3ZbDTcyFt38E9xjJI2vT+KbRNHVbnJ/e0I25Azwg==" }, + "ramda": { + "version": "0.29.0", + "resolved": "https://registry.npmjs.org/ramda/-/ramda-0.29.0.tgz", + "integrity": "sha512-BBea6L67bYLtdbOqfp8f58fPMqEwx0doL+pAi8TZyp2YWz8R9G8z9x75CZI8W+ftqhFHCpEX2cRnUUXK130iKA==" + }, "rc": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", diff --git a/package.json b/package.json index fea4700..a1cfbf9 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "demo-merge": "node ./bin/merge --parent-folder examples/split-times --split-file timings.json --output out-timings.json", "demo-preview": "node ./bin/preview --split 2", "demo-preview-spec": "SPEC=\"cypress/e2e/spec*.cy.js\" node ./bin/preview --split 2", + "demo-preview-shuffle": "SPLIT_RANDOM_SEED=42 node ./bin/preview --split 2", "unit": "ava test/*.test.js", "unit:watch": "ava --watch test/*.test.js" }, @@ -53,6 +54,7 @@ "arg": "^5.0.2", "console.table": "^0.10.0", "debug": "^4.3.4", + "fast-shuffle": "^6.1.0", "find-cypress-specs": "1.43.1", "globby": "^11.1.0", "humanize-duration": "^3.28.0" diff --git a/src/parse-inputs.js b/src/parse-inputs.js index 85233b7..430bf80 100644 --- a/src/parse-inputs.js +++ b/src/parse-inputs.js @@ -3,6 +3,7 @@ const debug = require('debug')('cypress-split') const path = require('path') const { getSpecs } = require('find-cypress-specs') const globby = require('globby') +const { createShuffle } = require('fast-shuffle') function parseSplitInputs(env = {}, configEnv = {}) { let SPLIT = env.SPLIT || configEnv.split || configEnv.SPLIT @@ -94,6 +95,14 @@ function getSpecsToSplit(env = {}, config) { } } + let splitRandomSeed = null + if (env.SPLIT_RANDOM_SEED) { + splitRandomSeed = Number(env.SPLIT_RANDOM_SEED) + debug('found random seed %d', splitRandomSeed) + } + + let foundSpecs + // potentially a list of files to run / split let SPEC = env.SPEC || config?.env?.spec || config?.env?.SPEC if (typeof SPEC === 'string' && SPEC) { @@ -134,7 +143,8 @@ function getSpecsToSplit(env = {}, config) { skipSpecs.length, ) } - return specs.filter((spec) => !skipSpecs.includes(spec)) + + foundSpecs = specs } else { const returnAbsolute = true const specs = getSpecs(config, undefined, returnAbsolute) @@ -146,8 +156,21 @@ function getSpecsToSplit(env = {}, config) { skipSpecs.length, ) } - return specs.filter((spec) => !skipSpecs.includes(spec)) + foundSpecs = specs } + + debug('skipping %d specs', skipSpecs.length) + const filteredSpecs = foundSpecs.filter((spec) => !skipSpecs.includes(spec)) + + if (splitRandomSeed) { + debug('shuffling specs using random seed %d', splitRandomSeed) + // shuffle the specs using the random seed + const shuffleSpecs = createShuffle(splitRandomSeed) + const shuffledSpecs = shuffleSpecs(filteredSpecs) + return shuffledSpecs + } + + return filteredSpecs } module.exports = { parseSplitInputs, getSpecsToSplit } diff --git a/test/parse-inputs.test.js b/test/parse-inputs.test.js index 693b287..4442bb8 100644 --- a/test/parse-inputs.test.js +++ b/test/parse-inputs.test.js @@ -63,3 +63,18 @@ test('getSpecsToSplit spec pattern with subfolder wildcards', (t) => { 'cypress/e2e/spec-e.cy.js', ]) }) + +test('getSpecsToSplit with random seed shuffle', (t) => { + const specs = getSpecsToSplit({ + SPEC: 'cypress/**/spec-*.cy.js', + SPLIT_RANDOM_SEED: '11', + }) + const relativeSpecs = toRelative(specs) + t.deepEqual(relativeSpecs, [ + 'cypress/e2e/spec-d.cy.js', + 'cypress/e2e/spec-e.cy.js', + 'cypress/e2e/spec-b.cy.js', + 'cypress/e2e/spec-c.cy.js', + 'cypress/e2e/spec-a.cy.js', + ]) +})