feat: add shuffle seed feature

.github/workflows/ci.yml

@@ -85,6 +85,26 @@ jobs:
           SPLIT_INDEX: ${{ strategy.job-index }}
           DEBUG: 'cypress-split'
 
+  test-random-order:
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        containers: [1, 2]
+    steps:
+      - name: Checkout 🛎
+        uses: actions/checkout@v4
+
+      - name: Run random order Cypress E2E tests 🧪
+        # https://github.com/cypress-io/github-action
+        uses: cypress-io/github-action@v6
+        # using operating system process environment variables
+        env:
+          SPLIT_RANDOM_SEED: 42
+          SPLIT: ${{ strategy.job-total }}
+          SPLIT_INDEX: ${{ strategy.job-index }}
+          DEBUG: 'cypress-split'
+
   test-subfolder:
     runs-on: ubuntu-22.04
     steps:
@@ -329,6 +349,7 @@ jobs:
     if: github.ref == 'refs/heads/main'
     needs:
       [
+        test-random-order,
         test-skipped-specs,
         test-unit,
         test-empty,

README.md

@@ -452,6 +452,18 @@ SPEC="cypress/e2e/**/*.cy.js" npx cypress run --spec "cypress/e2e/**/*.cy.js"
 npx cypress run --spec "cypress/e2e/**/*.cy.js" --env spec="cypress/e2e/**/*.cy.js"
 ```
 
+## Random shuffle
+
+You can shuffle the found specs before splitting using a stable seed
+
+```
+$ SPLIT_RANDOM_SEED=42 npx cypress run ...
+```
+
+This is useful to randomize the order of specs to find any dependencies between the tests.
+
+**Note:** all parallel machines usually compute the list of specs, thus the seed must be the same to guarantee the same list is generated and split correctly, otherwise some specs would be "lost".
+
 ## Relative specs output
 
 If `cypress-split` has `SPLIT` and the index and finds the specs, it sets the list of specs in the `config` object

package.json

@@ -29,6 +29,7 @@
     "demo-merge": "node ./bin/merge --parent-folder examples/split-times --split-file timings.json --output out-timings.json",
     "demo-preview": "node ./bin/preview --split 2",
     "demo-preview-spec": "SPEC=\"cypress/e2e/spec*.cy.js\" node ./bin/preview --split 2",
+    "demo-preview-shuffle": "SPLIT_RANDOM_SEED=42 node ./bin/preview --split 2",
     "unit": "ava test/*.test.js",
     "unit:watch": "ava --watch test/*.test.js"
   },
@@ -53,6 +54,7 @@
     "arg": "^5.0.2",
     "console.table": "^0.10.0",
     "debug": "^4.3.4",
+    "fast-shuffle": "^6.1.0",
     "find-cypress-specs": "1.43.1",
     "globby": "^11.1.0",
     "humanize-duration": "^3.28.0"

src/parse-inputs.js

@@ -3,6 +3,7 @@ const debug = require('debug')('cypress-split')
 const path = require('path')
 const { getSpecs } = require('find-cypress-specs')
 const globby = require('globby')
+const { createShuffle } = require('fast-shuffle')
 
 function parseSplitInputs(env = {}, configEnv = {}) {
   let SPLIT = env.SPLIT || configEnv.split || configEnv.SPLIT
@@ -94,6 +95,14 @@ function getSpecsToSplit(env = {}, config) {
     }
   }
 
+  let splitRandomSeed = null
+  if (env.SPLIT_RANDOM_SEED) {
+    splitRandomSeed = Number(env.SPLIT_RANDOM_SEED)
+    debug('found random seed %d', splitRandomSeed)
+  }
+
+  let foundSpecs
+
   // potentially a list of files to run / split
   let SPEC = env.SPEC || config?.env?.spec || config?.env?.SPEC
   if (typeof SPEC === 'string' && SPEC) {
@@ -134,7 +143,8 @@ function getSpecsToSplit(env = {}, config) {
         skipSpecs.length,
       )
     }
-    return specs.filter((spec) => !skipSpecs.includes(spec))
+
+    foundSpecs = specs
   } else {
     const returnAbsolute = true
     const specs = getSpecs(config, undefined, returnAbsolute)
@@ -146,8 +156,21 @@ function getSpecsToSplit(env = {}, config) {
         skipSpecs.length,
       )
     }
-    return specs.filter((spec) => !skipSpecs.includes(spec))
+    foundSpecs = specs
   }
+
+  debug('skipping %d specs', skipSpecs.length)
+  const filteredSpecs = foundSpecs.filter((spec) => !skipSpecs.includes(spec))
+
+  if (splitRandomSeed) {
+    debug('shuffling specs using random seed %d', splitRandomSeed)
+    // shuffle the specs using the random seed
+    const shuffleSpecs = createShuffle(splitRandomSeed)
+    const shuffledSpecs = shuffleSpecs(filteredSpecs)
+    return shuffledSpecs
+  }
+
+  return filteredSpecs
 }
 
 module.exports = { parseSplitInputs, getSpecsToSplit }

test/parse-inputs.test.js

@@ -63,3 +63,18 @@ test('getSpecsToSplit spec pattern with subfolder wildcards', (t) => {
     'cypress/e2e/spec-e.cy.js',
   ])
 })
+
+test('getSpecsToSplit with random seed shuffle', (t) => {
+  const specs = getSpecsToSplit({
+    SPEC: 'cypress/**/spec-*.cy.js',
+    SPLIT_RANDOM_SEED: '11',
+  })
+  const relativeSpecs = toRelative(specs)
+  t.deepEqual(relativeSpecs, [
+    'cypress/e2e/spec-d.cy.js',
+    'cypress/e2e/spec-e.cy.js',
+    'cypress/e2e/spec-b.cy.js',
+    'cypress/e2e/spec-c.cy.js',
+    'cypress/e2e/spec-a.cy.js',
+  ])
+})