feat: randomization support [WIP REFERENCE - DO NOT MERGE]

bin/_mocha

@@ -105,7 +105,9 @@ program
   .option('--trace-deprecation', 'show stack traces on deprecations')
   .option('--use_strict', 'enforce strict mode')
   .option('--watch-extensions <ext>,...', 'additional extensions to monitor with --watch', list, [])
-  .option('--delay', 'wait for async suite definition');
+  .option('--delay', 'wait for async suite definition')
+  .option('--generate-seed', 'generate a random seed and exit')
+  .option('--random <seed>', 'randomize order of tests (with required random seed)');
 
 program._name = 'mocha';
 
@@ -186,6 +188,11 @@ if (!process.env.LOADED_MOCHA_OPTS) {
 
 program.parse(process.argv);
 
+if (program.generateSeed) {
+  console.log(Mocha.seed());
+  process.exit(0);
+}
+
 // infinite stack traces
 
 Error.stackTraceLimit = Infinity; // TODO: config
@@ -275,6 +282,13 @@ if (program.fgrep) {
   mocha.fgrep(program.fgrep);
 }
 
+// --randomize
+
+if (program.randomize) {
+  var matches = program.randomize.split(':');
+  mocha.randomize(matches[0], matches[1]);
+}
+
 // --invert
 
 if (program.invert) {

browser-entry.js

@@ -171,6 +171,11 @@ mocha.run = function(fn) {
 
 Mocha.process = process;
 
+/**
+ * Expose Mocha.seed() for browser seed generation.
+ */
+mocha.seed = Mocha.seed;
+
 /**
  * Expose mocha.
  */

lib/interfaces/bdd.js

@@ -59,16 +59,33 @@ module.exports = function(suite) {
     };
 
     /**
-     * Exclusive suite.
+     * Run tests "in order"; do not randomize.
      */
+    context.describe.inOrder = function func(title, fn) {
+      var suite = context.describe(title, fn);
+      suite.randomized(false);
+      return suite;
+    };
+    // these three are disabled anyway, so don't actually implement inOrder
+    context.describe.inOrder.skip = context.describe.skip.inOrder
+      = context.xdescribe.inOrder = context.describe.skip;
 
+    /**
+     * Exclusive suite.
+     */
     context.describe.only = function(title, fn) {
       return common.suite.only({
         title: title,
         file: file,
         fn: fn
       });
     };
+    context.describe.only.inOrder = context.describe.inOrder.only
+      = function(title, fn) {
+        var suite = context.describe.inOrder(title, fn);
+        mocha.grep(suite.fullTitle());
+        return suite;
+      };
 
     /**
      * Describe a specification or test-case

lib/interfaces/qunit.js

@@ -57,7 +57,6 @@ module.exports = function(suite) {
     /**
      * Exclusive Suite.
      */
-
     context.suite.only = function(title) {
       if (suites.length > 1) {
         suites.shift();
@@ -68,6 +67,30 @@ module.exports = function(suite) {
       });
     };
 
+    /**
+     * Do not randomize.
+     */
+    context.suite.inOrder = function(title) {
+      var suite = common.suite.create({
+        title: title,
+        file: file
+      });
+      suite.enableRandomize(false);
+      return suite;
+    };
+
+    /**
+     * Do not randomize + exclusive
+     */
+    context.suite.only.inOrder = context.suite.inOrder.only = function(title) {
+      var suite = common.suite.only({
+        title: title,
+        file: file
+      });
+      suite.enableRandomize(false);
+      return suite;
+    };
+
     /**
      * Describe a specification or test-case
      * with the given `title` and callback `fn`

lib/interfaces/tdd.js

@@ -64,6 +64,16 @@ module.exports = function(suite) {
       });
     };
 
+    /**
+     * Do not randomize.
+     */
+    context.suite.inOrder = function(title, fn) {
+      var suite = context.suite(title, fn);
+      suite.randomized(false);
+      return suite;
+    };
+    context.suite.inOrder.skip = context.suite.skip;
+
     /**
      * Exclusive test-case.
      */
@@ -74,6 +84,11 @@ module.exports = function(suite) {
         fn: fn
       });
     };
+    context.suite.only.inOrder = context.suite.inOrder.only
+      = function(title, fn) {
+        var suite = context.suite.inOrder(title, fn);
+        mocha.grep(suite.fullTitle());
+      };
 
     /**
      * Describe a specification or test-case with the given `title` and

lib/mocha.js

@@ -12,6 +12,7 @@ var escapeRe = require('escape-string-regexp');
 var path = require('path');
 var reporters = require('./reporters');
 var utils = require('./utils');
+var Random = require('random-js');
 
 /**
  * Expose `Mocha`.
@@ -68,6 +69,7 @@ function image(name) {
  *   - `ignoreLeaks` ignore global leaks
  *   - `fullTrace` display the full stack-trace on failing
  *   - `grep` string or regexp to filter tests with
+ *   - `random` for randomization of tests w/ optional seed
  *
  * @param {Object} options
  * @api public
@@ -99,6 +101,9 @@ function Mocha(options) {
   if (options.slow) {
     this.slow(options.slow);
   }
+  if (options.random) {
+    this.randomize(options.random);
+  }
 }
 
 /**
@@ -301,6 +306,53 @@ Mocha.prototype.ignoreLeaks = function(ignore) {
   return this;
 };
 
+/**
+ * Enable or disable randomization of test execution order within suites.
+ *
+ * @param {(boolean|number|string)} seed Random seed.  `seed` must be
+ *   a 32-bit unsigned integer, or a string which can convert to one.
+ *   If `false`, randomization will be globally disabled (if it was enabled
+ *   previously).
+ * @return {Mocha}
+ * @api public
+ */
+Mocha.prototype.randomize = function randomize(seed) {
+  if (!arguments.length) {
+    throw new Error('Generate a random seed using --generate-seed on the '
+      + 'command line, or Mocha.seed() in the browser.');
+  }
+  if (seed !== false) {
+    var validSeed = utils.toSafeUnsignedInteger(seed);
+    if (isNaN(validSeed)) {
+      throw new Error('Invalid random seed. Expected unsigned 32-bit '
+        + 'integer or value which can convert.');
+    }
+
+    var engine = Random.engines.mt19937().seed(validSeed);
+    this.options.randomConfig = {
+      shuffleTests: function shuffleTests(tests) {
+        Random.shuffle(engine, tests);
+        return tests;
+      },
+      seed: validSeed,
+      hex: '0x' + Number(validSeed).toString(16).toUpperCase() // for display
+    };
+  } else {
+    delete this.options.randomConfig;
+  }
+  return this;
+};
+
+/**
+ * Generate a random seed.
+ * @returns {string} Hexadecimal string
+ */
+Mocha.seed = function seed() {
+  var engine = Random.engines.mt19937()
+    .autoSeed();
+  return utils.toHex(Random.integer(0, utils.MAX_SAFE_INTEGER)(engine));
+};
+
 /**
  * Enable global leak checking.
  *
@@ -506,6 +558,10 @@ Mocha.prototype.run = function(fn) {
   if (options.useColors !== undefined) {
     exports.reporters.Base.useColors = options.useColors;
   }
+  if (options.randomConfig) {
+    runner.randomConfig = options.randomConfig;
+  }
+
   exports.reporters.Base.inlineDiffs = options.useInlineDiffs;
 
   function done(failures) {

lib/runner.js

@@ -79,6 +79,7 @@ function Runner(suite, delay) {
   this._defaultGrep = /.*/;
   this.grep(this._defaultGrep);
   this.globals(this.globalProps().concat(extraGlobals()));
+  this.random = false;
 }
 
 /**
@@ -212,6 +213,19 @@ Runner.prototype.checkGlobals = function(test) {
   }
 };
 
+/**
+ * Get an array of a Suite's Tests, potentially in random order.
+ * @returns {Array.<Test>} Array of tests (a copy)
+ * @api private
+ */
+Runner.prototype.suiteTests = function suiteTests(suite) {
+  var tests = suite.tests.slice();
+  if (this.randomConfig && suite.randomized()) {
+    return this.randomConfig.shuffleTests(tests);
+  }
+  return tests;
+};
+
 /**
  * Fail the given `test`.
  *
@@ -447,7 +461,7 @@ Runner.prototype.runTest = function(fn) {
  */
 Runner.prototype.runTests = function(suite, fn) {
   var self = this;
-  var tests = suite.tests.slice();
+  var tests = self.suiteTests(suite);
   var test;
 
   function hookErr(_, errSuite, after) {

lib/suite.js

@@ -61,6 +61,9 @@ function Suite(title, parentContext) {
   this._slow = 75;
   this._bail = false;
   this._retries = -1;
+  // this will cause randomization to occur by default if enabled
+  // globally.
+  this._randomized = true;
   this._onlyTests = [];
   this._onlySuites = [];
   this.delayed = false;
@@ -86,6 +89,7 @@ Suite.prototype.clone = function() {
   suite.enableTimeouts(this.enableTimeouts());
   suite.slow(this.slow());
   suite.bail(this.bail());
+  suite.randomized(this.randomized());
   return suite;
 };
 
@@ -187,6 +191,25 @@ Suite.prototype.isPending = function() {
   return this.pending || (this.parent && this.parent.isPending());
 };
 
+/**
+ * If randomization is enabled, calling this with a falsy value
+ * will cause the Suite's tests to NOT be randomized.
+ *
+ * This cannot be used in the context of a test, since it will already be
+ * randomized (or not) by then.
+ *
+ * @param {boolean} enabled
+ * @return {Suite|Boolean} self or enabled
+ * @api private
+ */
+Suite.prototype.randomized = function randomized(enabled) {
+  if (!arguments.length) {
+    return this._randomized;
+  }
+  this._randomized = Boolean(enabled);
+  return this;
+};
+
 /**
  * Run `fn(test[, done])` before running tests.
  *