testFramework is undefined in setup() when using ava

[vladistan]

Issue description or question

This is a revisit of the previously closed issue #1078

I am working on a largish node application that uses AVA for testing framework, and a mix of native AVA assertions and chai assertions.

So here is my problem. Because of the mixed use of the assertions libraries, AVA is configured to not fail the tests that do not call native AVA assertion, but use chai's instead.

The configuration uses 'ava' section in package.json file.

Unfortunately when tests are run in Wallaby, it ignores this config and tests are failing for no reason.

Wallaby diagnostics report

{ editorVersion: 'WebStorm 2019.2.3',
  pluginVersion: '1.0.143',
  editorType: 'IntelliJ',
  osVersion: 'darwin 18.6.0',
  nodeVersion: 'v11.11.0',
  coreVersion: '1.0.767',
  config:
   { files: [ { pattern: 'src/**/*.js', ignore: false, trigger: true, load: true, instrument: true, order: 1 } ],
     tests: [ { pattern: 'test/**/test_*.js', ignore: false, trigger: true, load: true, test: true, order: 2 } ],
     env: { type: 'node', runner: 'node', params: {}, viewportSize: { width: 800, height: 600 }, options: { width: 800, height: 600 }, bundle: true },
     debug: true,
     trace: true,
     testFramework: { version: 'ava@0.3.0', configurator: 'ava@0.3.0', reporter: 'ava@0.3.0', starter: 'ava@0.3.0', jsPreprocessor: [Function: jsPreprocessor] },
     compilers: { '**/*.js': [Function], '**/*.?(lit)coffee?(.md)': [Function] },
     diagnostics: {},
     filesWithNoCoverageCalculated: [],
     runAllTestsInAffectedTestFile: false,
     preprocessors: {},
     maxConsoleMessagesPerTest: 100,
     autoConsoleLog: true,
     delays: { run: 0, edit: 100, update: 0 },
     workers: { initial: 0, regular: 0, recycle: false },
     teardown: undefined,
     hints:
      { ignoreCoverage: '__REGEXP /ignore coverage|istanbul ignore/',
        ignoreCoverageForFile: '__REGEXP /ignore file coverage/',
        commentAutoLog: '?',
        testFileSelection: { include: '__REGEXP /file\\.only/', exclude: '__REGEXP /file\\.skip/' } },
     automaticTestFileSelection: true,
     runSelectedTestsOnly: false,
     extensions: {},
     reportUnhandledPromises: true,
     slowTestThreshold: 75,
     lowCoverageThreshold: 80,
     loose: undefined,
     configCode:
      "const fs = require('fs');\nconst path = require('path');\nconst babelConfig = JSON.parse(fs.readFileSync(path.join(__dirname, '.babelrc')));\n\nmodule.exports = function(wallaby) {\n  return {\n    files: [\n      'src/**/*.js',\n      //  { pattern: 'src/**/*.js', instrument: true, load: true, ignore: false }\n    ],\n    tests: [\n      'test/**/test_*.js',\n    ],\n    env: {\n      type: 'node',\n      runner: 'node',\n    },\n    debug: true,\n    trace: true,\n    testFramework: 'ava',\n    compilers: {\n      '**/*.js': wallaby.compilers.babel(babelConfig),\n    },\n    setup: function(wallaby) {\n      // wallaby.testFramework is jasmine/QUnit/mocha object\n\n      wallaby.delayStart();\n      setTimeout(function() {\n        console.log('HELLO:::');\n        console.log(wallaby);\n\n        wallaby.start();\n      }, 300);\n    },\n  };\n};\n" },
  packageJSON:
   { dependencies:
      { '@babel/runtime': '^7.0.0', 'aws-sdk': '^2.528.0', bluebird: '^3.5.1', faker: '^4.1.0', request: '^2.88.0', 'request-promise': '^4.2.2', 'source-map-support': '^0.5.4', util: '^0.11.1', uuid: '^3.3.3' },
     devDependencies:
      { '@babel/cli': '^7.1.2',
        '@babel/core': '^7.1.2',
        '@babel/plugin-proposal-class-properties': '^7.0.0',
        '@babel/plugin-proposal-decorators': '^7.0.0',
        '@babel/plugin-transform-regenerator': '^7.0.0',
        '@babel/plugin-transform-runtime': '^7.2.0',
        '@babel/polyfill': '^7.2.5',
        '@babel/preset-env': '^7.0.0',
        '@babel/register': '^7.0.0',
        acorn: '^6.1.0',
        ava: '^1.4.1',
        'babel-loader': '^8.0.5',
        'babel-plugin-source-map-support': '^2.0.1',
        chai: '^4.2.0',
        chalk: '^2.4.1',
        'es6-plato': '^1.0.18',
        eslint: '^5.6.1',
        'eslint-config-google': '^0.11.0',
        'eslint-friendly-formatter': '^4.0.1',
        'eslint-loader': '^2.1.1',
        'eslint-plugin-import': '^2.14.0',
        nyc: '^14.0.0',
        serverless: '^1.43.0',
        'serverless-domain-manager': '^2.6.13',
        'serverless-offline': '^3.20.1',
        'serverless-webpack': '^5.3.0',
        webpack: '^4.2.0',
        'webpack-node-externals': '^1.7.2' } },
  fs: { numberOfFiles: 16 },
  debug:
   [ '2019-10-21T03:27:52.218Z project Wallaby Node version: v11.11.0\n',
     '2019-10-21T03:27:52.219Z project Wallaby config: <homeDir>/Proj/Persyst/PMC/app/sls-api/wallaby.js\n',
     '2019-10-21T03:27:52.765Z project File cache: <homeDir>/Library/Caches/WebStorm2019.2/wallaby/projects/8df6306567430ae6\n',
     '2019-10-21T03:27:52.776Z uiService Listening port 51235\n',
     '2019-10-21T03:27:52.783Z workers Parallelism for initial run: 10, for regular run: 5\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #0\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #1\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #2\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #3\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #4\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #5\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #6\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #7\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #8\n',
     '2019-10-21T03:27:52.783Z workers Starting run worker instance #9\n',
     '2019-10-21T03:27:52.784Z workers Web server is listening at 64270\n',
     '2019-10-21T03:27:52.921Z project Stopping process pool\n',
     '2019-10-21T03:27:52.921Z project File cache is up-to-date, starting full test run\n',
     '2019-10-21T03:27:52.929Z project Running postprocessor\n',
     '2019-10-21T03:27:52.943Z postprocessor New TypeScript language service is required\n',
     '2019-10-21T03:27:52.943Z postprocessor TypeScript postprocessor will be removed because in 16 project files none were found by patterns: **/*.ts?(x)\n',
     '2019-10-21T03:27:52.944Z project Postprocessor execution finished\n',
     '2019-10-21T03:27:52.944Z project Postprocessor is removed as requested by itself\n',
     '2019-10-21T03:27:52.944Z project Test run started; run priority: 3\n',
     '2019-10-21T03:27:52.945Z project Running all tests\n',
     '2019-10-21T03:27:52.946Z workers Starting test run, priority: 3\n',
     '2019-10-21T03:27:52.947Z workers Distributing tests between 10 workers\n',
     '2019-10-21T03:27:52.948Z workers Running tests in parallel\n',
     '2019-10-21T03:27:52.949Z nodeRunner Starting sandbox [worker #0, session #9y8x7]\n',
     '2019-10-21T03:27:52.949Z nodeRunner Starting sandbox [worker #1, session #y03so]\n',
     '2019-10-21T03:27:52.949Z nodeRunner Starting sandbox [worker #2, session #ct2ta]\n',
     '2019-10-21T03:27:52.953Z nodeRunner Starting sandbox [worker #3, session #8zqic]\n',
     '2019-10-21T03:27:52.954Z nodeRunner Starting sandbox [worker #4, session #r33c1]\n',
     '2019-10-21T03:27:52.954Z nodeRunner Preparing sandbox [worker #0, session #9y8x7]\n',
     '2019-10-21T03:27:52.954Z nodeRunner Preparing sandbox [worker #1, session #y03so]\n',
     '2019-10-21T03:27:52.955Z nodeRunner Preparing sandbox [worker #2, session #ct2ta]\n',
     '2019-10-21T03:27:52.955Z nodeRunner Preparing sandbox [worker #3, session #8zqic]\n',
     '2019-10-21T03:27:52.955Z nodeRunner Preparing sandbox [worker #4, session #r33c1]\n',
     '2019-10-21T03:27:53.305Z workers Started run worker instance (delayed) #3\n',
     '2019-10-21T03:27:53.306Z nodeRunner Prepared sandbox [worker #3, session #8zqic]\n',
     '2019-10-21T03:27:53.307Z workers [worker #3, session #8zqic] Running tests in sandbox\n',
     '2019-10-21T03:27:53.309Z workers Started run worker instance (delayed) #0\n',
     '2019-10-21T03:27:53.310Z nodeRunner Prepared sandbox [worker #0, session #9y8x7]\n',
     '2019-10-21T03:27:53.310Z workers [worker #0, session #9y8x7] Running tests in sandbox\n',
     '2019-10-21T03:27:53.328Z workers Started run worker instance (delayed) #2\n',
     '2019-10-21T03:27:53.328Z nodeRunner Prepared sandbox [worker #2, session #ct2ta]\n',
     '2019-10-21T03:27:53.328Z workers [worker #2, session #ct2ta] Running tests in sandbox\n',
     '2019-10-21T03:27:53.334Z workers Started run worker instance (delayed) #1\n',
     '2019-10-21T03:27:53.335Z nodeRunner Prepared sandbox [worker #1, session #y03so]\n',
     '2019-10-21T03:27:53.335Z workers [worker #1, session #y03so] Running tests in sandbox\n',
     '2019-10-21T03:27:53.338Z workers Started run worker instance (delayed) #5\n',
     '2019-10-21T03:27:53.339Z workers Started run worker instance (delayed) #4\n',
     '2019-10-21T03:27:53.339Z nodeRunner Prepared sandbox [worker #4, session #r33c1]\n',
     '2019-10-21T03:27:53.339Z workers [worker #4, session #r33c1] Running tests in sandbox\n',
     '2019-10-21T03:27:53.362Z workers Started run worker instance (delayed) #6\n',
     '2019-10-21T03:27:53.381Z workers Started run worker instance (delayed) #7\n',
     '2019-10-21T03:27:53.388Z workers Started run worker instance (delayed) #8\n',
     '2019-10-21T03:27:53.404Z workers Started run worker instance (delayed) #9\n',
     '2019-10-21T03:27:53.710Z workers [r33c1] Loaded unknown number of test(s)\n',
     '2019-10-21T03:27:53.711Z workers [r33c1] Test executed: Create folder from UID\n',
     '2019-10-21T03:27:53.716Z workers [r33c1] Run 1 test(s), skipped 0 test(s)\n',
     '2019-10-21T03:27:53.719Z workers [r33c1] Sandbox is responsive, closing it\n',
     '2019-10-21T03:27:53.810Z workers [8zqic] Loaded unknown number of test(s)\n',
     '2019-10-21T03:27:53.810Z workers [8zqic] Test executed: missing headers\n',
     '2019-10-21T03:27:53.810Z workers [8zqic] Test executed: missing auth header\n',
     '2019-10-21T03:27:53.811Z workers [8zqic] Test executed: malformed auth header\n',
     '2019-10-21T03:27:53.811Z workers [8zqic] Test executed: non-bearer token\n',
     '2019-10-21T03:27:53.811Z workers [ct2ta] Loaded unknown number of test(s)\n',
     '2019-10-21T03:27:53.812Z workers [8zqic] Test executed: mYSpAcEcASe schema\n',
     "2019-10-21T03:27:53.812Z workers Sandbox (active) [ct2ta] error: Cannot find module './util'\nRequire stack:\n- 

[smcenlly]

We have updated Wallaby to allow you to set test runner options for Ava. The testFramework for Ava now provides an empty object { } (was previously undefined). You may set ava's options by setting the properties of the testFramework. Please update to the latest version of Wallaby core (v1.0.768) that includes this change. If Wallaby does not update automatically, you may force a core update.

Please note that the following properties will always be overridden by Wallaby: file, projectDir, updateSnapshots.

You may set failWithoutAssertions to false by modifying the testFramework in the setup function of your Wallaby.js configuration file (see example below).

module.exports = function (wallaby) {
  return {
...
    testFramework: 'ava',
    setup: (wallaby) => {
      wallaby.testFramework.failWithoutAssertions = false;
    }
...
  };
};

[vladistan]

Thanks a lot