core(fr): add base config

lighthouse-core/config/config-helpers.js

@@ -6,10 +6,16 @@
 'use strict';
 
 const path = require('path');
+const isDeepEqual = require('lodash.isequal');
+const constants = require('./constants.js');
+const Budget = require('./budget.js');
 const Audit = require('../audits/audit.js');
 const Runner = require('../runner.js');
 const i18n = require('../lib/i18n/i18n.js');
 
+/** @typedef {typeof import('../gather/gatherers/gatherer.js')} GathererConstructor */
+/** @typedef {InstanceType<GathererConstructor>} Gatherer */
+
 /**
  * If any items with identical `path` properties are found in the input array,
  * merge their `options` properties into the first instance and then discard any
@@ -35,6 +41,77 @@ const mergeOptionsOfItems = function(items) {
   return mergedItems;
 };
 
+/**
+ * Recursively merges config fragment objects in a somewhat Lighthouse-specific way.
+ *
+ *    - `null` is treated similarly to `undefined` for whether a value should be overridden.
+ *    - `overwriteArrays` controls array extension behavior:
+ *        - true: Arrays are overwritten without any merging or concatenation.
+ *        - false: Arrays are concatenated and de-duped by isDeepEqual.
+ *    - Objects are recursively merged.
+ *    - If the `settings` key is encountered while traversing an object, its arrays are *always*
+ *      overridden, not concatenated. (`overwriteArrays` is flipped to `true`)
+ *
+ * More widely typed than exposed merge() function, below.
+ * @param {Object<string, any>|Array<any>|undefined|null} base
+ * @param {Object<string, any>|Array<any>} extension
+ * @param {boolean=} overwriteArrays
+ */
+function _mergeConfigFragment(base, extension, overwriteArrays = false) {
+  // If the default value doesn't exist or is explicitly null, defer to the extending value
+  if (typeof base === 'undefined' || base === null) {
+    return extension;
+  } else if (typeof extension === 'undefined') {
+    return base;
+  } else if (Array.isArray(extension)) {
+    if (overwriteArrays) return extension;
+    if (!Array.isArray(base)) throw new TypeError(`Expected array but got ${typeof base}`);
+    const merged = base.slice();
+    extension.forEach(item => {
+      if (!merged.some(candidate => isDeepEqual(candidate, item))) merged.push(item);
+    });
+
+    return merged;
+  } else if (typeof extension === 'object') {
+    if (typeof base !== 'object') throw new TypeError(`Expected object but got ${typeof base}`);
+    if (Array.isArray(base)) throw new TypeError('Expected object but got Array');
+    Object.keys(extension).forEach(key => {
+      const localOverwriteArrays = overwriteArrays ||
+        (key === 'settings' && typeof base[key] === 'object');
+      base[key] = _mergeConfigFragment(base[key], extension[key], localOverwriteArrays);
+    });
+    return base;
+  }
+
+  return extension;
+}
+
+/**
+ * Until support of jsdoc templates with constraints, type in config.d.ts.
+ * See /~https://github.com/Microsoft/TypeScript/issues/24283
+ * @type {LH.Config.Merge}
+ */
+const mergeConfigFragment = _mergeConfigFragment;
+
+/**
+ * Validate the settings after they've been built
+ * @param {LH.Config.Settings} settings
+ */
+function assertValidSettings(settings) {
+  if (!settings.formFactor) {
+    throw new Error(`\`settings.formFactor\` must be defined as 'mobile' or 'desktop'. See /~https://github.com/GoogleChrome/lighthouse/blob/master/docs/emulation.md`);
+  }
+
+  if (!settings.screenEmulation.disabled) {
+    // formFactor doesn't control emulation. So we don't want a mismatch:
+    //   Bad mismatch A: user wants mobile emulation but scoring is configured for desktop
+    //   Bad mismtach B: user wants everything desktop and set formFactor, but accidentally not screenEmulation
+    if (settings.screenEmulation.mobile !== (settings.formFactor === 'mobile')) {
+      throw new Error(`Screen emulation mobile setting (${settings.screenEmulation.mobile}) does not match formFactor setting (${settings.formFactor}). See /~https://github.com/GoogleChrome/lighthouse/blob/master/docs/emulation.md`);
+    }
+  }
+}
+
 /**
  * Throws an error if the provided object does not implement the required properties of an audit
  * definition.
@@ -83,35 +160,206 @@ function assertValidAudit(auditDefinition) {
   }
 }
 
+/**
+ * Expands a gatherer from user-specified to an internal gatherer definition format.
+ *
+ * Input Examples:
+ *  - 'my-gatherer'
+ *  - class MyGatherer extends Gatherer { }
+ *  - {instance: myGathererInstance}
+ *
+ * @param {LH.Config.GathererJson} gatherer
+ * @return {{instance?: Gatherer, implementation?: GathererConstructor, path?: string}} passes
+ */
+function expandGathererShorthand(gatherer) {
+  if (typeof gatherer === 'string') {
+    // just 'path/to/gatherer'
+    return {path: gatherer};
+  } else if ('implementation' in gatherer || 'instance' in gatherer) {
+    // {implementation: GathererConstructor, ...} or {instance: GathererInstance, ...}
+    return gatherer;
+  } else if ('path' in gatherer) {
+    // {path: 'path/to/gatherer', ...}
+    if (typeof gatherer.path !== 'string') {
+      throw new Error('Invalid Gatherer type ' + JSON.stringify(gatherer));
+    }
+    return gatherer;
+  } else if (typeof gatherer === 'function') {
+    // just GathererConstructor
+    return {implementation: gatherer};
+  } else if (gatherer && typeof gatherer.beforePass === 'function') {
+    // just GathererInstance
+    return {instance: gatherer};
+  } else {
+    throw new Error('Invalid Gatherer type ' + JSON.stringify(gatherer));
+  }
+}
+
 /**
  * Expands the audits from user-specified JSON to an internal audit definition format.
- * @param {LH.Config.Json['audits']} audits
- * @return {?Array<{id?: string, path: string, options?: {}} | {id?: string, implementation: typeof Audit, path?: string, options?: {}}>}
+ * @param {LH.Config.AuditJson} audit
+ * @return {{id?: string, path: string, options?: {}} | {id?: string, implementation: typeof Audit, path?: string, options?: {}}}
  */
-function expandAuditShorthand(audits) {
-  if (!audits) {
-    return null;
+function expandAuditShorthand(audit) {
+  if (typeof audit === 'string') {
+    // just 'path/to/audit'
+    return {path: audit, options: {}};
+  } else if ('implementation' in audit && typeof audit.implementation.audit === 'function') {
+    // {implementation: AuditClass, ...}
+    return audit;
+  } else if ('path' in audit && typeof audit.path === 'string') {
+    // {path: 'path/to/audit', ...}
+    return audit;
+  } else if ('audit' in audit && typeof audit.audit === 'function') {
+    // just AuditClass
+    return {implementation: audit, options: {}};
+  } else {
+    throw new Error('Invalid Audit type ' + JSON.stringify(audit));
   }
+}
+
+/**
+ * @param {string} gathererPath
+ * @param {Array<string>} coreGathererList
+ * @param {string=} configDir
+ * @return {LH.Config.GathererDefn}
+ */
+function requireGatherer(gathererPath, coreGathererList, configDir) {
+  const coreGatherer = coreGathererList.find(a => a === `${gathererPath}.js`);
 
-  const newAudits = audits.map(audit => {
-    if (typeof audit === 'string') {
-      // just 'path/to/audit'
-      return {path: audit, options: {}};
-    } else if ('implementation' in audit && typeof audit.implementation.audit === 'function') {
-      // {implementation: AuditClass, ...}
-      return audit;
-    } else if ('path' in audit && typeof audit.path === 'string') {
-      // {path: 'path/to/audit', ...}
-      return audit;
-    } else if ('audit' in audit && typeof audit.audit === 'function') {
-      // just AuditClass
-      return {implementation: audit, options: {}};
+  let requirePath = `../gather/gatherers/${gathererPath}`;
+  if (!coreGatherer) {
+    // Otherwise, attempt to find it elsewhere. This throws if not found.
+    requirePath = resolveModule(gathererPath, configDir, 'gatherer');
+  }
+
+  const GathererClass = /** @type {GathererConstructor} */ (require(requirePath));
+
+  return {
+    instance: new GathererClass(),
+    implementation: GathererClass,
+    path: gathererPath,
+  };
+}
+
+/**
+ *
+ * @param {string} auditPath
+ * @param {Array<string>} coreAuditList
+ * @param {string=} configDir
+ * @return {LH.Config.AuditDefn['implementation']}
+ */
+function requireAudit(auditPath, coreAuditList, configDir) {
+// See if the audit is a Lighthouse core audit.
+  const auditPathJs = `${auditPath}.js`;
+  const coreAudit = coreAuditList.find(a => a === auditPathJs);
+  let requirePath = `../audits/${auditPath}`;
+  if (!coreAudit) {
+  // TODO: refactor and delete `global.isDevtools`.
+    if (global.isDevtools || global.isLightrider) {
+    // This is for pubads bundling.
+      requirePath = auditPath;
     } else {
-      throw new Error('Invalid Audit type ' + JSON.stringify(audit));
+    // Otherwise, attempt to find it elsewhere. This throws if not found.
+      const absolutePath = resolveModule(auditPath, configDir, 'audit');
+      // Use a relative path so bundler can easily expose it.
+      requirePath = path.relative(__dirname, absolutePath);
     }
-  });
+  }
 
-  return newAudits;
+  return require(requirePath);
+}
+
+/**
+ * Creates a settings object from potential flags object by dropping all the properties
+ * that don't exist on Config.Settings.
+ * @param {Partial<LH.Flags>=} flags
+ * @return {RecursivePartial<LH.Config.Settings>}
+*/
+function cleanFlagsForSettings(flags = {}) {
+  /** @type {RecursivePartial<LH.Config.Settings>} */
+  const settings = {};
+
+  for (const key of Object.keys(flags)) {
+    if (key in constants.defaultSettings) {
+      // @ts-expect-error tsc can't yet express that key is only a single type in each iteration, not a union of types.
+      settings[key] = flags[key];
+    }
+  }
+
+  return settings;
+}
+
+/**
+ * @param {LH.SharedFlagsSettings} settingsJson
+ * @param {LH.Flags|undefined} overrides
+ * @return {LH.Config.Settings}
+ */
+function resolveSettings(settingsJson = {}, overrides = undefined) {
+  // If a locale is requested in flags or settings, use it. A typical CLI run will not have one,
+  // however `lookupLocale` will always determine which of our supported locales to use (falling
+  // back if necessary).
+  const locale = i18n.lookupLocale((overrides && overrides.locale) || settingsJson.locale);
+
+  // Fill in missing settings with defaults
+  const {defaultSettings} = constants;
+  const settingWithDefaults = mergeConfigFragment(deepClone(defaultSettings), settingsJson, true);
+
+  // Override any applicable settings with CLI flags
+  const settingsWithFlags = mergeConfigFragment(
+    settingWithDefaults,
+    cleanFlagsForSettings(overrides),
+    true
+  );
+
+  if (settingsWithFlags.budgets) {
+    settingsWithFlags.budgets = Budget.initializeBudget(settingsWithFlags.budgets);
+  }
+  // Locale is special and comes only from flags/settings/lookupLocale.
+  settingsWithFlags.locale = locale;
+
+  // Default constants uses the mobile UA. Explicitly stating to true asks LH to use the associated UA.
+  // It's a little awkward, but the alternatives are not allowing `true` or a dedicated `disableUAEmulation` setting.
+  if (settingsWithFlags.emulatedUserAgent === true) {
+    settingsWithFlags.emulatedUserAgent = constants.userAgents[settingsWithFlags.formFactor];
+  }
+
+  assertValidSettings(settingsWithFlags);
+  return settingsWithFlags;
+}
+
+
+/**
+ * Turns a GathererJson into a GathererDefn which involves a few main steps:
+ *    - Expanding the JSON shorthand the full definition format.
+ *    - `require`ing in the implementation.
+ *    - Creating a gatherer instance from the implementation.
+ * @param {LH.Config.GathererJson} gathererJson
+ * @param {Array<string>} coreGathererList
+ * @param {string=} configDir
+ * @return {LH.Config.GathererDefn}
+ */
+function resolveGathererToDefn(gathererJson, coreGathererList, configDir) {
+  const gathererDefn = expandGathererShorthand(gathererJson);
+  if (gathererDefn.instance) {
+    return {
+      instance: gathererDefn.instance,
+      implementation: gathererDefn.implementation,
+      path: gathererDefn.path,
+    };
+  } else if (gathererDefn.implementation) {
+    const GathererClass = gathererDefn.implementation;
+    return {
+      instance: new GathererClass(),
+      implementation: gathererDefn.implementation,
+      path: gathererDefn.path,
+    };
+  } else if (gathererDefn.path) {
+    const path = gathererDefn.path;
+    return requireGatherer(path, coreGathererList, configDir);
+  } else {
+    throw new Error('Invalid expanded Gatherer: ' + JSON.stringify(gathererDefn));
+  }
 }
 
 /**
@@ -122,41 +370,25 @@ function expandAuditShorthand(audits) {
  * @param {string=} configDir
  * @return {Array<LH.Config.AuditDefn>|null}
  */
-function requireAudits(audits, configDir) {
-  const expandedAudits = expandAuditShorthand(audits);
-  if (!expandedAudits) {
+function resolveAuditsToDefns(audits, configDir) {
+  if (!audits) {
     return null;
   }
 
   const coreList = Runner.getAuditList();
-  const auditDefns = expandedAudits.map(audit => {
+  const auditDefns = audits.map(auditJson => {
+    const auditDefn = expandAuditShorthand(auditJson);
     let implementation;
-    if ('implementation' in audit) {
-      implementation = audit.implementation;
+    if ('implementation' in auditDefn) {
+      implementation = auditDefn.implementation;
     } else {
-      // See if the audit is a Lighthouse core audit.
-      const auditPathJs = `${audit.path}.js`;
-      const coreAudit = coreList.find(a => a === auditPathJs);
-      let requirePath = `../audits/${audit.path}`;
-      if (!coreAudit) {
-        // TODO: refactor and delete `global.isDevtools`.
-        if (global.isDevtools || global.isLightrider) {
-          // This is for pubads bundling.
-          requirePath = audit.path;
-        } else {
-          // Otherwise, attempt to find it elsewhere. This throws if not found.
-          const absolutePath = resolveModule(audit.path, configDir, 'audit');
-          // Use a relative path so bundler can easily expose it.
-          requirePath = path.relative(__dirname, absolutePath);
-        }
-      }
-      implementation = /** @type {typeof Audit} */ (require(requirePath));
+      implementation = requireAudit(auditDefn.path, coreList, configDir);
     }
 
     return {
       implementation,
-      path: audit.path,
-      options: audit.options || {},
+      path: auditDefn.path,
+      options: auditDefn.options || {},
     };
   });
 
@@ -325,6 +557,9 @@ module.exports = {
   deepClone,
   deepCloneConfigJson,
   mergeOptionsOfItems,
-  requireAudits,
+  mergeConfigFragment,
+  resolveSettings,
+  resolveGathererToDefn,
+  resolveAuditsToDefns,
   resolveModule,
 };