From 54d06b9faa8e392d530185289a985a1212f6564f Mon Sep 17 00:00:00 2001 From: Thijs Louisse Date: Thu, 25 Jun 2020 18:50:50 +0200 Subject: [PATCH 1/2] feat(providence-analytics): filepath matching based on anymatch --- packages/providence-analytics/package.json | 4 +- .../src/cli/cli-helpers.js | 41 +-- packages/providence-analytics/src/cli/cli.js | 299 ++++++++++++++++++ .../src/cli/generate-extend-docs-data.js | 1 - .../providence-analytics/src/cli/index.js | 273 +--------------- .../src/program/analyzers/find-exports.js | 17 +- .../helpers/track-down-identifier.js | 8 +- .../src/program/analyzers/match-subclasses.js | 3 - .../src/program/services/InputDataService.js | 114 +++++-- .../src/program/types/index.js | 12 +- .../src/program/utils/memoize.js | 8 +- .../test-helpers/mock-project-helpers.js | 3 +- .../test-node/cli/cli.test.js | 172 ++++++++++ .../test-node/cli/cli.testx.js | 98 ------ .../analyzers/match-subclasses.test.js | 8 +- .../program/services/InputDataService.test.js | 142 ++++++++- yarn.lock | 166 ++++++---- 17 files changed, 847 insertions(+), 522 deletions(-) create mode 100755 packages/providence-analytics/src/cli/cli.js create mode 100644 packages/providence-analytics/test-node/cli/cli.test.js delete mode 100644 packages/providence-analytics/test-node/cli/cli.testx.js diff --git a/packages/providence-analytics/package.json b/packages/providence-analytics/package.json index 4045a52e6..288eb1f67 100644 --- a/packages/providence-analytics/package.json +++ b/packages/providence-analytics/package.json @@ -21,7 +21,7 @@ "scripts": { "dashboard": "node ./dashboard/src/server.js", "providence": "node --max-old-space-size=8192 ./src/cli/index.js", - "test:node": "mocha './test-node/program/**/*.test.js'", + "test:node": "mocha './test-node/**/*.test.js'", "test:node:e2e": "mocha './test-node/program/**/*.e2e.js' --timeout 60000", "test:node:watch": "yarn test:node --watch" }, @@ -34,6 +34,7 @@ "@babel/types": "^7.9.0", "@rollup/plugin-node-resolve": "^7.1.1", "@typescript-eslint/typescript-estree": "^2.0.0", + "anymatch": "^3.1.1", "chalk": "^2.4.2", "commander": "^2.20.0", "deepmerge": "^4.0.0", @@ -42,6 +43,7 @@ "glob": "^7.1.6", "htm": "^3.0.3", "inquirer": "^7.0.0", + "is-negated-glob": "^1.0.0", "lit-element": "^2.2.1", "ora": "^3.4.0", "parse5": "^5.1.1", diff --git a/packages/providence-analytics/src/cli/cli-helpers.js b/packages/providence-analytics/src/cli/cli-helpers.js index 1c5453a06..4a075600e 100644 --- a/packages/providence-analytics/src/cli/cli-helpers.js +++ b/packages/providence-analytics/src/cli/cli-helpers.js @@ -7,6 +7,10 @@ const { InputDataService } = require('../program/services/InputDataService.js'); const { LogService } = require('../program/services/LogService.js'); const { aForEach } = require('../program/utils/async-array-utils.js'); +function flatten(arr) { + return Array.prototype.concat.apply([], arr); +} + function csToArray(v) { return v.split(',').map(v => v.trim()); } @@ -28,22 +32,27 @@ function setQueryMethod(m) { /** * @returns {string[]} */ -function pathsArrayFromCs(t) { - return t - .split(',') - .map(t => { - const isGlob = t.includes('*'); - if (isGlob) { - return glob.sync(t); +function pathsArrayFromCs(t, cwd = process.cwd()) { + return flatten( + t.split(',').map(t => { + if (t.startsWith('/')) { + return t; } - return pathLib.resolve(process.cwd(), t.trim()); - }) - .flat(); + if (t.includes('*')) { + if (!t.endsWith('/')) { + // eslint-disable-next-line no-param-reassign + t = `${t}/`; + } + return glob.sync(t, { cwd, absolute: true }); + } + return pathLib.resolve(cwd, t.trim()); + }), + ); } /** * @param {string} name collection name found in eCfg - * @param {'search-target'|'reference'} [colType='search-targets'] collectioon type + * @param {'search-target'|'reference'} [colType='search-targets'] collection type * @param {object} eCfg external configuration. Usually providence.conf.js * @returns {string[]} */ @@ -60,21 +69,17 @@ function pathsArrayFromCollectionName(name, colType = 'search-target', eCfg) { return undefined; } -function spawnProcess(processArgStr, opts, { log } = {}) { +function spawnProcess(processArgStr, opts) { const processArgs = processArgStr.split(' '); const proc = child_process.spawn(processArgs[0], processArgs.slice(1), opts); let output; proc.stdout.on('data', data => { output += data; - if (log) { - LogService.debug(data); - } + LogService.debug(data.toString()); }); return new Promise((resolve, reject) => { proc.stderr.on('data', data => { - if (log) { - LogService.error(data); - } + LogService.error(data.toString()); reject(data.toString()); }); proc.on('close', code => { diff --git a/packages/providence-analytics/src/cli/cli.js b/packages/providence-analytics/src/cli/cli.js new file mode 100755 index 000000000..2c3c4d98d --- /dev/null +++ b/packages/providence-analytics/src/cli/cli.js @@ -0,0 +1,299 @@ +// @ts-ignore-next-line +require('../program/types/index.js'); + +const child_process = require('child_process'); // eslint-disable-line camelcase +const pathLib = require('path'); +const commander = require('commander'); +const providenceModule = require('../program/providence.js'); +const { LogService } = require('../program/services/LogService.js'); +const { QueryService } = require('../program/services/QueryService.js'); +const { InputDataService } = require('../program/services/InputDataService.js'); +const { promptAnalyzerMenu, promptAnalyzerConfigMenu } = require('./prompt-analyzer-menu.js'); +const { + extensionsFromCs, + setQueryMethod, + targetDefault, + appendProjectDependencyPaths, + installDeps, + pathsArrayFromCollectionName, + pathsArrayFromCs, +} = require('./cli-helpers.js'); +const { launchProvidenceWithExtendDocs } = require('./generate-extend-docs-data.js'); +const { version } = require('../../package.json'); + +async function cli({ cwd, addProjectDependencyPaths } = {}) { + let resolveCli; + let rejectCli; + const cliPromise = new Promise((resolve, reject) => { + resolveCli = resolve; + rejectCli = reject; + }); + + /** @type {'analyzer'|'queryString'} */ + let searchMode; + /** @type {object} */ + let analyzerOptions; + /** @type {object} */ + let featureOptions; + /** @type {object} */ + let regexSearchOptions; + + const externalConfig = InputDataService.getExternalConfig(); + + async function getQueryInputData( + /* eslint-disable no-shadow */ + searchMode, + regexSearchOptions, + featureOptions, + analyzerOptions, + /* eslint-enable no-shadow */ + showAnalyzerConfigMenu, + ) { + let queryConfig = null; + let queryMethod = null; + + if (searchMode === 'search-query') { + queryConfig = QueryService.getQueryConfigFromRegexSearchString( + regexSearchOptions.regexString, + ); + queryMethod = 'grep'; + } else if (searchMode === 'feature-query') { + queryConfig = QueryService.getQueryConfigFromFeatureString(featureOptions.queryString); + queryMethod = 'grep'; + } else if (searchMode === 'analyzer-query') { + let { name, config } = analyzerOptions; + if (!name) { + const answers = await promptAnalyzerMenu(); + name = answers.analyzerName; + } + if (showAnalyzerConfigMenu && !config) { + const answers = await promptAnalyzerConfigMenu(name, analyzerOptions.promptOptionalConfig); + config = answers.analyzerConfig; + } + // Will get metaConfig from ./providence.conf.js + const metaConfig = externalConfig ? externalConfig.metaConfig : {}; + config = { ...config, metaConfig }; + queryConfig = QueryService.getQueryConfigFromAnalyzer(name, config); + queryMethod = 'ast'; + } else { + LogService.error('Please define a feature, analyzer or search'); + process.exit(1); + } + return { queryConfig, queryMethod }; + } + + async function launchProvidence() { + const { queryConfig, queryMethod } = await getQueryInputData( + searchMode, + regexSearchOptions, + featureOptions, + analyzerOptions, + ); + + const searchTargetPaths = commander.searchTargetCollection || commander.searchTargetPaths; + let referencePaths; + if (queryConfig.analyzer.requiresReference) { + referencePaths = commander.referenceCollection || commander.referencePaths; + } + + let extendedSearchTargets; + if (addProjectDependencyPaths) { + extendedSearchTargets = await appendProjectDependencyPaths(searchTargetPaths); + } else { + extendedSearchTargets = searchTargetPaths; + } + + // TODO: filter out: + // - dependencies listed in reference (?) Or at least, inside match-imports, make sure that + // we do not test against ourselves... + // - + + providenceModule.providence(queryConfig, { + gatherFilesConfig: { + extensions: commander.extensions, + // ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), + filter: commander.whitelist, + }, + gatherFilesConfigReference: { + extensions: commander.extensions, + // ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), + filter: commander.whitelistReference, + }, + debugEnabled: commander.debug, + queryMethod, + targetProjectPaths: extendedSearchTargets, + referenceProjectPaths: referencePaths, + targetProjectRootPaths: searchTargetPaths, + writeLogFile: commander.writeLogFile, + }); + } + + async function manageSearchTargets(options) { + const basePath = pathLib.join(__dirname, '../..'); + if (options.update) { + LogService.info('git submodule update --init --recursive'); + + const updateResult = child_process.execSync('git submodule update --init --recursive', { + cwd: basePath, + }); + + LogService.info(String(updateResult)); + } + if (options.deps) { + await installDeps(commander.searchTargetPaths); + } + if (options.createVersionHistory) { + await installDeps(commander.searchTargetPaths); + } + } + + commander + .version(version, '-v, --version') + .option('-e, --extensions [extensions]', 'extensions like ".js, .html"', extensionsFromCs, [ + '.js', + '.html', + ]) + .option('-D, --debug', 'shows extensive logging') + .option( + '-t, --search-target-paths [targets]', + `path(s) to project(s) on which analysis/querying should take place. Requires + a list of comma seperated values relative to project root`, + v => pathsArrayFromCs(v, cwd), + targetDefault(), + ) + .option( + '-r, --reference-paths [references]', + `path(s) to project(s) which serve as a reference (applicable for certain analyzers like + 'match-imports'). Requires a list of comma seperated values relative to + project root (like 'node_modules/lion-based-ui, node_modules/lion-based-ui-labs').`, + v => pathsArrayFromCs(v, cwd), + InputDataService.referenceProjectPaths, + ) + .option('-w, --whitelist [whitelist]', `whitelisted paths, like './src, ./packages/*'`, v => + pathsArrayFromCs(v, cwd), + ) + .option( + '--whitelist-reference [whitelist-reference]', + `whitelisted paths for reference, like './src, ./packages/*'`, + v => pathsArrayFromCs(v, cwd), + ) + .option( + '--search-target-collection [collection-name]', + `path(s) to project(s) which serve as a reference (applicable for certain analyzers like + 'match-imports'). Should be a collection defined in providence.conf.js as paths relative to + project root.`, + v => pathsArrayFromCollectionName(v, 'search-target', externalConfig), + ) + .option( + '--reference-collection [collection-name]', + `path(s) to project(s) on which analysis/querying should take place. Should be a collection + defined in providence.conf.js as paths relative to project root.`, + v => pathsArrayFromCollectionName(v, 'reference', externalConfig), + ) + .option('--write-log-file', `Writes all logs to 'providence.log' file`); + + commander + .command('search ') + .alias('s') + .description('perfoms regex search string like "my-.*-comp"') + .action((regexString, options) => { + searchMode = 'search-query'; + regexSearchOptions = options; + regexSearchOptions.regexString = regexString; + launchProvidence() + .then(() => resolveCli()) + .catch(() => rejectCli()); + }); + + commander + .command('feature ') + .alias('f') + .description('query like "tg-icon[size=xs]"') + .option('-m, --method [method]', 'query method: "grep" or "ast"', setQueryMethod, 'grep') + .action((queryString, options) => { + searchMode = 'feature-query'; + featureOptions = options; + featureOptions.queryString = queryString; + launchProvidence() + .then(() => resolveCli()) + .catch(() => rejectCli()); + }); + + commander + .command('analyze [analyzer-name]') + .alias('a') + .description( + `predefined "query" for ast analysis. Can be a script found in program/analyzers, + like "find-imports"`, + ) + .option( + '-o, --prompt-optional-config', + `by default, only required configuration options are + asked for. When this flag is provided, optional configuration options are shown as well`, + ) + .option('-c, --config [config]', 'configuration object for analyzer', c => JSON.parse(c)) + .action((analyzerName, options) => { + searchMode = 'analyzer-query'; + analyzerOptions = options; + analyzerOptions.name = analyzerName; + launchProvidence() + .then(() => resolveCli()) + .catch(() => rejectCli()); + }); + + commander + .command('extend-docs') + .alias('e') + .description( + `Generates data for "babel-extend-docs" plugin. These data are generated by the "match-paths" + plugin, which automatically resolves import paths from reference projects + (say [@lion/input, @lion/textarea, ...etc]) to a target project (say "wolf-ui").`, + ) + .option( + '--prefix-from [prefix-from]', + `Prefix for components of reference layer. By default "lion"`, + a => a, + 'lion', + ) + .option( + '--prefix-to [prefix-to]', + `Prefix for components of reference layer. For instance "wolf"`, + ) + .option( + '--output-folder [output-folder]', + `This is the file path where the result file "providence-extend-docs-data.json" will be written to`, + p => pathLib.resolve(process.cwd(), p.trim()), + process.cwd(), + ) + .action(options => { + if (!options.prefixTo) { + LogService.error(`Please provide a "prefix to" like '--prefix-to "myprefix"'`); + process.exit(1); + } + if (!commander.referencePaths) { + LogService.error(`Please provide referencePaths path like '-r "node_modules/@lion/*"'`); + process.exit(1); + } + const prefixCfg = { from: options.prefixFrom, to: options.prefixTo }; + launchProvidenceWithExtendDocs(commander.referencePaths, prefixCfg, options.outputFolder); + }); + + commander + .command('manage-projects') + .description( + `Before running a query, be sure to have search-targets up to date (think of + npm/bower dependencies, latest version etc.)`, + ) + .option('-u, --update', 'gets latest of all search-targets and references') + .option('-d, --deps', 'installs npm/bower dependencies of search-targets') + .option('-h, --create-version-history', 'gets latest of all search-targets and references') + .action(options => { + manageSearchTargets(options); + }); + + commander.parse(process.argv); + + await cliPromise; +} + +module.exports = { cli }; diff --git a/packages/providence-analytics/src/cli/generate-extend-docs-data.js b/packages/providence-analytics/src/cli/generate-extend-docs-data.js index 974ea1078..166ac09c1 100644 --- a/packages/providence-analytics/src/cli/generate-extend-docs-data.js +++ b/packages/providence-analytics/src/cli/generate-extend-docs-data.js @@ -2,7 +2,6 @@ const fs = require('fs'); const pathLib = require('path'); const { performance } = require('perf_hooks'); - const { providence } = require('../program/providence.js'); const { QueryService } = require('../program/services/QueryService.js'); const { LogService } = require('../program/services/LogService.js'); diff --git a/packages/providence-analytics/src/cli/index.js b/packages/providence-analytics/src/cli/index.js index 461190dc6..8d26466e3 100755 --- a/packages/providence-analytics/src/cli/index.js +++ b/packages/providence-analytics/src/cli/index.js @@ -1,273 +1,4 @@ #!/usr/bin/env node +const { cli } = require('./cli.js'); -// @ts-ignore-next-line -require('../program/types/index.js'); - -const child_process = require('child_process'); // eslint-disable-line camelcase -const pathLib = require('path'); -const commander = require('commander'); -const { providence } = require('../program/providence.js'); -const { LogService } = require('../program/services/LogService.js'); -const { QueryService } = require('../program/services/QueryService.js'); -const { InputDataService } = require('../program/services/InputDataService.js'); -const { promptAnalyzerMenu, promptAnalyzerConfigMenu } = require('./prompt-analyzer-menu.js'); -const { - extensionsFromCs, - setQueryMethod, - targetDefault, - appendProjectDependencyPaths, - installDeps, - pathsArrayFromCollectionName, - pathsArrayFromCs, -} = require('./cli-helpers.js'); -const { launchProvidenceWithExtendDocs } = require('./generate-extend-docs-data.js'); - -// @ts-ignore-next-line -const { version } = require('../../package.json'); - -/** @type {'analyzer'|'queryString'} */ -let searchMode; -/** @type {object} */ -let analyzerOptions; -/** @type {object} */ -let featureOptions; -/** @type {object} */ -let regexSearchOptions; - -const externalConfig = InputDataService.getExternalConfig(); - -// eslint-disable-next-line no-shadow -async function getQueryInputData(searchMode, regexSearchOptions, featureOptions, analyzerOptions) { - let queryConfig = null; - let queryMethod = null; - - if (searchMode === 'search-query') { - queryConfig = QueryService.getQueryConfigFromRegexSearchString(regexSearchOptions.regexString); - queryMethod = 'grep'; - } else if (searchMode === 'feature-query') { - queryConfig = QueryService.getQueryConfigFromFeatureString(featureOptions.queryString); - queryMethod = 'grep'; - } else if (searchMode === 'analyzer-query') { - let { name, config } = analyzerOptions; - if (!name) { - const answers = await promptAnalyzerMenu(); - name = answers.analyzerName; - } - if (!config) { - const answers = await promptAnalyzerConfigMenu(name, analyzerOptions.promptOptionalConfig); - config = answers.analyzerConfig; - } - // Will get metaConfig from ./providence.conf.js - const metaConfig = externalConfig ? externalConfig.metaConfig : {}; - config = { ...config, metaConfig }; - queryConfig = QueryService.getQueryConfigFromAnalyzer(name, config); - queryMethod = 'ast'; - } else { - LogService.error('Please define a feature, analyzer or search'); - process.exit(1); - } - return { queryConfig, queryMethod }; -} - -async function launchProvidence() { - const { queryConfig, queryMethod } = await getQueryInputData( - searchMode, - regexSearchOptions, - featureOptions, - analyzerOptions, - ); - - const searchTargetPaths = commander.searchTargetCollection || commander.searchTargetPaths; - let referencePaths; - if (queryConfig.analyzer.requiresReference) { - referencePaths = commander.referenceCollection || commander.referencePaths; - } - - // const extendedSearchTargets = searchTargetPaths; - const extendedSearchTargets = await appendProjectDependencyPaths(searchTargetPaths); - - // TODO: filter out: - // - dependencies listed in reference (?) Or at least, inside match-imports, make sure that - // we do not test against ourselves... - // - - - providence(queryConfig, { - gatherFilesConfig: { - extensions: commander.extensions, - ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), - includePaths: commander.whitelist, - }, - gatherFilesConfigReference: { - extensions: commander.extensions, - ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), - includePaths: commander.whitelistReference, - }, - debugEnabled: commander.debug, - queryMethod, - targetProjectPaths: extendedSearchTargets, - referenceProjectPaths: referencePaths, - targetProjectRootPaths: searchTargetPaths, - writeLogFile: commander.writeLogFile, - }); -} - -async function manageSearchTargets(options) { - const basePath = pathLib.join(__dirname, '../..'); - if (options.update) { - LogService.info('git submodule update --init --recursive'); - - const updateResult = child_process.execSync('git submodule update --init --recursive', { - cwd: basePath, - }); - - LogService.info(String(updateResult)); - } - if (options.deps) { - await installDeps(commander.searchTargetPaths); - } - if (options.createVersionHistory) { - await installDeps(commander.searchTargetPaths); - } -} - -commander - .version(version, '-v, --version') - .option('-e, --extensions [extensions]', 'extensions like ".js, .html"', extensionsFromCs, [ - '.js', - '.html', - ]) - .option('-D, --debug', 'shows extensive logging') - .option( - '-t, --search-target-paths [targets]', - `path(s) to project(s) on which analysis/querying should take place. Requires - a list of comma seperated values relative to project root`, - pathsArrayFromCs, - targetDefault(), - ) - .option( - '-r, --reference-paths [references]', - `path(s) to project(s) which serve as a reference (applicable for certain analyzers like - 'match-imports'). Requires a list of comma seperated values relative to - project root (like 'node_modules/lion-based-ui, node_modules/lion-based-ui-labs').`, - pathsArrayFromCs, - InputDataService.referenceProjectPaths, - ) - .option( - '-w, --whitelist [whitelist]', - `whitelisted paths, like './src, ./packages/*'`, - pathsArrayFromCs, - ) - .option( - '--whitelist-reference [whitelist-reference]', - `whitelisted paths for reference, like './src, ./packages/*'`, - pathsArrayFromCs, - ) - .option( - '--search-target-collection [collection-name]', - `path(s) to project(s) which serve as a reference (applicable for certain analyzers like - 'match-imports'). Should be a collection defined in providence.conf.js as paths relative to - project root.`, - v => pathsArrayFromCollectionName(v, 'search-target', externalConfig), - ) - .option( - '--reference-collection [collection-name]', - `path(s) to project(s) on which analysis/querying should take place. Should be a collection - defined in providence.conf.js as paths relative to project root.`, - v => pathsArrayFromCollectionName(v, 'reference', externalConfig), - ) - .option('--write-log-file', `Writes all logs to 'providence.log' file`); - -commander - .command('search ') - .alias('s') - .description('perfoms regex search string like "my-.*-comp"') - .action((regexString, options) => { - searchMode = 'search-query'; - regexSearchOptions = options; - regexSearchOptions.regexString = regexString; - launchProvidence(); - }); - -commander - .command('feature ') - .alias('f') - .description('query like "tg-icon[size=xs]"') - .option('-m, --method [method]', 'query method: "grep" or "ast"', setQueryMethod, 'grep') - .action((queryString, options) => { - searchMode = 'feature-query'; - featureOptions = options; - featureOptions.queryString = queryString; - launchProvidence(); - }); - -commander - .command('analyze [analyzer-name]') - .alias('a') - .description( - `predefined "query" for ast analysis. Can be a script found in program/analyzers, - like "find-imports"`, - ) - .option( - '-o, --prompt-optional-config', - `by default, only required configuration options are - asked for. When this flag is provided, optional configuration options are shown as well`, - ) - .option('-c, --config [config]', 'configuration object for analyzer', c => JSON.parse(c)) - .action((analyzerName, options) => { - searchMode = 'analyzer-query'; - analyzerOptions = options; - analyzerOptions.name = analyzerName; - launchProvidence(); - }); - -commander - .command('extend-docs') - .alias('e') - .description( - `Generates data for "babel-extend-docs" plugin. These data are generated by the "match-paths" - plugin, which automatically resolves import paths from reference projects - (say [@lion/input, @lion/textarea, ...etc]) to a target project (say "wolf-ui").`, - ) - .option( - '--prefix-from [prefix-from]', - `Prefix for components of reference layer. By default "lion"`, - a => a, - 'lion', - ) - .option( - '--prefix-to [prefix-to]', - `Prefix for components of reference layer. For instance "wolf"`, - ) - .option( - '--output-folder [output-folder]', - `This is the file path where the result file "providence-extend-docs-data.json" will be written to`, - p => pathLib.resolve(process.cwd(), p.trim()), - process.cwd(), - ) - .action(options => { - if (!options.prefixTo) { - LogService.error(`Please provide a "prefix to" like '--prefix-to "myprefix"'`); - process.exit(1); - } - if (!commander.referencePaths) { - LogService.error(`Please provide referencePaths path like '-r "node_modules/@lion/*"'`); - process.exit(1); - } - const prefixCfg = { from: options.prefixFrom, to: options.prefixTo }; - launchProvidenceWithExtendDocs(commander.referencePaths, prefixCfg, options.outputFolder); - }); - -commander - .command('manage-projects') - .description( - `Before running a query, be sure to have search-targets up to date (think of - npm/bower dependencies, latest version etc.)`, - ) - .option('-u, --update', 'gets latest of all search-targets and references') - .option('-d, --deps', 'installs npm/bower dependencies of search-targets') - .option('-h, --create-version-history', 'gets latest of all search-targets and references') - .action(options => { - manageSearchTargets(options); - }); - -commander.parse(process.argv); +cli(); diff --git a/packages/providence-analytics/src/program/analyzers/find-exports.js b/packages/providence-analytics/src/program/analyzers/find-exports.js index dfc6491b0..396f87645 100644 --- a/packages/providence-analytics/src/program/analyzers/find-exports.js +++ b/packages/providence-analytics/src/program/analyzers/find-exports.js @@ -133,9 +133,9 @@ function getLocalNameSpecifiers(node) { /** * @desc Finds import specifiers and sources for a given ast result * @param {BabelAst} ast - * @param {boolean} searchForFileImports + * @param {FindExportsConfig} config */ -function findExportsPerAstEntry(ast, searchForFileImports) { +function findExportsPerAstEntry(ast, { skipFileImports }) { // Visit AST... const transformedEntry = []; // Unfortunately, we cannot have async functions in babel traverse. @@ -155,7 +155,7 @@ function findExportsPerAstEntry(ast, searchForFileImports) { }, }); - if (searchForFileImports) { + if (!skipFileImports) { // Always add an entry for just the file 'relativePath' // (since this also can be imported directly from a search target project) transformedEntry.push({ @@ -181,14 +181,13 @@ class FindExportsAnalyzer extends Analyzer { /** * @typedef FindExportsConfig * @property {boolean} [onlyInternalSources=false] - * @property {{ [category]: (filePath) => boolean }} [customConfig.categories] object with - * categories as keys and (not necessarily mutually exlusive) functions that define a category - * @property {boolean} searchForFileImports Instead of only focusing on specifiers like - * [import {specifier} 'lion-based-ui/foo.js'], also list [import 'lion-based-ui/foo.js'] as a result + * @property {boolean} [skipFileImports=false] Instead of both focusing on specifiers like + * [import {specifier} 'lion-based-ui/foo.js'], and [import 'lion-based-ui/foo.js'] as a result, + * not list file exports */ const cfg = { targetProjectPath: null, - metaConfig: null, + skipFileImports: false, ...customConfig, }; @@ -205,7 +204,7 @@ class FindExportsAnalyzer extends Analyzer { */ const projectPath = cfg.targetProjectPath; const queryOutput = await this._traverse(async (ast, { relativePath }) => { - let transformedEntry = findExportsPerAstEntry(ast, cfg, relativePath, projectPath); + let transformedEntry = findExportsPerAstEntry(ast, cfg); transformedEntry = await normalizeSourcePaths(transformedEntry, relativePath, projectPath); transformedEntry = await trackdownRoot(transformedEntry, relativePath, projectPath); diff --git a/packages/providence-analytics/src/program/analyzers/helpers/track-down-identifier.js b/packages/providence-analytics/src/program/analyzers/helpers/track-down-identifier.js index fe658f71d..f5059398a 100644 --- a/packages/providence-analytics/src/program/analyzers/helpers/track-down-identifier.js +++ b/packages/providence-analytics/src/program/analyzers/helpers/track-down-identifier.js @@ -9,9 +9,6 @@ const { AstService } = require('../../services/AstService.js'); const { LogService } = require('../../services/LogService.js'); const { memoizeAsync } = require('../../utils/memoize.js'); -// TODO: memoize trackDownIdentifierFromScope (we can do so if tests are not mocked under same -// filesystem paths) - /** @typedef {import('./types').RootFile} RootFile */ /** @@ -82,7 +79,6 @@ function getImportSourceFromAst(astPath, identifierName) { } let trackDownIdentifier; - /** * @example *```js @@ -237,7 +233,7 @@ trackDownIdentifier = memoizeAsync(trackDownIdentifierFn); * @param {string} fullCurrentFilePath * @param {string} projectPath */ -async function trackDownIdentifierFromScope( +async function trackDownIdentifierFromScopeFn( astPath, identifierNameInScope, fullCurrentFilePath, @@ -261,6 +257,8 @@ async function trackDownIdentifierFromScope( return rootFile; } +const trackDownIdentifierFromScope = memoizeAsync(trackDownIdentifierFromScopeFn); + module.exports = { trackDownIdentifier, getImportSourceFromAst, diff --git a/packages/providence-analytics/src/program/analyzers/match-subclasses.js b/packages/providence-analytics/src/program/analyzers/match-subclasses.js index 5e1c722eb..2fee203e8 100644 --- a/packages/providence-analytics/src/program/analyzers/match-subclasses.js +++ b/packages/providence-analytics/src/program/analyzers/match-subclasses.js @@ -132,12 +132,9 @@ function matchSubclassesPostprocess( klass => klass.rootFile.specifier === exportSpecifier, ); - // console.log('classEntryResult', classEntryResult); - if (!classMatch) { return; } - // console.log(exportSpecifier, classEntryResult.superClasses && classEntryResult.superClasses.map(k => k.rootFile.specifier)); /** * @example diff --git a/packages/providence-analytics/src/program/services/InputDataService.js b/packages/providence-analytics/src/program/services/InputDataService.js index 22526a498..21bb38479 100644 --- a/packages/providence-analytics/src/program/services/InputDataService.js +++ b/packages/providence-analytics/src/program/services/InputDataService.js @@ -6,10 +6,53 @@ const fs = require('fs'); const pathLib = require('path'); const child_process = require('child_process'); // eslint-disable-line camelcase const glob = require('glob'); +const anymatch = require('anymatch'); +const isNegatedGlob = require('is-negated-glob'); const { LogService } = require('./LogService.js'); const { AstService } = require('./AstService.js'); const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js'); +function getGitIgnorePaths(rootPath) { + let fileContent; + try { + fileContent = fs.readFileSync(`${rootPath}/.gitignore`, 'utf8'); + } catch (_) { + return []; + } + + const entries = fileContent.split('\n').filter(entry => { + entry = entry.trim(); + if (entry.startsWith('#')) { + return false; + } + return entry.trim().length; + }); + return entries; +} + +/** + * Gives back all files and folders that need to be added to npm artifact + */ +function getNpmPackagePaths(rootPath) { + let pkgJson; + try { + const fileContent = fs.readFileSync(`${rootPath}/package.json`, 'utf8'); + pkgJson = JSON.parse(fileContent); + } catch (_) { + return []; + } + if (pkgJson.files) { + return pkgJson.files.map(fileOrFolder => { + const isFolderGlob = !fileOrFolder.includes('*') && !fileOrFolder.includes('.'); + if (isFolderGlob) { + return `${fileOrFolder}/**/*`; + } + return fileOrFolder; + }); + } + return []; +} + /** * * @param {string|array} v @@ -34,14 +77,6 @@ function multiGlobSync(patterns, { keepDirs = false } = {}) { return Array.from(res); } -const defaultGatherFilesConfig = { - extensions: ['.js'], - excludeFiles: [], - excludeFolders: ['node_modules', 'bower_components'], - includePaths: [], - depth: Infinity, -}; - /** * @typedef {Object} ProjectData * @property {string} project project name @@ -70,7 +105,7 @@ class InputDataService { path: projectPath, }, entries: this.gatherFilesFromDir(projectPath, { - ...defaultGatherFilesConfig, + ...this.defaultGatherFilesConfig, ...gatherFilesConfig, }), })); @@ -218,8 +253,12 @@ class InputDataService { this.__targetProjectPaths = ensureArray(v); } - static getDefaultGatherFilesConfig() { - return defaultGatherFilesConfig; + static get defaultGatherFilesConfig() { + return { + extensions: ['.js'], + filter: ['!node_modules/**', '!bower_components/**', '!**/*.conf.js', '!**/*.config.js'], + depth: Infinity, + }; } static getGlobPattern(startPath, cfg, withoutDepth = false) { @@ -243,37 +282,47 @@ class InputDataService { * @param {string[]} [result] - list of file paths, for internal (recursive) calls * @returns {string[]} result list of file paths */ - static gatherFilesFromDir(startPath, customConfig) { + static gatherFilesFromDir(startPath, customConfig = {}) { const cfg = { - ...defaultGatherFilesConfig, + ...this.defaultGatherFilesConfig, ...customConfig, }; + if (!customConfig.omitDefaultFilter) { + cfg.filter = [...this.defaultGatherFilesConfig.filter, ...(customConfig.filter || [])]; + } + + const gitIgnorePaths = getGitIgnorePaths(startPath); + const npmPackagePaths = getNpmPackagePaths(startPath); + const removeFilter = gitIgnorePaths.map(p => `!${p}`); + const keepFilter = npmPackagePaths; + + cfg.filter.forEach(filterEntry => { + const { negated, pattern } = isNegatedGlob(filterEntry); + if (negated) { + removeFilter.push(pattern); + } else { + keepFilter.push(filterEntry); + } + }); let globPattern = this.getGlobPattern(startPath, cfg); globPattern += `.{${cfg.extensions.map(e => e.slice(1)).join(',')},}`; const globRes = multiGlobSync(globPattern); - const globPatternWithoutDepth = this.getGlobPattern(startPath, cfg, true); - let excludedGlobFiles; - if (cfg.exclude) { - excludedGlobFiles = multiGlobSync(`${globPatternWithoutDepth}/${cfg.exclude}`); - } - - let filteredGlobRes = globRes.filter(gr => { - const localGr = gr.replace(startPath, ''); - return ( - !cfg.excludeFolders.some(f => localGr.includes(`${f}/`)) && - !cfg.excludeFiles.some(f => localGr.includes(f)) && - !(excludedGlobFiles && excludedGlobFiles.some(f => gr.includes(f))) - ); + const filteredGlobRes = globRes.filter(filePath => { + const localFilePath = filePath.replace(`${startPath}/`, ''); + if (removeFilter.length) { + const remove = anymatch(removeFilter, localFilePath); + if (remove) { + return false; + } + } + if (!keepFilter.length) { + return true; + } + return anymatch(keepFilter, localFilePath); }); - if (cfg.includePaths && cfg.includePaths.length) { - filteredGlobRes = globRes.filter(gr => - cfg.includePaths.some(p => gr.startsWith(pathLib.resolve(startPath, p))), - ); - } - if (!filteredGlobRes || !filteredGlobRes.length) { LogService.warn(`No files found for path '${startPath}'`); } @@ -293,5 +342,6 @@ class InputDataService { } } } +InputDataService.cacheDisabled = false; module.exports = { InputDataService }; diff --git a/packages/providence-analytics/src/program/types/index.js b/packages/providence-analytics/src/program/types/index.js index 186e5ed2e..561c50e5e 100644 --- a/packages/providence-analytics/src/program/types/index.js +++ b/packages/providence-analytics/src/program/types/index.js @@ -49,9 +49,17 @@ * @typedef {InputDataProject[]} InputData - all files found that are queryable */ +/** + * @typedef {Array|String|RegExp|Function} AnyMatchString see: https://www.npmjs.com/package/anymatch + * Allows negations as well. See: https://www.npmjs.com/package/is-negated-glob + * @example + * `'scripts/**\/*.js' + * '!scripts / vendor/**' + * 'scripts/vendor/react.js' + */ + /** * @typedef {Object} GatherFilesConfig * @property {string[]} [extensions] file extension like ['.js', '.html'] - * @property {string[]} [excludeFiles] file names filtered out - * @property {string[]} [excludeFolders] folder names filtered outs + * @property {AnyMatchString[]} [filter] file patterns filtered out. See: https://www.npmjs.com/package/anymatch */ diff --git a/packages/providence-analytics/src/program/utils/memoize.js b/packages/providence-analytics/src/program/utils/memoize.js index faf5b8317..6a34b52ec 100644 --- a/packages/providence-analytics/src/program/utils/memoize.js +++ b/packages/providence-analytics/src/program/utils/memoize.js @@ -1,10 +1,13 @@ +const { InputDataService } = require('../services/InputDataService.js'); + function memoize(func, externalStorage) { const storage = externalStorage || {}; // eslint-disable-next-line func-names return function () { // eslint-disable-next-line prefer-rest-params const args = [...arguments]; - if (args in storage) { + // Allow disabling of cache for testing purposes + if (!InputDataService.cacheDisabled && args in storage) { return storage[args]; } const outcome = func.apply(this, args); @@ -19,7 +22,8 @@ function memoizeAsync(func, externalStorage) { return async function () { // eslint-disable-next-line prefer-rest-params const args = [...arguments]; - if (args in storage) { + // Allow disabling of cache for testing purposes + if (!InputDataService.cacheDisabled && args in storage) { return storage[args]; } const outcome = await func.apply(this, args); diff --git a/packages/providence-analytics/test-helpers/mock-project-helpers.js b/packages/providence-analytics/test-helpers/mock-project-helpers.js index c07339c09..b17f58a6c 100644 --- a/packages/providence-analytics/test-helpers/mock-project-helpers.js +++ b/packages/providence-analytics/test-helpers/mock-project-helpers.js @@ -38,7 +38,8 @@ function mockProject(files, cfg = {}, existingMock = {}) { } const optionalPackageJson = {}; - const hasPackageJson = cfg.filePaths && cfg.filePaths.includes('./package.json'); + const hasPackageJson = + (cfg.filePaths && cfg.filePaths.includes('./package.json')) || files['./package.json']; if (!hasPackageJson) { optionalPackageJson[projPath] = { 'package.json': `{ "name": "${projName}" , "version": "${cfg.version || '0.1.0-mock'}" }`, diff --git a/packages/providence-analytics/test-node/cli/cli.test.js b/packages/providence-analytics/test-node/cli/cli.test.js new file mode 100644 index 000000000..4ebbafe66 --- /dev/null +++ b/packages/providence-analytics/test-node/cli/cli.test.js @@ -0,0 +1,172 @@ +const sinon = require('sinon'); +const pathLib = require('path'); +const { expect } = require('chai'); +const { + mockProject, + restoreMockedProjects, +} = require('../../test-helpers/mock-project-helpers.js'); +const { + mockWriteToJson, + restoreWriteToJson, +} = require('../../test-helpers/mock-report-service-helpers.js'); +const { + suppressNonCriticalLogs, + restoreSuppressNonCriticalLogs, +} = require('../../test-helpers/mock-log-service-helpers.js'); + +const { QueryService } = require('../../src/program/services/QueryService.js'); +const { LogService } = require('../../src/program/services/LogService.js'); + +const providenceModule = require('../../src/program/providence.js'); +const dummyAnalyzer = require('../../test-helpers/templates/analyzer-template.js'); +const { cli } = require('../../src/cli/cli.js'); +const { pathsArrayFromCs } = require('../../src/cli/cli-helpers.js'); + +const queryResults = []; +const rootDir = pathLib.resolve(__dirname, '../../'); + +describe('Providence CLI', () => { + before(() => { + LogService.debugEnabled = true; + suppressNonCriticalLogs(); + mockWriteToJson(queryResults); + + mockProject( + { + './src/OriginalComp.js': `export class OriginalComp {}`, + './src/inbetween.js': `export { OriginalComp as InBetweenComp } from './OriginalComp.js'`, + './index.js': `export { InBetweenComp as MyComp } from './src/inbetween.js'`, + }, + { + projectName: 'example-project', + projectPath: '/mocked/path/example-project', + }, + ); + }); + + after(() => { + restoreSuppressNonCriticalLogs(); + restoreWriteToJson(); + restoreMockedProjects(); + }); + + let providenceStub; + let qConfStub; + beforeEach(() => { + qConfStub = sinon.stub(QueryService, 'getQueryConfigFromAnalyzer').returns({ analyzer: {} }); + providenceStub = sinon.stub(providenceModule, 'providence'); + }); + + afterEach(() => { + providenceStub.restore(); + qConfStub.restore(); + }); + + async function runCli(args, cwd) { + process.argv = [...process.argv.slice(0, 2), ...args.split(' ')]; + await cli({ cwd, addProjectDependencyPaths: false }); + } + + const analyzCmd = 'analyze find-exports'; + + it('creates a QueryConfig', async () => { + await runCli('analyze find-exports -t /mocked/path/example-project'); + expect(qConfStub.called).to.be.true; + expect(qConfStub.args[0][0]).to.equal('find-exports'); + }); + + it('calls providence', async () => { + await runCli(`${analyzCmd} -t /mocked/path/example-project`); + expect(providenceStub.called).to.be.true; + }); + + describe('Global options', () => { + it('"-e --extensions"', async () => { + await runCli(`${analyzCmd} --extensions bla,blu`); + expect(providenceStub.args[0][1].gatherFilesConfig.extensions).to.eql(['.bla', '.blu']); + }); + + describe('"-t", "--search-target-paths"', async () => { + it('allows absolute paths', async () => { + await runCli(`${analyzCmd} -t /mocked/path/example-project`, rootDir); + expect(providenceStub.args[0][1].targetProjectPaths).to.eql([ + '/mocked/path/example-project', + ]); + }); + + it('allows relative paths', async () => { + await runCli( + `${analyzCmd} -t ./test-helpers/project-mocks/importing-target-project`, + rootDir, + ); + expect(providenceStub.args[0][1].targetProjectPaths).to.eql([ + `${rootDir}/test-helpers/project-mocks/importing-target-project`, + ]); + + await runCli( + `${analyzCmd} -t test-helpers/project-mocks/importing-target-project`, + rootDir, + ); + expect(providenceStub.args[0][1].targetProjectPaths).to.eql([ + `${rootDir}/test-helpers/project-mocks/importing-target-project`, + ]); + }); + + // TODO: globbing via cli-helpers doesn't work for some reason when run in this test + it.skip('allows globs', async () => { + await runCli(`${analyzCmd} -t test-helpers/*`, rootDir); + expect(providenceStub.args[0][1].targetProjectPaths).to.eql([ + `${process.cwd()}/needed-for-test/pass-glob`, + ]); + }); + }); + + it('"-r", "--reference-paths"', async () => {}); + it('"--search-target-collection"', async () => {}); + it('"--reference-collection"', async () => {}); + + it.skip('"-R --verbose-report"', async () => {}); + it.skip('"-D", "--debug"', async () => {}); + }); + + describe('Commands', () => { + describe('Analyze', () => { + it('calls providence', async () => { + expect(typeof dummyAnalyzer.name).to.equal('string'); + }); + describe('Options', () => { + it('"-o", "--prompt-optional-config"', async () => {}); + it('"-c", "--config"', async () => {}); + }); + }); + describe('Query', () => {}); + describe('Search', () => {}); + describe('Manage', () => {}); + }); +}); + +describe('CLI helpers', () => { + describe('pathsArrayFromCs', () => { + it('allows absolute paths', async () => { + expect(pathsArrayFromCs('/mocked/path/example-project', rootDir)).to.eql([ + '/mocked/path/example-project', + ]); + }); + + it('allows relative paths', async () => { + expect( + pathsArrayFromCs('./test-helpers/project-mocks/importing-target-project', rootDir), + ).to.eql([`${rootDir}/test-helpers/project-mocks/importing-target-project`]); + expect( + pathsArrayFromCs('test-helpers/project-mocks/importing-target-project', rootDir), + ).to.eql([`${rootDir}/test-helpers/project-mocks/importing-target-project`]); + }); + + it('allows globs', async () => { + expect(pathsArrayFromCs('test-helpers/project-mocks*', rootDir)).to.eql([ + `${process.cwd()}/test-helpers/project-mocks`, + `${process.cwd()}/test-helpers/project-mocks-analyzer-outputs`, + ]); + }); + }); +}); diff --git a/packages/providence-analytics/test-node/cli/cli.testx.js b/packages/providence-analytics/test-node/cli/cli.testx.js deleted file mode 100644 index dcf8b2ed1..000000000 --- a/packages/providence-analytics/test-node/cli/cli.testx.js +++ /dev/null @@ -1,98 +0,0 @@ -const sinon = require('sinon'); -const pathLib = require('path'); -const { expect } = require('chai'); -const { - mockProject, - // restoreMockedProjects, -} = require('../../test-helpers/mock-project-helpers.js'); -const { - mockWriteToJson, - restoreWriteToJson, -} = require('../../test-helpers/mock-report-service-helpers.js'); -const { - suppressNonCriticalLogs, - restoreSuppressNonCriticalLogs, -} = require('../../test-helpers/mock-log-service-helpers.js'); - -const { spawnProcess } = require('../../src/cli/cli-helpers.js'); -const { QueryService } = require('../../src/program/services/QueryService.js'); -const providenceModule = require('../../src/program/providence.js'); -const dummyAnalyzer = require('../../test-helpers/templates/analyzer-template.js'); - -const queryResults = []; - -describe('Providence CLI', () => { - before(() => { - suppressNonCriticalLogs(); - mockWriteToJson(queryResults); - }); - - after(() => { - restoreSuppressNonCriticalLogs(); - restoreWriteToJson(); - }); - - mockProject( - { - './src/OriginalComp.js': `export class OriginalComp {}`, - './src/inbetween.js': `export { OriginalComp as InBetweenComp } from './OriginalComp.js'`, - './index.js': `export { InBetweenComp as MyComp } from './src/inbetween.js'`, - }, - { - project: 'example-project', - path: '/mocked/path', - }, - ); - - const rootDir = pathLib.resolve(__dirname, '../../'); - async function cli(args) { - return spawnProcess(`node ./src/cli/index.js ${args}`, { cwd: rootDir }); - } - - async function cliAnalyze(args) { - return spawnProcess(`node ./src/cli/index.js analyze find-exports ${args}`, { cwd: rootDir }); - } - - it('creates a QueryConfig', async () => { - const stub = sinon.stub(QueryService, 'getQueryConfigFromAnalyzer'); - await cliAnalyze('-t "/mocked/path/example-project"'); - expect(stub.args[0]).to.equal('find-exports'); - }); - - it('calls providence', async () => { - const providenceStub = sinon.stub(providenceModule, 'providence'); - await cliAnalyze('-t "/mocked/path/example-project"'); - expect(providenceStub).to.have.been.called; - }); - - describe('Global options', () => { - it('"-e --extensions"', async () => { - const providenceStub = sinon.stub(providenceModule, 'providence'); - await cli('--extensions ".bla, .blu"'); - expect(providenceStub.args[1].gatherFilesConfig.extensions).to.eql(['bla', 'blu']); - }); - - it('"-t", "--search-target-paths"', async () => {}); - it('"-r", "--reference-paths"', async () => {}); - it('"--search-target-collection"', async () => {}); - it('"--reference-collection"', async () => {}); - - it.skip('"-R --verbose-report"', async () => {}); - it.skip('"-D", "--debug"', async () => {}); - }); - - describe('Commands', () => { - describe('Analyze', () => { - it('calls providence', async () => { - expect(typeof dummyAnalyzer.name).to.equal('string'); - }); - describe('Options', () => { - it('"-o", "--prompt-optional-config"', async () => {}); - it('"-c", "--config"', async () => {}); - }); - }); - describe('Query', () => {}); - describe('Search', () => {}); - describe('Manage', () => {}); - }); -}); diff --git a/packages/providence-analytics/test-node/program/analyzers/match-subclasses.test.js b/packages/providence-analytics/test-node/program/analyzers/match-subclasses.test.js index a5091c817..4f6d8a70d 100644 --- a/packages/providence-analytics/test-node/program/analyzers/match-subclasses.test.js +++ b/packages/providence-analytics/test-node/program/analyzers/match-subclasses.test.js @@ -138,19 +138,23 @@ const expectedMatchesOutput = [ describe('Analyzer "match-subclasses"', () => { const originalReferenceProjectPaths = InputDataService.referenceProjectPaths; const queryResults = []; - const cacheDisabledInitialValue = QueryService.cacheDisabled; + const cacheDisabledQInitialValue = QueryService.cacheDisabled; + const cacheDisabledIInitialValue = InputDataService.cacheDisabled; before(() => { QueryService.cacheDisabled = true; + InputDataService.cacheDisabled = true; suppressNonCriticalLogs(); }); after(() => { - QueryService.cacheDisabled = cacheDisabledInitialValue; + QueryService.cacheDisabled = cacheDisabledQInitialValue; + InputDataService.cacheDisabled = cacheDisabledIInitialValue; restoreSuppressNonCriticalLogs(); }); beforeEach(() => { + InputDataService.cacheDisabled = true; InputDataService.referenceProjectPaths = []; mockWriteToJson(queryResults); }); diff --git a/packages/providence-analytics/test-node/program/services/InputDataService.test.js b/packages/providence-analytics/test-node/program/services/InputDataService.test.js index 00699ddbc..6392cdea3 100644 --- a/packages/providence-analytics/test-node/program/services/InputDataService.test.js +++ b/packages/providence-analytics/test-node/program/services/InputDataService.test.js @@ -54,11 +54,10 @@ describe('InputDataService', () => { '/test-helpers/project-mocks/importing-target-project', ), ).to.equal(true); - - expect(inputDataPerProject[0].entries.length).to.equal(6); + expect(inputDataPerProject[0].entries.length).to.equal(11); expect(inputDataPerProject[0].entries[0].context.code).to.not.be.undefined; expect(inputDataPerProject[0].entries[0].file).to.equal( - './target-src/find-customelements/multiple.js', + './node_modules/exporting-ref-project/index.js', ); }); @@ -122,7 +121,7 @@ describe('InputDataService', () => { it('allows passing excludeFolders', async () => { const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { extensions: ['.html', '.js'], - excludeFolders: ['nested'], + filter: ['!nested/**'], }); expect(globOutput).to.eql([ '/fictional/project/index.html', @@ -133,10 +132,10 @@ describe('InputDataService', () => { ]); }); - it('allows passing excludeFiles', async () => { + it('allows passing excluded files', async () => { const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { extensions: ['.html', '.js'], - excludeFiles: ['index.js'], + filter: ['!index.js', '!**/*/index.js'], }); expect(globOutput).to.eql([ '/fictional/project/index.html', @@ -150,7 +149,7 @@ describe('InputDataService', () => { it('allows passing exclude globs', async () => { const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { extensions: ['.html', '.js'], - exclude: '**/*.test.{html,js}', + filter: ['!**/*.test.{html,js}'], }); expect(globOutput).to.eql([ '/fictional/project/index.html', @@ -159,6 +158,135 @@ describe('InputDataService', () => { '/fictional/project/nested/index.js', ]); }); + + it('omits node_modules and bower_components at root level by default', async () => { + mockProject({ + './index.js': '', + './node_modules/pkg/x.js': '', + './bower_components/pkg/y.js': '', + './nested/node_modules/pkg/x.js': '', + './nested/bower_components/pkg/y.js': '', + }); + + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput).to.eql([ + '/fictional/project/index.js', + '/fictional/project/nested/bower_components/pkg/y.js', + '/fictional/project/nested/node_modules/pkg/x.js', + ]); + }); + + it('allows to add root level files', async () => { + mockProject({ + './root-lvl.js': '', + './omitted/file.js': '', + './added/file.js': '', + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + filter: ['*', 'added/**/*'], + }); + expect(globOutput).to.eql([ + '/fictional/project/added/file.js', + '/fictional/project/root-lvl.js', + ]); + }); + + describe('Default config', () => { + it('omits config files by default', async () => { + mockProject({ + './index.js': '', + './karma.conf.js': '', + './commitlint.config.js': '', + './some-pkg/commitlint.config.js': '', + './some-other-pkg/commitlint.conf.js': '', + }); + + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput).to.eql(['/fictional/project/index.js']); + }); + + it('omits hidden files by default', async () => { + mockProject({ + './.blablarc.js': '', + './index.js': '', + }); + + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput).to.eql(['/fictional/project/index.js']); + }); + + it('filters npm files entries', async () => { + mockProject({ + './docs/x.js': '', + './src/y.js': '', + './file.add.js': '', + './omit.js': '', + './package.json': JSON.stringify({ + files: ['*.add.js', 'docs', 'src'], + }), + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput).to.eql([ + '/fictional/project/docs/x.js', + '/fictional/project/file.add.js', + '/fictional/project/src/y.js', + ]); + }); + + it('filters .gitignore entries', async () => { + mockProject({ + './coverage/file.js': '', + './storybook-static/index.js': '', + './build/index.js': '', + '.gitignore': ` +/coverage +# comment +/storybook-static/ + +build/ + `, + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput).to.eql([]); + }); + + describe('Default filter', () => { + it('merges default config filter with configured filter', async () => { + mockProject({ + './node_modules/root-lvl.js': '', + './bower_components/omitted/file.js': '', + './added.js': '', + './omit.js': '', + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + filter: ['added*'], + }); + expect(globOutput).to.eql(['/fictional/project/added.js']); + }); + + it('allows to omit default config filter', async () => { + mockProject({ + './node_modules/root-lvl.js': '', + './bower_components/omitted/file.js': '', + './xyz.conf.js': '', + './abc.config.js': '', + './added.js': '', + './omit.js': '', + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + filter: ['!omit*'], + omitDefaultFilter: true, + }); + expect(globOutput).to.eql([ + '/fictional/project/abc.config.js', + '/fictional/project/added.js', + '/fictional/project/bower_components/omitted/file.js', + '/fictional/project/node_modules/root-lvl.js', + '/fictional/project/xyz.conf.js', + ]); + }); + }); + }); }); }); }); diff --git a/yarn.lock b/yarn.lock index e61146f7c..9c41f0d04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1359,6 +1359,13 @@ resolved "https://registry.yarnpkg.com/@istanbuljs/schema/-/schema-0.1.2.tgz#26520bf09abe4a5644cd5414e37125a8954241dd" integrity sha512-tsAQNx32a8CoFhjhijUIhI4kccIAgmGhy8LZMZgGfmXcpMbPRUqn5LWmgRttILi6yeGmBJd2xsPkFMs0PzgPCw== +"@koa/cors@^3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.1.0.tgz#618bb073438cfdbd3ebd0e648a76e33b84f3a3b2" + integrity sha512-7ulRC1da/rBa6kj6P4g2aJfnET3z8Uf3SWu60cjbtxTA5g8lxRdX/Bd2P92EagGwwAhANeNw8T8if99rJliR6Q== + dependencies: + vary "^1.1.2" + "@lerna/add@^3.4.1": version "3.21.0" resolved "https://registry.yarnpkg.com/@lerna/add/-/add-3.21.0.tgz#27007bde71cc7b0a2969ab3c2f0ae41578b4577b" @@ -2262,9 +2269,9 @@ "@types/node" ">= 8" "@open-wc/building-rollup@^1.2.1": - version "1.3.1" - resolved "https://registry.yarnpkg.com/@open-wc/building-rollup/-/building-rollup-1.3.1.tgz#7e9fa27f2d7de7c5907248eb184b7941b524c3ec" - integrity sha512-TN/gGP2fY77S0g8+3A0M3lQ2xsWv2/yADUOvZAQH3PyBj6BcXemTQ1mIhBz6eHhyohAR0D5J61qwQSlHS8P0Lg== + version "1.3.2" + resolved "https://registry.yarnpkg.com/@open-wc/building-rollup/-/building-rollup-1.3.2.tgz#bee216105a1e42fe655c0926b1046ee255fd8dca" + integrity sha512-1fT3lkugEHXfddv7t/ahbgWCw4MFcjqBFocKfQV4btaJwg1XPQhWd4em+4TJ7p6u0FP0kr3jPJwLwOHGIrVTow== dependencies: "@babel/core" "^7.9.0" "@babel/helpers" "^7.9.2" @@ -2286,6 +2293,7 @@ deepmerge "^4.2.2" magic-string "^0.25.7" parse5 "^5.1.1" + regenerator-runtime "^0.13.3" rollup-plugin-babel "^5.0.0-alpha.1" rollup-plugin-terser "^5.2.0" rollup-plugin-workbox "^5.0.1" @@ -2337,9 +2345,9 @@ integrity sha512-9A3WohqNxEloJa4y1DuBL5zH12cNRNW1vsrkiaLMnOGuQdhibs2XY1oliudsKpvIeNjDXRVRPUdIIzn65BypCw== "@open-wc/demoing-storybook@^2.0.2": - version "2.3.10" - resolved "https://registry.yarnpkg.com/@open-wc/demoing-storybook/-/demoing-storybook-2.3.10.tgz#75d894bcb019977442e1846a965eae1f6d404872" - integrity sha512-71ielxlJyRm4PGfZ027NB39fLLNbV3lGtu0wmAGj7st8ADVxwzIJs1OBA5rHBLqyvr0EnHKLHX9pIVXRuk0Rsg== + version "2.3.11" + resolved "https://registry.yarnpkg.com/@open-wc/demoing-storybook/-/demoing-storybook-2.3.11.tgz#4c4887760591c6e58fb25852c92f0357c3445246" + integrity sha512-y5xfhAKptqzqTjhi6rfAR+2MRxPw4/ShJi3M1SKLP8Bg0QAXWZdEA6mzyZWA8BNziSpxBFxZxwobmChDv2PShA== dependencies: "@babel/core" "^7.9.0" "@babel/generator" "^7.9.6" @@ -2357,7 +2365,7 @@ command-line-args "^5.0.2" command-line-usage "^6.1.0" deepmerge "^4.2.2" - es-dev-server "^1.55.1" + es-dev-server "^1.56.0" es-module-lexer "^0.3.13" fs-extra "^8.1.0" glob "^7.1.3" @@ -2384,16 +2392,16 @@ eslint-plugin-no-only-tests "^2.3.1" eslint-plugin-wc "^1.2.0" -"@open-wc/karma-esm@^2.16.15": - version "2.16.15" - resolved "https://registry.yarnpkg.com/@open-wc/karma-esm/-/karma-esm-2.16.15.tgz#762a543dcf7a7eddd1c8204d8cbd2b7c1908b28e" - integrity sha512-L5/Aux653aLAQrbH7MLnYAMCYVp6Wdwe5JkGIt9pOgEBBIDRosBkxOpYpcvgPN7O8wnGUx3J9VnsJ7romS7VwA== +"@open-wc/karma-esm@^2.16.16": + version "2.16.16" + resolved "https://registry.yarnpkg.com/@open-wc/karma-esm/-/karma-esm-2.16.16.tgz#6ebff57f249e95f777b7e04782ef08ed41e22f53" + integrity sha512-IALT10JfwK+h7T0hGKTUliGdkWzQbyQg195D+RfUteIoTof6Z5+dBp7JUh2fQygIyNj7IIYHJ9ej816QlgHjdA== dependencies: "@open-wc/building-utils" "^2.18.0" babel-plugin-istanbul "^5.1.4" chokidar "^3.0.0" deepmerge "^4.2.2" - es-dev-server "^1.55.1" + es-dev-server "^1.56.0" minimatch "^3.0.4" node-fetch "^2.6.0" polyfills-loader "^1.6.1" @@ -2451,20 +2459,20 @@ lit-html "^1.0.0" "@open-wc/testing-karma-bs@^1.3.30": - version "1.3.81" - resolved "https://registry.yarnpkg.com/@open-wc/testing-karma-bs/-/testing-karma-bs-1.3.81.tgz#5294bdf3f2b513b4cc4ef68eaab53bf681e92005" - integrity sha512-jkQIZczYEW8ls/zJElC/L7HUwqRa29gInCstmflrJS0hQPkWpPAxZeLOIEBob7g15HCCOI0sKVroxD5fLCgmEg== + version "1.3.82" + resolved "https://registry.yarnpkg.com/@open-wc/testing-karma-bs/-/testing-karma-bs-1.3.82.tgz#bc3cf7ecc02f16e6d0c91bd83fdbc09b77d3a1d9" + integrity sha512-lEhnxczmr9AoatOAaK7z2AFQYAbHlAJ02dWixXU1tz/HtE3vzIlFYkGch3sumTT/gYLmFOi/Gd9DUvGe+0RiTQ== dependencies: - "@open-wc/testing-karma" "^3.4.5" + "@open-wc/testing-karma" "^3.4.6" "@types/node" "^11.13.0" karma-browserstack-launcher "^1.0.0" -"@open-wc/testing-karma@^3.2.30", "@open-wc/testing-karma@^3.4.5": - version "3.4.5" - resolved "https://registry.yarnpkg.com/@open-wc/testing-karma/-/testing-karma-3.4.5.tgz#116205167418e0b879992b2eac3f3d727f7a716e" - integrity sha512-PfudlhY8N0hiW7nKv+WvyjaGVsWXLqMvkmGSC9mDaIoK2A3Re3cQZuO8Rum3q+jAKC/3qlaYb4AYd64HtpZ1pw== +"@open-wc/testing-karma@^3.2.30", "@open-wc/testing-karma@^3.4.6": + version "3.4.6" + resolved "https://registry.yarnpkg.com/@open-wc/testing-karma/-/testing-karma-3.4.6.tgz#aacf5df4d17887017970bc22bcc7d9167f13f663" + integrity sha512-DTCzmZY1NOlhTnwef8N9ENd/TOykkIAXlJG36hjMRvBE3l0gYI1t+9N9x3f8jWHzrEzfTk5Hi7//iMUU0eLcZg== dependencies: - "@open-wc/karma-esm" "^2.16.15" + "@open-wc/karma-esm" "^2.16.16" "@types/karma" "^5.0.0" "@types/karma-coverage-istanbul-reporter" "^2.1.0" "@types/karma-mocha" "^1.3.0" @@ -2521,9 +2529,9 @@ "@webcomponents/shadycss" "^1.9.1" "@reach/router@^1.2.1": - version "1.3.3" - resolved "https://registry.yarnpkg.com/@reach/router/-/router-1.3.3.tgz#58162860dce6c9449d49be86b0561b5ef46d80db" - integrity sha512-gOIAiFhWdiVGSVjukKeNKkCRBLmnORoTPyBihI/jLunICPgxdP30DroAvPQuf1eVfQbfGJQDJkwhJXsNPMnVWw== + version "1.3.4" + resolved "https://registry.yarnpkg.com/@reach/router/-/router-1.3.4.tgz#d2574b19370a70c80480ed91f3da840136d10f8c" + integrity sha512-+mtn9wjlB9NN2CNnnC/BRYtwdKBfSyyasPYraNAyvaV1occr/5NnB4CVzjEZipNHwYebQwcndGUmpFzxAUoqSA== dependencies: create-react-context "0.3.0" invariant "^2.2.3" @@ -2980,6 +2988,13 @@ "@types/koa-compose" "*" "@types/node" "*" +"@types/koa__cors@^3.0.1": + version "3.0.1" + resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.0.1.tgz#a8cf8535f0fe682c9421f1b9379837c585f8b66b" + integrity sha512-loqZNXliley8kncc4wrX9KMqLGN6YfiaO3a3VFX+yVkkXJwOrZU4lipdudNjw5mFyC+5hd7h9075hQWcVVpeOg== + dependencies: + "@types/koa" "*" + "@types/lru-cache@^5.1.0": version "5.1.0" resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.0.tgz#57f228f2b80c046b4a1bd5cac031f81f207f4f03" @@ -3508,7 +3523,7 @@ anymatch@^2.0.0: micromatch "^3.1.4" normalize-path "^2.1.1" -anymatch@~3.1.1: +anymatch@^3.1.1, anymatch@~3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/anymatch/-/anymatch-3.1.1.tgz#c55ecf02185e2469259399310c173ce31233b142" integrity sha512-mM8522psRCqzV+6LhomX5wgp25YVibjh8Wj23I5RPkPppSVSjyKD2A2mBJmWGa+KN7f2D6LNh9jkBCeyLktzjg== @@ -4039,9 +4054,9 @@ binary-extensions@^1.0.0: integrity sha512-Un7MIEDdUC5gNpcGDV97op1Ywk748MpHcFTHoYs6qnj1Z3j7I53VG3nwZhKzoBZmbdRNnb6WRdFlwl7tSDuZGw== binary-extensions@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.0.0.tgz#23c0df14f6a88077f5f986c0d167ec03c3d5537c" - integrity sha512-Phlt0plgpIIBOGTT/ehfFnbNlfsDEiqmzE2KRXoX1bLIlir4X/MR+zSyBEkL05ffWgnRSf/DXv+WrUAVr93/ow== + version "2.1.0" + resolved "https://registry.yarnpkg.com/binary-extensions/-/binary-extensions-2.1.0.tgz#30fa40c9e7fe07dbc895678cd287024dea241dd9" + integrity sha512-1Yj8h9Q+QDF5FzhMs/c9+6UntbD5MkRfRwac8DoEm9ZfUBZ7tZ55YcGVAzEe4bXsdQHEk+s9S5wsOKVdZrw0tQ== bindings@^1.5.0: version "1.5.0" @@ -4220,12 +4235,12 @@ browserslist-useragent@^3.0.2: useragent "^2.3.0" browserslist@^4.0.0, browserslist@^4.12.0, browserslist@^4.8.5, browserslist@^4.9.1: - version "4.12.1" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.12.1.tgz#6d08bef149b70d153930780ba762644e0f329122" - integrity sha512-WMjXwFtPskSW1pQUDJRxvRKRkeCr7usN0O/Za76N+F4oadaTdQHotSGcX9jT/Hs7mSKPkyMFNvqawB/1HzYDKQ== + version "4.12.2" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.12.2.tgz#76653d7e4c57caa8a1a28513e2f4e197dc11a711" + integrity sha512-MfZaeYqR8StRZdstAK9hCKDd2StvePCYp5rHzQCPicUjfFliDgmuaBNPHYUTpAywBN8+Wc/d7NYVFkO0aqaBUw== dependencies: caniuse-lite "^1.0.30001088" - electron-to-chromium "^1.3.481" + electron-to-chromium "^1.3.483" escalade "^3.0.1" node-releases "^1.1.58" @@ -4532,9 +4547,9 @@ caniuse-api@^3.0.0: lodash.uniq "^4.5.0" caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001033, caniuse-lite@^1.0.30001088: - version "1.0.30001088" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001088.tgz#23a6b9e192106107458528858f2c0e0dba0d9073" - integrity sha512-6eYUrlShRYveyqKG58HcyOfPgh3zb2xqs7NvT2VVtP3hEUeeWvc3lqhpeMTxYWBBeeaT9A4bKsrtjATm66BTHg== + version "1.0.30001090" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001090.tgz#ff7766332f60e80fea4903f30d360622e5551850" + integrity sha512-QzPRKDCyp7RhjczTPZaqK3CjPA5Ht2UnXhZhCI4f7QiB5JK6KEuZBxIzyWnB3wO4hgAj4GMRxAhuiacfw0Psjg== caseless@~0.12.0: version "0.12.0" @@ -6168,7 +6183,7 @@ ejs@^2.6.1: resolved "https://registry.yarnpkg.com/ejs/-/ejs-2.7.4.tgz#48661287573dcc53e366c7a1ae52c3a120eec9ba" integrity sha512-7vmuyh5+kuUyJKePhQfRQBhXV5Ce+RnaeeQArKu1EAMpL3WbgMt5WG6uQZpEVvYSSsxMXRKOewtDk9RaTKXRlA== -electron-to-chromium@^1.3.481: +electron-to-chromium@^1.3.483: version "1.3.483" resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.3.483.tgz#9269e7cfc1c8e72709824da171cbe47ca5e3ca9e" integrity sha512-+05RF8S9rk8S0G8eBCqBRBaRq7+UN3lDs2DAvnG8SBSgQO3hjy0+qt4CmRk5eiuGbTcaicgXfPmBi31a+BD3lg== @@ -6362,10 +6377,10 @@ es-abstract@^1.17.0, es-abstract@^1.17.0-next.1, es-abstract@^1.17.5: string.prototype.trimend "^1.0.1" string.prototype.trimstart "^1.0.1" -es-dev-server@^1.18.1, es-dev-server@^1.55.1: - version "1.55.1" - resolved "https://registry.yarnpkg.com/es-dev-server/-/es-dev-server-1.55.1.tgz#27927eafe8c17fb4be6094d9d0f456fb94b315e6" - integrity sha512-oTkbQzisJza2jCKmRnrxv5TSE2X1I1FDi+bAqIijSCpNkUGYQ5w34unoUfnQdVznWmhFrNJVjn8whiAzridMkQ== +es-dev-server@^1.18.1, es-dev-server@^1.56.0: + version "1.56.0" + resolved "https://registry.yarnpkg.com/es-dev-server/-/es-dev-server-1.56.0.tgz#8703af87595f02fe9a1c92a07e64b7c7cc915a87" + integrity sha512-SL4CXdiku0hiB8zpsBLtEd7b8etIZE6IV0tIi02m0CcpTYV0rDMEvCBUYsQIN5hggJDDTBURgQjOWcT5kQv2eA== dependencies: "@babel/core" "^7.9.0" "@babel/plugin-proposal-dynamic-import" "^7.8.3" @@ -6378,6 +6393,7 @@ es-dev-server@^1.18.1, es-dev-server@^1.55.1: "@babel/plugin-syntax-optional-chaining" "^7.8.3" "@babel/plugin-transform-template-literals" "^7.8.3" "@babel/preset-env" "^7.9.0" + "@koa/cors" "^3.1.0" "@open-wc/building-utils" "^2.18.0" "@rollup/plugin-node-resolve" "^7.1.1" "@rollup/pluginutils" "^3.0.0" @@ -6392,6 +6408,7 @@ es-dev-server@^1.18.1, es-dev-server@^1.55.1: "@types/koa-compress" "^2.0.9" "@types/koa-etag" "^3.0.0" "@types/koa-static" "^4.0.1" + "@types/koa__cors" "^3.0.1" "@types/lru-cache" "^5.1.0" "@types/minimatch" "^3.0.3" "@types/path-is-inside" "^1.0.0" @@ -6526,9 +6543,9 @@ eslint-plugin-html@^6.0.0: htmlparser2 "^4.1.0" eslint-plugin-import@^2.18.0: - version "2.21.2" - resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.21.2.tgz#8fef77475cc5510801bedc95f84b932f7f334a7c" - integrity sha512-FEmxeGI6yaz+SnEB6YgNHlQK1Bs2DKLM+YF+vuTk5H8J9CLbJLtlPvRFgZZ2+sXiKAlN5dpdlrWOjK8ZoZJpQA== + version "2.22.0" + resolved "https://registry.yarnpkg.com/eslint-plugin-import/-/eslint-plugin-import-2.22.0.tgz#92f7736fe1fde3e2de77623c838dd992ff5ffb7e" + integrity sha512-66Fpf1Ln6aIS5Gr/55ts19eUuoDhAbZgnr6UxK5hbDx6l/QgQgx61AePq+BV4PP2uXQFClgMVzep5zZ94qqsxg== dependencies: array-includes "^3.1.1" array.prototype.flat "^1.2.3" @@ -8003,14 +8020,14 @@ http-errors@1.7.2: statuses ">= 1.5.0 < 2" toidentifier "1.0.0" -http-errors@^1.6.3, http-errors@~1.7.2: - version "1.7.3" - resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" - integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== +http-errors@^1.6.3: + version "1.8.0" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.8.0.tgz#75d1bbe497e1044f51e4ee9e704a62f28d336507" + integrity sha512-4I8r0C5JDhT5VkvI47QktDW75rNlGVsUf/8hzjCC/wkWI/jdTRmBb9aI7erSG82r1bjKY3F6k28WnsVxB1C73A== dependencies: depd "~1.1.2" inherits "2.0.4" - setprototypeof "1.1.1" + setprototypeof "1.2.0" statuses ">= 1.5.0 < 2" toidentifier "1.0.0" @@ -8024,6 +8041,17 @@ http-errors@~1.6.2: setprototypeof "1.1.0" statuses ">= 1.4.0 < 2" +http-errors@~1.7.2: + version "1.7.3" + resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.7.3.tgz#6c619e4f9c60308c38519498c14fbb10aacebb06" + integrity sha512-ZTTX0MWrsQ2ZAhA1cejAwDLycFsd7I7nVtnkT3Ol0aqodaKW+0CTZDQ1uBv5whptCnc8e8HeRRJxRs0kmm/Qfw== + dependencies: + depd "~1.1.2" + inherits "2.0.4" + setprototypeof "1.1.1" + statuses ">= 1.5.0 < 2" + toidentifier "1.0.0" + http-proxy-agent@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-2.1.0.tgz#e4821beef5b2142a2026bd73926fe537631c5405" @@ -8587,11 +8615,9 @@ is-plain-object@^2.0.3, is-plain-object@^2.0.4: isobject "^3.0.1" is-plain-object@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-3.0.0.tgz#47bfc5da1b5d50d64110806c199359482e75a928" - integrity sha512-tZIpofR+P05k8Aocp7UI/2UTa9lTJSebCXpFFoR9aibpokDj/uXBsJ8luUu0tTVYKkMU6URDUuOfJZ7koewXvg== - dependencies: - isobject "^4.0.0" + version "3.0.1" + resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-3.0.1.tgz#662d92d24c0aa4302407b0d45d21f2251c85f85b" + integrity sha512-Xnpx182SBMrr/aBik8y+GuR4U1L9FqMSojwDQwPMmxyC6bvEqly9UBCxhauBF5vNh2gwWJNX6oDV7O+OM4z34g== is-potential-custom-element-name@^1.0.0: version "1.0.0" @@ -8756,11 +8782,6 @@ isobject@^3.0.0, isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha1-TkMekrEalzFjaqH5yNHMvP2reN8= -isobject@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/isobject/-/isobject-4.0.0.tgz#3f1c9155e73b192022a80819bacd0343711697b0" - integrity sha512-S/2fF5wH8SJA/kmwr6HYhK/RI/OkhD84k8ntalo0iJjZikgq1XFvR5M8NPT1x5F7fBwCG3qHfnzeP/Vh/ZxCUA== - isomorphic-fetch@^2.1.1: version "2.2.1" resolved "https://registry.yarnpkg.com/isomorphic-fetch/-/isomorphic-fetch-2.2.1.tgz#611ae1acf14f5e81f729507472819fe9733558a9" @@ -11584,9 +11605,9 @@ posix-character-classes@^0.1.0: integrity sha1-AerA/jta9xoqbAL+q7jB/vfgDqs= prebuild-install@^5.3.3: - version "5.3.4" - resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.3.4.tgz#6982d10084269d364c1856550b7d090ea31fa293" - integrity sha512-AkKN+pf4fSEihjapLEEj8n85YIw/tN6BQqkhzbDc0RvEZGdkpJBGMUYx66AAMcPG2KzmPQS7Cm16an4HVBRRMA== + version "5.3.5" + resolved "https://registry.yarnpkg.com/prebuild-install/-/prebuild-install-5.3.5.tgz#e7e71e425298785ea9d22d4f958dbaccf8bb0e1b" + integrity sha512-YmMO7dph9CYKi5IR/BzjOJlRzpxGGVo1EsLSUZ0mt/Mq0HWZIHOKHHcHdT69yG54C9m6i45GpItwRHpk0Py7Uw== dependencies: detect-libc "^1.0.3" expand-template "^2.0.3" @@ -12814,9 +12835,9 @@ rollup@^1.31.1: acorn "^7.1.0" rollup@^2.0.0, rollup@^2.7.2: - version "2.18.0" - resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.18.0.tgz#f03801e5dd01415e5675dcf61c824ea493ca0392" - integrity sha512-LhuQQp3WpnHo3HlKCRrdMXpB6jdLsGOoXXSfMjbv74s5VdV3WZhkYJT0Z6w/EH3UgPH+g/S9T4GJrKW/5iD8TA== + version "2.18.1" + resolved "https://registry.yarnpkg.com/rollup/-/rollup-2.18.1.tgz#1662c679da5dfc89ec6fda75951a2bcca1a4f77d" + integrity sha512-w4X77ADA+WTGlapC8Z6yggdJtODw3SBl6R2LSkA7ZW5MtdkgcB7sfaSD1UWyx8diXbMcGIb0eI9gCx/dyqOgNQ== optionalDependencies: fsevents "~2.1.2" @@ -12974,6 +12995,11 @@ setprototypeof@1.1.1: resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.1.1.tgz#7e95acb24aa92f5885e0abef5ba131330d4ae683" integrity sha512-JvdAWfbXeIGaZ9cILp38HntZSFSo3mWg6xGcJJsd+d4aRMOqauag1C63dJfDw7OaMYwEbHMOxEZ1lqVRYP2OAw== +setprototypeof@1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/setprototypeof/-/setprototypeof-1.2.0.tgz#66c9a24a73f9fc28cbe66b09fed3d33dcaf1b424" + integrity sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw== + sha.js@^2.4.0, sha.js@^2.4.8: version "2.4.11" resolved "https://registry.yarnpkg.com/sha.js/-/sha.js-2.4.11.tgz#37a5cf0b81ecbc6943de109ba2960d1b26584ae7" @@ -13115,9 +13141,9 @@ slide@^1.1.6: integrity sha1-VusCfWW00tzmyy4tMsTUr8nh1wc= slugify@^1.3.1: - version "1.4.2" - resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.4.2.tgz#2c9b2b3321c43110a0058982eea7102b998d5068" - integrity sha512-7UfMG5rtkxfOI5jg/+f4DMQS3ikUqfWnfMvitrhwdTV4pibWXq9mN4RNLHSV3M1lR++x7z+AG7znsiozdBP+aA== + version "1.4.4" + resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.4.4.tgz#2f032ffa52b1e1ca2a27737c1ce47baae3d0883a" + integrity sha512-N2+9NJ8JzfRMh6PQLrBeDEnVDQZSytE/W4BTC4fNNPmO90Uu58uNwSlIJSs+lmPgWsaAF79WLhVPe5tuy7spjw== smart-buffer@^4.1.0: version "4.1.0" @@ -13768,9 +13794,9 @@ symbol-observable@^1.0.4: integrity sha512-e900nM8RRtGhlV36KGEU9k65K3mPb1WV70OdjfxlG2EAuM1noi/E/BaW/uMhL7bPEssK8QV57vN3esixjUvcXQ== systemjs@^6.3.1: - version "6.3.2" - resolved "https://registry.yarnpkg.com/systemjs/-/systemjs-6.3.2.tgz#2c3407f949260fdfb3b837c191869db933d1fe0f" - integrity sha512-zcALS1RIYtsQBG4fbaE+cJzKx+UoEuSM8xCkGGH99i7p7Ym3ALvhi9QrpF2lo0CMQaejqrE1GnbkuG2m/+H7ew== + version "6.3.3" + resolved "https://registry.yarnpkg.com/systemjs/-/systemjs-6.3.3.tgz#c0f2bec5cc72d0b36a8b971b1fa32bfc828b50d4" + integrity sha512-djQ6mZ4/cWKnVnhAWvr/4+5r7QHnC7WiA8sS9VuYRdEv3wYZYTIIQv8zPT79PdDSUwfX3bgvu5mZ8eTyLm2YQA== table-layout@^1.0.0: version "1.0.1" From 729966504b1e67969ea35e645a2271f61703f20a Mon Sep 17 00:00:00 2001 From: Thijs Louisse Date: Mon, 29 Jun 2020 13:22:34 +0200 Subject: [PATCH 2/2] feat(providence-analytics): cli arguments extend-docs chore: fix failing tests --- .../src/cli/cli-helpers.js | 4 ++ packages/providence-analytics/src/cli/cli.js | 16 +++++-- .../src/cli/generate-extend-docs-data.js | 21 ++++++--- .../src/program/analyzers/helpers/Analyzer.js | 2 +- .../test-node/cli/cli.test.js | 45 ++++++++++++++++++- 5 files changed, 76 insertions(+), 12 deletions(-) diff --git a/packages/providence-analytics/src/cli/cli-helpers.js b/packages/providence-analytics/src/cli/cli-helpers.js index 4a075600e..f113a088e 100644 --- a/packages/providence-analytics/src/cli/cli-helpers.js +++ b/packages/providence-analytics/src/cli/cli-helpers.js @@ -33,6 +33,10 @@ function setQueryMethod(m) { * @returns {string[]} */ function pathsArrayFromCs(t, cwd = process.cwd()) { + if (!t) { + return undefined; + } + return flatten( t.split(',').map(t => { if (t.startsWith('/')) { diff --git a/packages/providence-analytics/src/cli/cli.js b/packages/providence-analytics/src/cli/cli.js index 2c3c4d98d..25e3014b5 100755 --- a/packages/providence-analytics/src/cli/cli.js +++ b/packages/providence-analytics/src/cli/cli.js @@ -18,7 +18,7 @@ const { pathsArrayFromCollectionName, pathsArrayFromCs, } = require('./cli-helpers.js'); -const { launchProvidenceWithExtendDocs } = require('./generate-extend-docs-data.js'); +const extendDocsModule = require('./generate-extend-docs-data.js'); const { version } = require('../../package.json'); async function cli({ cwd, addProjectDependencyPaths } = {}) { @@ -111,12 +111,10 @@ async function cli({ cwd, addProjectDependencyPaths } = {}) { providenceModule.providence(queryConfig, { gatherFilesConfig: { extensions: commander.extensions, - // ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), filter: commander.whitelist, }, gatherFilesConfigReference: { extensions: commander.extensions, - // ...(commander.filteredTarget ? { excludeFolders: commander.filteredTarget } : {}), filter: commander.whitelistReference, }, debugEnabled: commander.debug, @@ -275,7 +273,17 @@ async function cli({ cwd, addProjectDependencyPaths } = {}) { process.exit(1); } const prefixCfg = { from: options.prefixFrom, to: options.prefixTo }; - launchProvidenceWithExtendDocs(commander.referencePaths, prefixCfg, options.outputFolder); + extendDocsModule + .launchProvidenceWithExtendDocs({ + referenceProjectPaths: commander.referencePaths, + prefixCfg, + outputFolder: options.outputFolder, + extensions: commander.referencePaths, + whitelist: commander.whitelist, + whitelistReference: commander.whitelistReference, + }) + .then(() => resolveCli()) + .catch(() => rejectCli()); }); commander diff --git a/packages/providence-analytics/src/cli/generate-extend-docs-data.js b/packages/providence-analytics/src/cli/generate-extend-docs-data.js index 166ac09c1..f61782bbd 100644 --- a/packages/providence-analytics/src/cli/generate-extend-docs-data.js +++ b/packages/providence-analytics/src/cli/generate-extend-docs-data.js @@ -6,20 +6,31 @@ const { providence } = require('../program/providence.js'); const { QueryService } = require('../program/services/QueryService.js'); const { LogService } = require('../program/services/LogService.js'); -async function launchProvidenceWithExtendDocs(referencePaths, prefixObj, outputFolder) { +async function launchProvidenceWithExtendDocs({ + referenceProjectPaths, + prefixCfg, + outputFolder, + extensions, + whitelist, + whitelistReference, +}) { const t0 = performance.now(); const results = await providence( - QueryService.getQueryConfigFromAnalyzer('match-paths', { prefix: prefixObj }), + QueryService.getQueryConfigFromAnalyzer('match-paths', { prefix: prefixCfg }), { gatherFilesConfig: { - extensions: ['.js', '.html'], - excludeFolders: ['coverage', 'test'], + extensions: extensions || ['.js'], + filter: whitelist || ['!coverage', '!test'], + }, + gatherFilesConfigReference: { + extensions: extensions || ['.js'], + filter: whitelistReference || ['!coverage', '!test'], }, queryMethod: 'ast', report: false, targetProjectPaths: [pathLib.resolve(process.cwd())], - referenceProjectPaths: referencePaths, + referenceProjectPaths, }, ); diff --git a/packages/providence-analytics/src/program/analyzers/helpers/Analyzer.js b/packages/providence-analytics/src/program/analyzers/helpers/Analyzer.js index bc1144569..13e4fb0c8 100644 --- a/packages/providence-analytics/src/program/analyzers/helpers/Analyzer.js +++ b/packages/providence-analytics/src/program/analyzers/helpers/Analyzer.js @@ -165,7 +165,7 @@ class Analyzer { `skipping ${LogService.pad(this.name, 16)} for ${ this.identifier }: (${reason})\n${cfg.targetProjectPath.replace( - '/Users/hu84jr/git/providence/providence-input-data/search-targets/', + `${process.cwd()}/providence-input-data/search-targets/`, '', )}`, ); diff --git a/packages/providence-analytics/test-node/cli/cli.test.js b/packages/providence-analytics/test-node/cli/cli.test.js index 4ebbafe66..36405d453 100644 --- a/packages/providence-analytics/test-node/cli/cli.test.js +++ b/packages/providence-analytics/test-node/cli/cli.test.js @@ -15,9 +15,10 @@ const { } = require('../../test-helpers/mock-log-service-helpers.js'); const { QueryService } = require('../../src/program/services/QueryService.js'); -const { LogService } = require('../../src/program/services/LogService.js'); const providenceModule = require('../../src/program/providence.js'); +const extendDocsModule = require('../../src/cli/generate-extend-docs-data.js'); + const dummyAnalyzer = require('../../test-helpers/templates/analyzer-template.js'); const { cli } = require('../../src/cli/cli.js'); const { pathsArrayFromCs } = require('../../src/cli/cli-helpers.js'); @@ -27,7 +28,6 @@ const rootDir = pathLib.resolve(__dirname, '../../'); describe('Providence CLI', () => { before(() => { - LogService.debugEnabled = true; suppressNonCriticalLogs(); mockWriteToJson(queryResults); @@ -142,6 +142,47 @@ describe('Providence CLI', () => { describe('Query', () => {}); describe('Search', () => {}); describe('Manage', () => {}); + describe('Extend docs', () => { + let extendDocsStub; + beforeEach(() => { + extendDocsStub = sinon.stub(extendDocsModule, 'launchProvidenceWithExtendDocs').returns( + new Promise(resolve => { + resolve(); + }), + ); + }); + + afterEach(() => { + extendDocsStub.restore(); + }); + + it('allows configuration', async () => { + await runCli( + [ + 'extend-docs', + '-t /xyz', + '-r /xyz/x', + '--prefix-from pfrom --prefix-to pto', + '--output-folder /outp', + '--extensions bla', + '--whitelist wl --whitelist-reference wlr', + ].join(' '), + rootDir, + ); + expect(extendDocsStub.called).to.be.true; + expect(extendDocsStub.args[0][0]).to.eql({ + referenceProjectPaths: ['/xyz/x'], + prefixCfg: { + from: 'pfrom', + to: 'pto', + }, + outputFolder: '/outp', + extensions: ['/xyz/x'], + whitelist: [`${process.cwd()}/wl`], + whitelistReference: [`${process.cwd()}/wlr`], + }); + }); + }); }); });