From ca6c8e62789649c4a1de9e76cb042ee62ed79057 Mon Sep 17 00:00:00 2001 From: Thijs Louisse Date: Mon, 10 Aug 2020 13:44:32 +0200 Subject: [PATCH] feat(providence-analytics): add --allowlist-mode option --- .changeset/clever-trains-remember.md | 7 ++ packages/providence-analytics/src/cli/cli.js | 29 ++++++-- .../src/cli/generate-extend-docs-data.js | 8 +- .../src/program/services/InputDataService.js | 58 ++++++++++++--- .../match-subclasses/ExtendedComp.js | 2 +- .../test-node/cli/cli.test.js | 26 +++++-- .../program/services/InputDataService.test.js | 73 +++++++++++++++++-- 7 files changed, 168 insertions(+), 35 deletions(-) create mode 100644 .changeset/clever-trains-remember.md diff --git a/.changeset/clever-trains-remember.md b/.changeset/clever-trains-remember.md new file mode 100644 index 000000000..3c4861fad --- /dev/null +++ b/.changeset/clever-trains-remember.md @@ -0,0 +1,7 @@ +--- +'providence-analytics': minor +--- + +Allowlist mode: autodetects whether analyzed repository is a "git" or "npm" (published artifact) repository. +Via the cli `--allowlist-mode 'npm|git|all'` and `--allowlist-mode-reference 'npm|git|all'` can be +configured to override the autodetected mode. diff --git a/packages/providence-analytics/src/cli/cli.js b/packages/providence-analytics/src/cli/cli.js index ab9648c1a..58ef3fe73 100755 --- a/packages/providence-analytics/src/cli/cli.js +++ b/packages/providence-analytics/src/cli/cli.js @@ -116,11 +116,13 @@ async function cli({ cwd } = {}) { providenceModule.providence(queryConfig, { gatherFilesConfig: { extensions: commander.extensions, - filter: commander.whitelist, + allowlistMode: commander.allowlistMode, + filter: commander.allowlist, }, gatherFilesConfigReference: { extensions: commander.extensions, - filter: commander.whitelistReference, + allowlistMode: commander.allowlistModeReference, + filter: commander.allowlistReference, }, debugEnabled: commander.debug, queryMethod, @@ -172,12 +174,12 @@ async function cli({ cwd } = {}) { v => cliHelpers.pathsArrayFromCs(v, cwd), InputDataService.referenceProjectPaths, ) - .option('-w, --whitelist [whitelist]', `whitelisted paths, like './src, ./packages/*'`, v => + .option('-a, --allowlist [allowlist]', `allowlisted paths, like './src, ./packages/*'`, v => cliHelpers.pathsArrayFromCs(v, cwd), ) .option( - '--whitelist-reference [whitelist-reference]', - `whitelisted paths for reference, like './src, ./packages/*'`, + '--allowlist-reference [allowlist-reference]', + `allowed paths for reference, like './src, ./packages/*'`, v => cliHelpers.pathsArrayFromCs(v, cwd), ) .option( @@ -201,6 +203,19 @@ async function cli({ cwd } = {}) { without argument, it will act as boolean and include all dependencies. When a regex is supplied like --target-dependencies /^my-brand-/, it will filter all packages that comply with the regex`, + ) + .option( + '--allowlist-mode [allowlist-mode]', + `Depending on whether we are dealing with a published artifact + (a dependency installed via npm) or a git repository, different paths will be + automatically put in the appropiate mode. + A mode of 'npm' will look at the package.json "files" entry and a mode of + 'git' will look at '.gitignore' entry. The mode will be auto detected, but can be overridden + via this option.`, + ) + .option( + '--allowlist-mode-reference [allowlist-mode-reference]', + `allowlist mode applied to refernce project`, ); commander @@ -286,8 +301,8 @@ async function cli({ cwd } = {}) { prefixCfg, outputFolder: options.outputFolder, extensions: commander.extensions, - whitelist: commander.whitelist, - whitelistReference: commander.whitelistReference, + allowlist: commander.allowlist, + allowlistReference: commander.allowlistReference, }) .then(resolveCli) .catch(rejectCli); diff --git a/packages/providence-analytics/src/cli/generate-extend-docs-data.js b/packages/providence-analytics/src/cli/generate-extend-docs-data.js index f61782bbd..e2234508e 100644 --- a/packages/providence-analytics/src/cli/generate-extend-docs-data.js +++ b/packages/providence-analytics/src/cli/generate-extend-docs-data.js @@ -11,8 +11,8 @@ async function launchProvidenceWithExtendDocs({ prefixCfg, outputFolder, extensions, - whitelist, - whitelistReference, + allowlist, + allowlistReference, }) { const t0 = performance.now(); @@ -21,11 +21,11 @@ async function launchProvidenceWithExtendDocs({ { gatherFilesConfig: { extensions: extensions || ['.js'], - filter: whitelist || ['!coverage', '!test'], + filter: allowlist || ['!coverage', '!test'], }, gatherFilesConfigReference: { extensions: extensions || ['.js'], - filter: whitelistReference || ['!coverage', '!test'], + filter: allowlistReference || ['!coverage', '!test'], }, queryMethod: 'ast', report: false, diff --git a/packages/providence-analytics/src/program/services/InputDataService.js b/packages/providence-analytics/src/program/services/InputDataService.js index d29799275..d8b8fd6d3 100644 --- a/packages/providence-analytics/src/program/services/InputDataService.js +++ b/packages/providence-analytics/src/program/services/InputDataService.js @@ -12,11 +12,17 @@ const { LogService } = require('./LogService.js'); const { AstService } = require('./AstService.js'); const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js'); -function getGitIgnorePaths(rootPath) { - let fileContent; +function getGitignoreFile(rootPath) { try { - fileContent = fs.readFileSync(`${rootPath}/.gitignore`, 'utf8'); + return fs.readFileSync(`${rootPath}/.gitignore`, 'utf8'); } catch (_) { + return undefined; + } +} + +function getGitIgnorePaths(rootPath) { + const fileContent = getGitignoreFile(rootPath); + if (!fileContent) { return []; } @@ -25,9 +31,28 @@ function getGitIgnorePaths(rootPath) { if (entry.startsWith('#')) { return false; } + if (entry.startsWith('!')) { + return false; // negated folders will be kept + } return entry.trim().length; }); - return entries; + + // normalize entries to be compatible with anymatch + const normalizedEntries = entries.map(e => { + let entry = e; + + if (entry.startsWith('/')) { + entry = entry.slice(1); + } + const isFile = entry.indexOf('.') > 0; // index of 0 means hidden file. + if (entry.endsWith('/')) { + entry += '**'; + } else if (!isFile) { + entry += '/**'; + } + return entry; + }); + return normalizedEntries; } /** @@ -290,10 +315,27 @@ class InputDataService { if (!customConfig.omitDefaultFilter) { cfg.filter = [...this.defaultGatherFilesConfig.filter, ...(customConfig.filter || [])]; } + const allowlistModes = ['npm', 'git', 'all']; + if (customConfig.allowlistMode && !allowlistModes.includes(customConfig.allowlistMode)) { + throw new Error( + `[gatherFilesConfig] Please provide a valid allowListMode like "${allowlistModes.join( + '|', + )}". Found: "${customConfig.allowlistMode}"`, + ); + } - const gitIgnorePaths = getGitIgnorePaths(startPath); - const npmPackagePaths = getNpmPackagePaths(startPath); - const removeFilter = gitIgnorePaths.map(p => `!${p}`); + let gitIgnorePaths = []; + let npmPackagePaths = []; + + const hasGitIgnore = getGitignoreFile(startPath); + const allowlistMode = cfg.allowlistMode || (hasGitIgnore ? 'git' : 'npm'); + + if (allowlistMode === 'git') { + gitIgnorePaths = getGitIgnorePaths(startPath); + } else if (allowlistMode === 'npm') { + npmPackagePaths = getNpmPackagePaths(startPath); + } + const removeFilter = gitIgnorePaths; const keepFilter = npmPackagePaths; cfg.filter.forEach(filterEntry => { @@ -322,11 +364,9 @@ class InputDataService { } return anymatch(keepFilter, localFilePath); }); - if (!filteredGlobRes || !filteredGlobRes.length) { LogService.warn(`No files found for path '${startPath}'`); } - return filteredGlobRes; } diff --git a/packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js b/packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js index 62cb318b9..d3986bc97 100644 --- a/packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js +++ b/packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js @@ -5,7 +5,7 @@ import MyCompMixin from './internalProxy.js'; export class ExtendedComp extends MyCompMixin(RefClass) { /** - * Whitelisted members + * allowed members */ get getterSetter() {} set getterSetter(v) {} diff --git a/packages/providence-analytics/test-node/cli/cli.test.js b/packages/providence-analytics/test-node/cli/cli.test.js index c94e987df..227189c78 100644 --- a/packages/providence-analytics/test-node/cli/cli.test.js +++ b/packages/providence-analytics/test-node/cli/cli.test.js @@ -226,8 +226,8 @@ describe('Providence CLI', () => { ]); }); - it('"-w --whitelist"', async () => { - await runCli(`${analyzeCmd} -w /mocked/path/example-project`, rootDir); + it('"-a --allowlist"', async () => { + await runCli(`${analyzeCmd} -a /mocked/path/example-project`, rootDir); expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project'); expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([ '/mocked/path/example-project', @@ -236,21 +236,31 @@ describe('Providence CLI', () => { pathsArrayFromCsStub.resetHistory(); providenceStub.resetHistory(); - await runCli(`${analyzeCmd} --whitelist /mocked/path/example-project`, rootDir); + await runCli(`${analyzeCmd} --allowlist /mocked/path/example-project`, rootDir); expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project'); expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([ '/mocked/path/example-project', ]); }); - it('"--whitelist-reference"', async () => { - await runCli(`${analyzeCmd} --whitelist-reference /mocked/path/example-project`, rootDir); + it('"--allowlist-reference"', async () => { + await runCli(`${analyzeCmd} --allowlist-reference /mocked/path/example-project`, rootDir); expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project'); expect(providenceStub.args[0][1].gatherFilesConfigReference.filter).to.eql([ '/mocked/path/example-project', ]); }); + it('--allowlist-mode', async () => { + await runCli(`${analyzeCmd} --allowlist-mode git`, rootDir); + expect(providenceStub.args[0][1].gatherFilesConfig.allowlistMode).to.equal('git'); + }); + + it('--allowlist-mode-reference', async () => { + await runCli(`${analyzeCmd} --allowlist-mode-reference npm`, rootDir); + expect(providenceStub.args[0][1].gatherFilesConfigReference.allowlistMode).to.equal('npm'); + }); + it('"-D --debug"', async () => { await runCli(`${analyzeCmd} -D`, rootDir); expect(providenceStub.args[0][1].debugEnabled).to.equal(true); @@ -356,7 +366,7 @@ describe('Providence CLI', () => { '--prefix-from pfrom --prefix-to pto', '--output-folder /outp', '--extensions bla', - '--whitelist wl --whitelist-reference wlr', + '--allowlist al --allowlist-reference alr', ].join(' '), rootDir, ); @@ -369,8 +379,8 @@ describe('Providence CLI', () => { }, outputFolder: '/outp', extensions: ['.bla'], - whitelist: [`${rootDir}/wl`], - whitelistReference: [`${rootDir}/wlr`], + allowlist: [`${rootDir}/al`], + allowlistReference: [`${rootDir}/alr`], }); }); }); diff --git a/packages/providence-analytics/test-node/program/services/InputDataService.test.js b/packages/providence-analytics/test-node/program/services/InputDataService.test.js index 2a5536168..d653388d8 100644 --- a/packages/providence-analytics/test-node/program/services/InputDataService.test.js +++ b/packages/providence-analytics/test-node/program/services/InputDataService.test.js @@ -54,10 +54,10 @@ describe('InputDataService', () => { '/test-helpers/project-mocks/importing-target-project', ), ).to.equal(true); - expect(inputDataPerProject[0].entries.length).to.equal(11); + expect(inputDataPerProject[0].entries.length).to.equal(6); expect(inputDataPerProject[0].entries[0].context.code).to.not.be.undefined; expect(inputDataPerProject[0].entries[0].file).to.equal( - './node_modules/exporting-ref-project/index.js', + './target-src/find-customelements/multiple.js', ); }); @@ -215,7 +215,7 @@ describe('InputDataService', () => { expect(globOutput).to.eql(['/fictional/project/index.js']); }); - it('filters npm files entries', async () => { + it('filters npm "files" entries when allowlistMode is "npm"', async () => { mockProject({ './docs/x.js': '', './src/y.js': '', @@ -225,7 +225,9 @@ describe('InputDataService', () => { files: ['*.add.js', 'docs', 'src'], }), }); - const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + allowlistMode: 'npm', + }); expect(globOutput).to.eql([ '/fictional/project/docs/x.js', '/fictional/project/file.add.js', @@ -233,21 +235,80 @@ describe('InputDataService', () => { ]); }); - it('filters .gitignore entries', async () => { + it('filters .gitignore entries when allowlistMode is "git"', async () => { mockProject({ './coverage/file.js': '', './storybook-static/index.js': '', './build/index.js': '', + './shall/pass.js': '', + './keep/it.js': '', '.gitignore': ` /coverage # comment /storybook-static/ build/ +!keep/ + `, + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + allowlistMode: 'git', + }); + expect(globOutput).to.eql([ + '/fictional/project/keep/it.js', + '/fictional/project/shall/pass.js', + ]); + }); + + it('filters no entries when allowlistMode is "all"', async () => { + mockProject({ + './dist/bundle.js': '', + './src/file.js': '', + './package.json': JSON.stringify({ + files: ['dist', 'src'], + }), + '.gitignore': ` +/dist + `, + }); + const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', { + allowlistMode: 'all', + }); + expect(globOutput).to.eql([ + '/fictional/project/dist/bundle.js', + '/fictional/project/src/file.js', + ]); + }); + + it('autodetects allowlistMode', async () => { + mockProject({ + './dist/bundle.js': '', + './package.json': JSON.stringify({ + files: ['dist'], + }), + '.gitignore': ` +/dist `, }); const globOutput = InputDataService.gatherFilesFromDir('/fictional/project'); - expect(globOutput).to.eql([]); + expect(globOutput).to.eql([ + // This means allowlistMode is 'git' + ]); + + restoreOriginalInputDataPaths(); + restoreMockedProjects(); + + mockProject({ + './dist/bundle.js': '', + './package.json': JSON.stringify({ + files: ['dist'], + }), + }); + const globOutput2 = InputDataService.gatherFilesFromDir('/fictional/project'); + expect(globOutput2).to.eql([ + // This means allowlistMode is 'npm' + '/fictional/project/dist/bundle.js', + ]); }); describe('Default filter', () => {