Merge pull request #870 from ing-bank/feat/providenceWhitelistMode

feat(providence-analytics): add --allowlist-mode option

.changeset/clever-trains-remember.md

@@ -0,0 +1,15 @@
+---
+'providence-analytics': minor
+---
+
+Allowlist modes
+
+#### Features
+
+- Allowlist mode: autodetects whether analyzed repository is a "git" or "npm" (published artifact) repository.
+  Via the cli `--allowlist-mode 'npm|git|all'` and `--allowlist-mode-reference 'npm|git|all'` can be
+  configured to override the autodetected mode.
+
+#### Bugfixes
+
+- Clean output extend-docs: strings like '[no-dependency]' will not end up in aggregated result

packages/providence-analytics/src/cli/cli-helpers.js

@@ -178,4 +178,5 @@ module.exports = {
   spawnProcess,
   installDeps,
   pathsArrayFromCollectionName,
+  flatten,
 };

packages/providence-analytics/src/cli/cli.js

@@ -116,11 +116,13 @@ async function cli({ cwd } = {}) {
     providenceModule.providence(queryConfig, {
       gatherFilesConfig: {
         extensions: commander.extensions,
-        filter: commander.whitelist,
+        allowlistMode: commander.allowlistMode,
+        filter: commander.allowlist,
       },
       gatherFilesConfigReference: {
         extensions: commander.extensions,
-        filter: commander.whitelistReference,
+        allowlistMode: commander.allowlistModeReference,
+        filter: commander.allowlistReference,
       },
       debugEnabled: commander.debug,
       queryMethod,
@@ -172,12 +174,12 @@ async function cli({ cwd } = {}) {
       v => cliHelpers.pathsArrayFromCs(v, cwd),
       InputDataService.referenceProjectPaths,
     )
-    .option('-w, --whitelist [whitelist]', `whitelisted paths, like './src, ./packages/*'`, v =>
+    .option('-a, --allowlist [allowlist]', `allowlisted paths, like './src, ./packages/*'`, v =>
       cliHelpers.pathsArrayFromCs(v, cwd),
     )
     .option(
-      '--whitelist-reference [whitelist-reference]',
+      '--allowlist-reference [allowlist-reference]',
-      `whitelisted paths for reference, like './src, ./packages/*'`,
+      `allowed paths for reference, like './src, ./packages/*'`,
       v => cliHelpers.pathsArrayFromCs(v, cwd),
     )
     .option(
@@ -201,6 +203,19 @@ async function cli({ cwd } = {}) {
     without argument, it will act as boolean and include all dependencies.
     When a regex is supplied like --target-dependencies /^my-brand-/, it will filter
     all packages that comply with the regex`,
+    )
+    .option(
+      '--allowlist-mode [allowlist-mode]',
+      `Depending on whether we are dealing with a published artifact
+      (a dependency installed via npm) or a git repository, different paths will be
+      automatically put in the appropiate mode.
+      A mode of 'npm' will look at the package.json "files" entry and a mode of
+      'git' will look at '.gitignore' entry. The mode will be auto detected, but can be overridden
+      via this option.`,
+    )
+    .option(
+      '--allowlist-mode-reference [allowlist-mode-reference]',
+      `allowlist mode applied to refernce project`,
     );
 
   commander
@@ -286,8 +301,8 @@ async function cli({ cwd } = {}) {
           prefixCfg,
           outputFolder: options.outputFolder,
           extensions: commander.extensions,
-          whitelist: commander.whitelist,
+          allowlist: commander.allowlist,
-          whitelistReference: commander.whitelistReference,
+          allowlistReference: commander.allowlistReference,
         })
         .then(resolveCli)
         .catch(rejectCli);

packages/providence-analytics/src/cli/generate-extend-docs-data.js

@@ -5,14 +5,15 @@ const { performance } = require('perf_hooks');
 const { providence } = require('../program/providence.js');
 const { QueryService } = require('../program/services/QueryService.js');
 const { LogService } = require('../program/services/LogService.js');
+const { flatten } = require('./cli-helpers.js');
 
 async function launchProvidenceWithExtendDocs({
   referenceProjectPaths,
   prefixCfg,
   outputFolder,
   extensions,
-  whitelist,
+  allowlist,
-  whitelistReference,
+  allowlistReference,
 }) {
   const t0 = performance.now();
 
@@ -21,11 +22,11 @@ async function launchProvidenceWithExtendDocs({
     {
       gatherFilesConfig: {
         extensions: extensions || ['.js'],
-        filter: whitelist || ['!coverage', '!test'],
+        filter: allowlist || ['!coverage', '!test'],
       },
       gatherFilesConfigReference: {
         extensions: extensions || ['.js'],
-        filter: whitelistReference || ['!coverage', '!test'],
+        filter: allowlistReference || ['!coverage', '!test'],
       },
       queryMethod: 'ast',
       report: false,
@@ -35,11 +36,13 @@ async function launchProvidenceWithExtendDocs({
   );
 
   const outputFilePath = pathLib.join(outputFolder, 'providence-extend-docs-data.json');
-  const queryOutputs = results.map(result => result.queryOutput).flat();
+  const queryOutputs = flatten(
+    results.map(result => result.queryOutput).filter(o => typeof o !== 'string'), // filter out '[no-dependency]' etc.
+  );
   if (fs.existsSync(outputFilePath)) {
     fs.unlinkSync(outputFilePath);
   }
-  fs.writeFile(outputFilePath, JSON.stringify(queryOutputs), err => {
+  fs.writeFile(outputFilePath, JSON.stringify(queryOutputs, null, 2), err => {
     if (err) {
       throw err;
     }

packages/providence-analytics/src/program/services/InputDataService.js

@@ -12,11 +12,17 @@ const { LogService } = require('./LogService.js');
 const { AstService } = require('./AstService.js');
 const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js');
 
-function getGitIgnorePaths(rootPath) {
+function getGitignoreFile(rootPath) {
-  let fileContent;
   try {
-    fileContent = fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
+    return fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
   } catch (_) {
+    return undefined;
+  }
+}
+
+function getGitIgnorePaths(rootPath) {
+  const fileContent = getGitignoreFile(rootPath);
+  if (!fileContent) {
     return [];
   }
 
@@ -25,9 +31,28 @@ function getGitIgnorePaths(rootPath) {
     if (entry.startsWith('#')) {
       return false;
     }
+    if (entry.startsWith('!')) {
+      return false; // negated folders will be kept
+    }
     return entry.trim().length;
   });
-  return entries;
+
+  // normalize entries to be compatible with anymatch
+  const normalizedEntries = entries.map(e => {
+    let entry = e;
+
+    if (entry.startsWith('/')) {
+      entry = entry.slice(1);
+    }
+    const isFile = entry.indexOf('.') > 0; // index of 0 means hidden file.
+    if (entry.endsWith('/')) {
+      entry += '**';
+    } else if (!isFile) {
+      entry += '/**';
+    }
+    return entry;
+  });
+  return normalizedEntries;
 }
 
 /**
@@ -290,10 +315,27 @@ class InputDataService {
     if (!customConfig.omitDefaultFilter) {
       cfg.filter = [...this.defaultGatherFilesConfig.filter, ...(customConfig.filter || [])];
     }
+    const allowlistModes = ['npm', 'git', 'all'];
+    if (customConfig.allowlistMode && !allowlistModes.includes(customConfig.allowlistMode)) {
+      throw new Error(
+        `[gatherFilesConfig] Please provide a valid allowListMode like "${allowlistModes.join(
+          '|',
+        )}". Found: "${customConfig.allowlistMode}"`,
+      );
+    }
 
-    const gitIgnorePaths = getGitIgnorePaths(startPath);
+    let gitIgnorePaths = [];
-    const npmPackagePaths = getNpmPackagePaths(startPath);
+    let npmPackagePaths = [];
-    const removeFilter = gitIgnorePaths.map(p => `!${p}`);
+
+    const hasGitIgnore = getGitignoreFile(startPath);
+    const allowlistMode = cfg.allowlistMode || (hasGitIgnore ? 'git' : 'npm');
+
+    if (allowlistMode === 'git') {
+      gitIgnorePaths = getGitIgnorePaths(startPath);
+    } else if (allowlistMode === 'npm') {
+      npmPackagePaths = getNpmPackagePaths(startPath);
+    }
+    const removeFilter = gitIgnorePaths;
     const keepFilter = npmPackagePaths;
 
     cfg.filter.forEach(filterEntry => {
@@ -322,11 +364,9 @@ class InputDataService {
       }
       return anymatch(keepFilter, localFilePath);
     });
-
     if (!filteredGlobRes || !filteredGlobRes.length) {
       LogService.warn(`No files found for path '${startPath}'`);
     }
-
     return filteredGlobRes;
   }
 

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-classes.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "find-classes",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock__-297820780",
+      "identifier": "importing-target-project_0.0.2-target-mock__1364353669",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-customelements.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "find-customelements",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock__-2006922104",
+      "identifier": "importing-target-project_0.0.2-target-mock__-1063914889",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-exports.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "find-exports",
       "requiredAst": "babel",
-      "identifier": "exporting-ref-project_1.0.0__-1083884764",
+      "identifier": "exporting-ref-project_1.0.0__309114983",
       "targetProject": {
         "mainEntry": "./index.js",
         "name": "exporting-ref-project",
@@ -12,7 +12,7 @@
         "commitHash": "[not-a-git-root]"
       },
       "configuration": {
-        "metaConfig": null,
+        "skipFileImports": false,
         "gatherFilesConfig": {}
       }
     }

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-imports.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "find-imports",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock__139587347",
+      "identifier": "importing-target-project_0.0.2-target-mock__-1398544254",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-imports.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "match-imports",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__453069400",
+      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__1498484172",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",
@@ -18,7 +18,9 @@
         "commitHash": "[not-a-git-root]"
       },
       "configuration": {
-        "gatherFilesConfig": {}
+        "gatherFilesConfig": {},
+        "targetProjectResult": null,
+        "referenceProjectResult": null
       }
     }
   },

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-paths.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "match-paths",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__-238486383",
+      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__1241369081",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",

packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-subclasses.json

@@ -4,7 +4,7 @@
     "analyzerMeta": {
       "name": "match-subclasses",
       "requiredAst": "babel",
-      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__453069400",
+      "identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__-933483072",
       "targetProject": {
         "mainEntry": "./target-src/match-imports/root-level-imports.js",
         "name": "importing-target-project",

packages/providence-analytics/test-helpers/project-mocks/README.md

@@ -9,4 +9,4 @@ Whenever new Analyzers are added, please make sure the needed ingredients for a
 end to end test are added to one of the above projects (or both).
 
 Be sure to update 'test-helpers/project-mocks-analyzer-output'.
-This can be done by running `npm run test:e2e -- --generate-e2e-mode` once.
+This can be done by running `yarn test:node:e2e --generate-e2e-mode` once.

packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js

@@ -5,7 +5,7 @@ import MyCompMixin from './internalProxy.js';
 
 export class ExtendedComp extends MyCompMixin(RefClass) {
   /**
-   * Whitelisted members
+   * allowed members
    */
   get getterSetter() {}
   set getterSetter(v) {}

packages/providence-analytics/test-node/cli/cli.test.js

@@ -226,8 +226,8 @@ describe('Providence CLI', () => {
       ]);
     });
 
-    it('"-w --whitelist"', async () => {
+    it('"-a --allowlist"', async () => {
-      await runCli(`${analyzeCmd} -w /mocked/path/example-project`, rootDir);
+      await runCli(`${analyzeCmd} -a /mocked/path/example-project`, rootDir);
       expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
       expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
         '/mocked/path/example-project',
@@ -236,21 +236,31 @@ describe('Providence CLI', () => {
       pathsArrayFromCsStub.resetHistory();
       providenceStub.resetHistory();
 
-      await runCli(`${analyzeCmd} --whitelist /mocked/path/example-project`, rootDir);
+      await runCli(`${analyzeCmd} --allowlist /mocked/path/example-project`, rootDir);
       expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
       expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
         '/mocked/path/example-project',
       ]);
     });
 
-    it('"--whitelist-reference"', async () => {
+    it('"--allowlist-reference"', async () => {
-      await runCli(`${analyzeCmd} --whitelist-reference /mocked/path/example-project`, rootDir);
+      await runCli(`${analyzeCmd} --allowlist-reference /mocked/path/example-project`, rootDir);
       expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
       expect(providenceStub.args[0][1].gatherFilesConfigReference.filter).to.eql([
         '/mocked/path/example-project',
       ]);
     });
 
+    it('--allowlist-mode', async () => {
+      await runCli(`${analyzeCmd} --allowlist-mode git`, rootDir);
+      expect(providenceStub.args[0][1].gatherFilesConfig.allowlistMode).to.equal('git');
+    });
+
+    it('--allowlist-mode-reference', async () => {
+      await runCli(`${analyzeCmd} --allowlist-mode-reference npm`, rootDir);
+      expect(providenceStub.args[0][1].gatherFilesConfigReference.allowlistMode).to.equal('npm');
+    });
+
     it('"-D --debug"', async () => {
       await runCli(`${analyzeCmd} -D`, rootDir);
       expect(providenceStub.args[0][1].debugEnabled).to.equal(true);
@@ -356,7 +366,7 @@ describe('Providence CLI', () => {
             '--prefix-from pfrom --prefix-to pto',
             '--output-folder /outp',
             '--extensions bla',
-            '--whitelist wl --whitelist-reference wlr',
+            '--allowlist al --allowlist-reference alr',
           ].join(' '),
           rootDir,
         );
@@ -369,8 +379,8 @@ describe('Providence CLI', () => {
           },
           outputFolder: '/outp',
           extensions: ['.bla'],
-          whitelist: [`${rootDir}/wl`],
+          allowlist: [`${rootDir}/al`],
-          whitelistReference: [`${rootDir}/wlr`],
+          allowlistReference: [`${rootDir}/alr`],
         });
       });
     });

packages/providence-analytics/test-node/program/analyzers/e2e/all-analyzers.e2e.js

@@ -3,26 +3,13 @@ const { expect } = require('chai');
 const { providence } = require('../../../../src/program/providence.js');
 const { QueryService } = require('../../../../src/program/services/QueryService.js');
 const { ReportService } = require('../../../../src/program/services/ReportService.js');
-const { LogService } = require('../../../../src/program/services/LogService.js');
 
 const {
   mockWriteToJson,
   restoreWriteToJson,
 } = require('../../../../test-helpers/mock-report-service-helpers.js');
-const {
-  suppressNonCriticalLogs,
-  restoreSuppressNonCriticalLogs,
-} = require('../../../../test-helpers/mock-log-service-helpers.js');
 
 describe('Analyzers file-system integration', () => {
-  before(() => {
-    suppressNonCriticalLogs();
-  });
-
-  after(() => {
-    restoreSuppressNonCriticalLogs();
-  });
-
   const generateE2eMode = process.argv.includes('--generate-e2e-mode');
 
   const queryResults = [];
@@ -116,15 +103,16 @@ describe('Analyzers file-system integration', () => {
       const findExportsQueryConfig = QueryService.getQueryConfigFromAnalyzer(analyzerName);
       await providence(findExportsQueryConfig, providenceConfig);
       if (generateE2eMode) {
-        LogService.info(
+        console.info(
           'Successfully created mocks. Do not forget to rerun tests now without "--generate-e2e-mode"',
         );
         return;
       }
       // eslint-disable-next-line import/no-dynamic-require, global-require
       const expectedOutput = require(`../../../../test-helpers/project-mocks-analyzer-outputs/${analyzerName}.json`);
-      const queryResult = JSON.parse(JSON.stringify(queryResults[0])).queryOutput;
+      const { queryOutput } = JSON.parse(JSON.stringify(queryResults[0]));
-      expect(queryResult).to.eql(expectedOutput.queryOutput);
+      expect(queryOutput).not.to.eql([]);
+      expect(queryOutput).to.eql(expectedOutput.queryOutput);
     });
   }
 });

packages/providence-analytics/test-node/program/services/InputDataService.test.js

@@ -54,10 +54,10 @@ describe('InputDataService', () => {
           '/test-helpers/project-mocks/importing-target-project',
         ),
       ).to.equal(true);
-      expect(inputDataPerProject[0].entries.length).to.equal(11);
+      expect(inputDataPerProject[0].entries.length).to.equal(6);
       expect(inputDataPerProject[0].entries[0].context.code).to.not.be.undefined;
       expect(inputDataPerProject[0].entries[0].file).to.equal(
-        './node_modules/exporting-ref-project/index.js',
+        './target-src/find-customelements/multiple.js',
       );
     });
 
@@ -215,7 +215,7 @@ describe('InputDataService', () => {
           expect(globOutput).to.eql(['/fictional/project/index.js']);
         });
 
-        it('filters npm files entries', async () => {
+        it('filters npm "files" entries when allowlistMode is "npm"', async () => {
           mockProject({
             './docs/x.js': '',
             './src/y.js': '',
@@ -225,7 +225,9 @@ describe('InputDataService', () => {
               files: ['*.add.js', 'docs', 'src'],
             }),
           });
-          const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
+          const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
+            allowlistMode: 'npm',
+          });
           expect(globOutput).to.eql([
             '/fictional/project/docs/x.js',
             '/fictional/project/file.add.js',
@@ -233,21 +235,80 @@ describe('InputDataService', () => {
           ]);
         });
 
-        it('filters .gitignore entries', async () => {
+        it('filters .gitignore entries when allowlistMode is "git"', async () => {
           mockProject({
             './coverage/file.js': '',
             './storybook-static/index.js': '',
             './build/index.js': '',
+            './shall/pass.js': '',
+            './keep/it.js': '',
             '.gitignore': `
 /coverage
 # comment
 /storybook-static/
 
 build/
+!keep/
+            `,
+          });
+          const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
+            allowlistMode: 'git',
+          });
+          expect(globOutput).to.eql([
+            '/fictional/project/keep/it.js',
+            '/fictional/project/shall/pass.js',
+          ]);
+        });
+
+        it('filters no entries when allowlistMode is "all"', async () => {
+          mockProject({
+            './dist/bundle.js': '',
+            './src/file.js': '',
+            './package.json': JSON.stringify({
+              files: ['dist', 'src'],
+            }),
+            '.gitignore': `
+/dist
+            `,
+          });
+          const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
+            allowlistMode: 'all',
+          });
+          expect(globOutput).to.eql([
+            '/fictional/project/dist/bundle.js',
+            '/fictional/project/src/file.js',
+          ]);
+        });
+
+        it('autodetects allowlistMode', async () => {
+          mockProject({
+            './dist/bundle.js': '',
+            './package.json': JSON.stringify({
+              files: ['dist'],
+            }),
+            '.gitignore': `
+/dist
             `,
           });
           const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
-          expect(globOutput).to.eql([]);
+          expect(globOutput).to.eql([
+            // This means allowlistMode is 'git'
+          ]);
+
+          restoreOriginalInputDataPaths();
+          restoreMockedProjects();
+
+          mockProject({
+            './dist/bundle.js': '',
+            './package.json': JSON.stringify({
+              files: ['dist'],
+            }),
+          });
+          const globOutput2 = InputDataService.gatherFilesFromDir('/fictional/project');
+          expect(globOutput2).to.eql([
+            // This means allowlistMode is 'npm'
+            '/fictional/project/dist/bundle.js',
+          ]);
         });
 
         describe('Default filter', () => {