ayo/lion
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #870 from ing-bank/feat/providenceWhitelistMode

Browse source 
feat(providence-analytics): add --allowlist-mode option
This commit is contained in:
Thijs Louisse 2020-08-12 10:03:02 +02:00 committed by GitHub
commit e374608c95
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 198 additions and 63 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
.changeset/clever-trains-remember.md Normal file
View file

@ -0,0 +1,15 @@
---
'providence-analytics': minor
---
Allowlist modes
#### Features
- Allowlist mode: autodetects whether analyzed repository is a "git" or "npm" (published artifact) repository.
Via the cli `--allowlist-mode 'npm|git|all'` and `--allowlist-mode-reference 'npm|git|all'` can be
configured to override the autodetected mode.
#### Bugfixes
- Clean output extend-docs: strings like '[no-dependency]' will not end up in aggregated result

1
packages/providence-analytics/src/cli/cli-helpers.js
View file

@ -178,4 +178,5 @@ module.exports = {
spawnProcess,
installDeps,
pathsArrayFromCollectionName,
flatten,
};

29
packages/providence-analytics/src/cli/cli.js
View file

@ -116,11 +116,13 @@ async function cli({ cwd } = {}) {
providenceModule.providence(queryConfig, {
gatherFilesConfig: {
extensions: commander.extensions,
filter: commander.whitelist,
allowlistMode: commander.allowlistMode,
filter: commander.allowlist,
},
gatherFilesConfigReference: {
extensions: commander.extensions,
filter: commander.whitelistReference,
allowlistMode: commander.allowlistModeReference,
filter: commander.allowlistReference,
},
debugEnabled: commander.debug,
queryMethod,
@ -172,12 +174,12 @@ async function cli({ cwd } = {}) {
v => cliHelpers.pathsArrayFromCs(v, cwd),
InputDataService.referenceProjectPaths,
)
.option('-w, --whitelist [whitelist]', `whitelisted paths, like './src, ./packages/*'`, v =>
.option('-a, --allowlist [allowlist]', `allowlisted paths, like './src, ./packages/*'`, v =>
cliHelpers.pathsArrayFromCs(v, cwd),
)
.option(
'--whitelist-reference [whitelist-reference]',
`whitelisted paths for reference, like './src, ./packages/*'`,
'--allowlist-reference [allowlist-reference]',
`allowed paths for reference, like './src, ./packages/*'`,
v => cliHelpers.pathsArrayFromCs(v, cwd),
)
.option(
@ -201,6 +203,19 @@ async function cli({ cwd } = {}) {
without argument, it will act as boolean and include all dependencies.
When a regex is supplied like --target-dependencies /^my-brand-/, it will filter
all packages that comply with the regex`,
)
.option(
'--allowlist-mode [allowlist-mode]',
`Depending on whether we are dealing with a published artifact
(a dependency installed via npm) or a git repository, different paths will be
automatically put in the appropiate mode.
A mode of 'npm' will look at the package.json "files" entry and a mode of
'git' will look at '.gitignore' entry. The mode will be auto detected, but can be overridden
via this option.`,
)
.option(
'--allowlist-mode-reference [allowlist-mode-reference]',
`allowlist mode applied to refernce project`,
);
commander
@ -286,8 +301,8 @@ async function cli({ cwd } = {}) {
prefixCfg,
outputFolder: options.outputFolder,
extensions: commander.extensions,
whitelist: commander.whitelist,
whitelistReference: commander.whitelistReference,
allowlist: commander.allowlist,
allowlistReference: commander.allowlistReference,
})
.then(resolveCli)
.catch(rejectCli);

15
packages/providence-analytics/src/cli/generate-extend-docs-data.js
View file

@ -5,14 +5,15 @@ const { performance } = require('perf_hooks');
const { providence } = require('../program/providence.js');
const { QueryService } = require('../program/services/QueryService.js');
const { LogService } = require('../program/services/LogService.js');
const { flatten } = require('./cli-helpers.js');
async function launchProvidenceWithExtendDocs({
referenceProjectPaths,
prefixCfg,
outputFolder,
extensions,
whitelist,
whitelistReference,
allowlist,
allowlistReference,
}) {
const t0 = performance.now();
@ -21,11 +22,11 @@ async function launchProvidenceWithExtendDocs({
{
gatherFilesConfig: {
extensions: extensions || ['.js'],
filter: whitelist || ['!coverage', '!test'],
filter: allowlist || ['!coverage', '!test'],
},
gatherFilesConfigReference: {
extensions: extensions || ['.js'],
filter: whitelistReference || ['!coverage', '!test'],
filter: allowlistReference || ['!coverage', '!test'],
},
queryMethod: 'ast',
report: false,
@ -35,11 +36,13 @@ async function launchProvidenceWithExtendDocs({
);
const outputFilePath = pathLib.join(outputFolder, 'providence-extend-docs-data.json');
const queryOutputs = results.map(result => result.queryOutput).flat();
const queryOutputs = flatten(
results.map(result => result.queryOutput).filter(o => typeof o !== 'string'), // filter out '[no-dependency]' etc.
);
if (fs.existsSync(outputFilePath)) {
fs.unlinkSync(outputFilePath);
}
fs.writeFile(outputFilePath, JSON.stringify(queryOutputs), err => {
fs.writeFile(outputFilePath, JSON.stringify(queryOutputs, null, 2), err => {
if (err) {
throw err;
}

58
packages/providence-analytics/src/program/services/InputDataService.js
View file

@ -12,11 +12,17 @@ const { LogService } = require('./LogService.js');
const { AstService } = require('./AstService.js');
const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js');
function getGitIgnorePaths(rootPath) {
let fileContent;
function getGitignoreFile(rootPath) {
try {
fileContent = fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
return fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
} catch (_) {
return undefined;
}
}
function getGitIgnorePaths(rootPath) {
const fileContent = getGitignoreFile(rootPath);
if (!fileContent) {
return [];
}
@ -25,9 +31,28 @@ function getGitIgnorePaths(rootPath) {
if (entry.startsWith('#')) {
return false;
}
if (entry.startsWith('!')) {
return false; // negated folders will be kept
}
return entry.trim().length;
});
return entries;
// normalize entries to be compatible with anymatch
const normalizedEntries = entries.map(e => {
let entry = e;
if (entry.startsWith('/')) {
entry = entry.slice(1);
}
const isFile = entry.indexOf('.') > 0; // index of 0 means hidden file.
if (entry.endsWith('/')) {
entry += '**';
} else if (!isFile) {
entry += '/**';
}
return entry;
});
return normalizedEntries;
}
/**
@ -290,10 +315,27 @@ class InputDataService {
if (!customConfig.omitDefaultFilter) {
cfg.filter = [...this.defaultGatherFilesConfig.filter, ...(customConfig.filter || [])];
}
const allowlistModes = ['npm', 'git', 'all'];
if (customConfig.allowlistMode && !allowlistModes.includes(customConfig.allowlistMode)) {
throw new Error(
`[gatherFilesConfig] Please provide a valid allowListMode like "${allowlistModes.join(
'|',
)}". Found: "${customConfig.allowlistMode}"`,
);
}
const gitIgnorePaths = getGitIgnorePaths(startPath);
const npmPackagePaths = getNpmPackagePaths(startPath);
const removeFilter = gitIgnorePaths.map(p => `!${p}`);
let gitIgnorePaths = [];
let npmPackagePaths = [];
const hasGitIgnore = getGitignoreFile(startPath);
const allowlistMode = cfg.allowlistMode || (hasGitIgnore ? 'git' : 'npm');
if (allowlistMode === 'git') {
gitIgnorePaths = getGitIgnorePaths(startPath);
} else if (allowlistMode === 'npm') {
npmPackagePaths = getNpmPackagePaths(startPath);
}
const removeFilter = gitIgnorePaths;
const keepFilter = npmPackagePaths;
cfg.filter.forEach(filterEntry => {
@ -322,11 +364,9 @@ class InputDataService {
}
return anymatch(keepFilter, localFilePath);
});
if (!filteredGlobRes || !filteredGlobRes.length) {
LogService.warn(`No files found for path '${startPath}'`);
}
return filteredGlobRes;
}

2
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-classes.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "find-classes",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock__-297820780",
"identifier": "importing-target-project_0.0.2-target-mock__1364353669",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",

2
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-customelements.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "find-customelements",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock__-2006922104",
"identifier": "importing-target-project_0.0.2-target-mock__-1063914889",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",

4
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-exports.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "find-exports",
"requiredAst": "babel",
"identifier": "exporting-ref-project_1.0.0__-1083884764",
"identifier": "exporting-ref-project_1.0.0__309114983",
"targetProject": {
"mainEntry": "./index.js",
"name": "exporting-ref-project",
@ -12,7 +12,7 @@
"commitHash": "[not-a-git-root]"
},
"configuration": {
"metaConfig": null,
"skipFileImports": false,
"gatherFilesConfig": {}
}
}

2
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/find-imports.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "find-imports",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock__139587347",
"identifier": "importing-target-project_0.0.2-target-mock__-1398544254",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",

6
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-imports.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "match-imports",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__453069400",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__1498484172",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",
@ -18,7 +18,9 @@
"commitHash": "[not-a-git-root]"
},
"configuration": {
"gatherFilesConfig": {}
"gatherFilesConfig": {},
"targetProjectResult": null,
"referenceProjectResult": null
}
}
},

2
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-paths.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "match-paths",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__-238486383",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__1241369081",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",

2
packages/providence-analytics/test-helpers/project-mocks-analyzer-outputs/match-subclasses.json
View file

@ -4,7 +4,7 @@
"analyzerMeta": {
"name": "match-subclasses",
"requiredAst": "babel",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__453069400",
"identifier": "importing-target-project_0.0.2-target-mock_+_exporting-ref-project_1.0.0__-933483072",
"targetProject": {
"mainEntry": "./target-src/match-imports/root-level-imports.js",
"name": "importing-target-project",

2
packages/providence-analytics/test-helpers/project-mocks/README.md
View file

@ -9,4 +9,4 @@ Whenever new Analyzers are added, please make sure the needed ingredients for a
end to end test are added to one of the above projects (or both).
Be sure to update 'test-helpers/project-mocks-analyzer-output'.
This can be done by running `npm run test:e2e -- --generate-e2e-mode` once.
This can be done by running `yarn test:node:e2e --generate-e2e-mode` once.

2
packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js
View file

@ -5,7 +5,7 @@ import MyCompMixin from './internalProxy.js';
export class ExtendedComp extends MyCompMixin(RefClass) {
/**
* Whitelisted members
* allowed members
*/
get getterSetter() {}
set getterSetter(v) {}

26
packages/providence-analytics/test-node/cli/cli.test.js
View file

@ -226,8 +226,8 @@ describe('Providence CLI', () => {
]);
});
it('"-w --whitelist"', async () => {
await runCli(`${analyzeCmd} -w /mocked/path/example-project`, rootDir);
it('"-a --allowlist"', async () => {
await runCli(`${analyzeCmd} -a /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
'/mocked/path/example-project',
@ -236,21 +236,31 @@ describe('Providence CLI', () => {
pathsArrayFromCsStub.resetHistory();
providenceStub.resetHistory();
await runCli(`${analyzeCmd} --whitelist /mocked/path/example-project`, rootDir);
await runCli(`${analyzeCmd} --allowlist /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
'/mocked/path/example-project',
]);
});
it('"--whitelist-reference"', async () => {
await runCli(`${analyzeCmd} --whitelist-reference /mocked/path/example-project`, rootDir);
it('"--allowlist-reference"', async () => {
await runCli(`${analyzeCmd} --allowlist-reference /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfigReference.filter).to.eql([
'/mocked/path/example-project',
]);
});
it('--allowlist-mode', async () => {
await runCli(`${analyzeCmd} --allowlist-mode git`, rootDir);
expect(providenceStub.args[0][1].gatherFilesConfig.allowlistMode).to.equal('git');
});
it('--allowlist-mode-reference', async () => {
await runCli(`${analyzeCmd} --allowlist-mode-reference npm`, rootDir);
expect(providenceStub.args[0][1].gatherFilesConfigReference.allowlistMode).to.equal('npm');
});
it('"-D --debug"', async () => {
await runCli(`${analyzeCmd} -D`, rootDir);
expect(providenceStub.args[0][1].debugEnabled).to.equal(true);
@ -356,7 +366,7 @@ describe('Providence CLI', () => {
'--prefix-from pfrom --prefix-to pto',
'--output-folder /outp',
'--extensions bla',
'--whitelist wl --whitelist-reference wlr',
'--allowlist al --allowlist-reference alr',
].join(' '),
rootDir,
);
@ -369,8 +379,8 @@ describe('Providence CLI', () => {
},
outputFolder: '/outp',
extensions: ['.bla'],
whitelist: [`${rootDir}/wl`],
whitelistReference: [`${rootDir}/wlr`],
allowlist: [`${rootDir}/al`],
allowlistReference: [`${rootDir}/alr`],
});
});
});

20
packages/providence-analytics/test-node/program/analyzers/e2e/all-analyzers.e2e.js
View file

@ -3,26 +3,13 @@ const { expect } = require('chai');
const { providence } = require('../../../../src/program/providence.js');
const { QueryService } = require('../../../../src/program/services/QueryService.js');
const { ReportService } = require('../../../../src/program/services/ReportService.js');
const { LogService } = require('../../../../src/program/services/LogService.js');
const {
mockWriteToJson,
restoreWriteToJson,
} = require('../../../../test-helpers/mock-report-service-helpers.js');
const {
suppressNonCriticalLogs,
restoreSuppressNonCriticalLogs,
} = require('../../../../test-helpers/mock-log-service-helpers.js');
describe('Analyzers file-system integration', () => {
before(() => {
suppressNonCriticalLogs();
});
after(() => {
restoreSuppressNonCriticalLogs();
});
const generateE2eMode = process.argv.includes('--generate-e2e-mode');
const queryResults = [];
@ -116,15 +103,16 @@ describe('Analyzers file-system integration', () => {
const findExportsQueryConfig = QueryService.getQueryConfigFromAnalyzer(analyzerName);
await providence(findExportsQueryConfig, providenceConfig);
if (generateE2eMode) {
LogService.info(
console.info(
'Successfully created mocks. Do not forget to rerun tests now without "--generate-e2e-mode"',
);
return;
}
// eslint-disable-next-line import/no-dynamic-require, global-require
const expectedOutput = require(`../../../../test-helpers/project-mocks-analyzer-outputs/${analyzerName}.json`);
const queryResult = JSON.parse(JSON.stringify(queryResults[0])).queryOutput;
expect(queryResult).to.eql(expectedOutput.queryOutput);
const { queryOutput } = JSON.parse(JSON.stringify(queryResults[0]));
expect(queryOutput).not.to.eql([]);
expect(queryOutput).to.eql(expectedOutput.queryOutput);
});
}
});

73
packages/providence-analytics/test-node/program/services/InputDataService.test.js
View file

@ -54,10 +54,10 @@ describe('InputDataService', () => {
'/test-helpers/project-mocks/importing-target-project',
),
).to.equal(true);
expect(inputDataPerProject[0].entries.length).to.equal(11);
expect(inputDataPerProject[0].entries.length).to.equal(6);
expect(inputDataPerProject[0].entries[0].context.code).to.not.be.undefined;
expect(inputDataPerProject[0].entries[0].file).to.equal(
'./node_modules/exporting-ref-project/index.js',
'./target-src/find-customelements/multiple.js',
);
});
@ -215,7 +215,7 @@ describe('InputDataService', () => {
expect(globOutput).to.eql(['/fictional/project/index.js']);
});
it('filters npm files entries', async () => {
it('filters npm "files" entries when allowlistMode is "npm"', async () => {
mockProject({
'./docs/x.js': '',
'./src/y.js': '',
@ -225,7 +225,9 @@ describe('InputDataService', () => {
files: ['*.add.js', 'docs', 'src'],
}),
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'npm',
});
expect(globOutput).to.eql([
'/fictional/project/docs/x.js',
'/fictional/project/file.add.js',
@ -233,21 +235,80 @@ describe('InputDataService', () => {
]);
});
it('filters .gitignore entries', async () => {
it('filters .gitignore entries when allowlistMode is "git"', async () => {
mockProject({
'./coverage/file.js': '',
'./storybook-static/index.js': '',
'./build/index.js': '',
'./shall/pass.js': '',
'./keep/it.js': '',
'.gitignore': `
/coverage
# comment
/storybook-static/
build/
!keep/
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'git',
});
expect(globOutput).to.eql([
'/fictional/project/keep/it.js',
'/fictional/project/shall/pass.js',
]);
});
it('filters no entries when allowlistMode is "all"', async () => {
mockProject({
'./dist/bundle.js': '',
'./src/file.js': '',
'./package.json': JSON.stringify({
files: ['dist', 'src'],
}),
'.gitignore': `
/dist
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'all',
});
expect(globOutput).to.eql([
'/fictional/project/dist/bundle.js',
'/fictional/project/src/file.js',
]);
});
it('autodetects allowlistMode', async () => {
mockProject({
'./dist/bundle.js': '',
'./package.json': JSON.stringify({
files: ['dist'],
}),
'.gitignore': `
/dist
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
expect(globOutput).to.eql([]);
expect(globOutput).to.eql([
// This means allowlistMode is 'git'
]);
restoreOriginalInputDataPaths();
restoreMockedProjects();
mockProject({
'./dist/bundle.js': '',
'./package.json': JSON.stringify({
files: ['dist'],
}),
});
const globOutput2 = InputDataService.gatherFilesFromDir('/fictional/project');
expect(globOutput2).to.eql([
// This means allowlistMode is 'npm'
'/fictional/project/dist/bundle.js',
]);
});
describe('Default filter', () => {