ayo/lion
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(providence-analytics): add --allowlist-mode option

Browse source
This commit is contained in:
Thijs Louisse 2020-08-10 13:44:32 +02:00
parent d6db31b7b2
commit ca6c8e6278
7 changed files with 168 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
.changeset/clever-trains-remember.md Normal file
View file

@ -0,0 +1,7 @@
---
'providence-analytics': minor
---
Allowlist mode: autodetects whether analyzed repository is a "git" or "npm" (published artifact) repository.
Via the cli `--allowlist-mode 'npm|git|all'` and `--allowlist-mode-reference 'npm|git|all'` can be
configured to override the autodetected mode.

29
packages/providence-analytics/src/cli/cli.js
View file

@ -116,11 +116,13 @@ async function cli({ cwd } = {}) {
providenceModule.providence(queryConfig, {
gatherFilesConfig: {
extensions: commander.extensions,
filter: commander.whitelist,
allowlistMode: commander.allowlistMode,
filter: commander.allowlist,
},
gatherFilesConfigReference: {
extensions: commander.extensions,
filter: commander.whitelistReference,
allowlistMode: commander.allowlistModeReference,
filter: commander.allowlistReference,
},
debugEnabled: commander.debug,
queryMethod,
@ -172,12 +174,12 @@ async function cli({ cwd } = {}) {
v => cliHelpers.pathsArrayFromCs(v, cwd),
InputDataService.referenceProjectPaths,
)
.option('-w, --whitelist [whitelist]', `whitelisted paths, like './src, ./packages/*'`, v =>
.option('-a, --allowlist [allowlist]', `allowlisted paths, like './src, ./packages/*'`, v =>
cliHelpers.pathsArrayFromCs(v, cwd),
)
.option(
'--whitelist-reference [whitelist-reference]',
`whitelisted paths for reference, like './src, ./packages/*'`,
'--allowlist-reference [allowlist-reference]',
`allowed paths for reference, like './src, ./packages/*'`,
v => cliHelpers.pathsArrayFromCs(v, cwd),
)
.option(
@ -201,6 +203,19 @@ async function cli({ cwd } = {}) {
without argument, it will act as boolean and include all dependencies.
When a regex is supplied like --target-dependencies /^my-brand-/, it will filter
all packages that comply with the regex`,
)
.option(
'--allowlist-mode [allowlist-mode]',
`Depending on whether we are dealing with a published artifact
(a dependency installed via npm) or a git repository, different paths will be
automatically put in the appropiate mode.
A mode of 'npm' will look at the package.json "files" entry and a mode of
'git' will look at '.gitignore' entry. The mode will be auto detected, but can be overridden
via this option.`,
)
.option(
'--allowlist-mode-reference [allowlist-mode-reference]',
`allowlist mode applied to refernce project`,
);
commander
@ -286,8 +301,8 @@ async function cli({ cwd } = {}) {
prefixCfg,
outputFolder: options.outputFolder,
extensions: commander.extensions,
whitelist: commander.whitelist,
whitelistReference: commander.whitelistReference,
allowlist: commander.allowlist,
allowlistReference: commander.allowlistReference,
})
.then(resolveCli)
.catch(rejectCli);

8
packages/providence-analytics/src/cli/generate-extend-docs-data.js
View file

@ -11,8 +11,8 @@ async function launchProvidenceWithExtendDocs({
prefixCfg,
outputFolder,
extensions,
whitelist,
whitelistReference,
allowlist,
allowlistReference,
}) {
const t0 = performance.now();
@ -21,11 +21,11 @@ async function launchProvidenceWithExtendDocs({
{
gatherFilesConfig: {
extensions: extensions || ['.js'],
filter: whitelist || ['!coverage', '!test'],
filter: allowlist || ['!coverage', '!test'],
},
gatherFilesConfigReference: {
extensions: extensions || ['.js'],
filter: whitelistReference || ['!coverage', '!test'],
filter: allowlistReference || ['!coverage', '!test'],
},
queryMethod: 'ast',
report: false,

58
packages/providence-analytics/src/program/services/InputDataService.js
View file

@ -12,11 +12,17 @@ const { LogService } = require('./LogService.js');
const { AstService } = require('./AstService.js');
const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js');
function getGitIgnorePaths(rootPath) {
let fileContent;
function getGitignoreFile(rootPath) {
try {
fileContent = fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
return fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
} catch (_) {
return undefined;
}
}
function getGitIgnorePaths(rootPath) {
const fileContent = getGitignoreFile(rootPath);
if (!fileContent) {
return [];
}
@ -25,9 +31,28 @@ function getGitIgnorePaths(rootPath) {
if (entry.startsWith('#')) {
return false;
}
if (entry.startsWith('!')) {
return false; // negated folders will be kept
}
return entry.trim().length;
});
return entries;
// normalize entries to be compatible with anymatch
const normalizedEntries = entries.map(e => {
let entry = e;
if (entry.startsWith('/')) {
entry = entry.slice(1);
}
const isFile = entry.indexOf('.') > 0; // index of 0 means hidden file.
if (entry.endsWith('/')) {
entry += '**';
} else if (!isFile) {
entry += '/**';
}
return entry;
});
return normalizedEntries;
}
/**
@ -290,10 +315,27 @@ class InputDataService {
if (!customConfig.omitDefaultFilter) {
cfg.filter = [...this.defaultGatherFilesConfig.filter, ...(customConfig.filter || [])];
}
const allowlistModes = ['npm', 'git', 'all'];
if (customConfig.allowlistMode && !allowlistModes.includes(customConfig.allowlistMode)) {
throw new Error(
`[gatherFilesConfig] Please provide a valid allowListMode like "${allowlistModes.join(
'|',
)}". Found: "${customConfig.allowlistMode}"`,
);
}
const gitIgnorePaths = getGitIgnorePaths(startPath);
const npmPackagePaths = getNpmPackagePaths(startPath);
const removeFilter = gitIgnorePaths.map(p => `!${p}`);
let gitIgnorePaths = [];
let npmPackagePaths = [];
const hasGitIgnore = getGitignoreFile(startPath);
const allowlistMode = cfg.allowlistMode || (hasGitIgnore ? 'git' : 'npm');
if (allowlistMode === 'git') {
gitIgnorePaths = getGitIgnorePaths(startPath);
} else if (allowlistMode === 'npm') {
npmPackagePaths = getNpmPackagePaths(startPath);
}
const removeFilter = gitIgnorePaths;
const keepFilter = npmPackagePaths;
cfg.filter.forEach(filterEntry => {
@ -322,11 +364,9 @@ class InputDataService {
}
return anymatch(keepFilter, localFilePath);
});
if (!filteredGlobRes || !filteredGlobRes.length) {
LogService.warn(`No files found for path '${startPath}'`);
}
return filteredGlobRes;
}

2
packages/providence-analytics/test-helpers/project-mocks/importing-target-project/target-src/match-subclasses/ExtendedComp.js
View file

@ -5,7 +5,7 @@ import MyCompMixin from './internalProxy.js';
export class ExtendedComp extends MyCompMixin(RefClass) {
/**
* Whitelisted members
* allowed members
*/
get getterSetter() {}
set getterSetter(v) {}

26
packages/providence-analytics/test-node/cli/cli.test.js
View file

@ -226,8 +226,8 @@ describe('Providence CLI', () => {
]);
});
it('"-w --whitelist"', async () => {
await runCli(`${analyzeCmd} -w /mocked/path/example-project`, rootDir);
it('"-a --allowlist"', async () => {
await runCli(`${analyzeCmd} -a /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
'/mocked/path/example-project',
@ -236,21 +236,31 @@ describe('Providence CLI', () => {
pathsArrayFromCsStub.resetHistory();
providenceStub.resetHistory();
await runCli(`${analyzeCmd} --whitelist /mocked/path/example-project`, rootDir);
await runCli(`${analyzeCmd} --allowlist /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfig.filter).to.eql([
'/mocked/path/example-project',
]);
});
it('"--whitelist-reference"', async () => {
await runCli(`${analyzeCmd} --whitelist-reference /mocked/path/example-project`, rootDir);
it('"--allowlist-reference"', async () => {
await runCli(`${analyzeCmd} --allowlist-reference /mocked/path/example-project`, rootDir);
expect(pathsArrayFromCsStub.args[0][0]).to.equal('/mocked/path/example-project');
expect(providenceStub.args[0][1].gatherFilesConfigReference.filter).to.eql([
'/mocked/path/example-project',
]);
});
it('--allowlist-mode', async () => {
await runCli(`${analyzeCmd} --allowlist-mode git`, rootDir);
expect(providenceStub.args[0][1].gatherFilesConfig.allowlistMode).to.equal('git');
});
it('--allowlist-mode-reference', async () => {
await runCli(`${analyzeCmd} --allowlist-mode-reference npm`, rootDir);
expect(providenceStub.args[0][1].gatherFilesConfigReference.allowlistMode).to.equal('npm');
});
it('"-D --debug"', async () => {
await runCli(`${analyzeCmd} -D`, rootDir);
expect(providenceStub.args[0][1].debugEnabled).to.equal(true);
@ -356,7 +366,7 @@ describe('Providence CLI', () => {
'--prefix-from pfrom --prefix-to pto',
'--output-folder /outp',
'--extensions bla',
'--whitelist wl --whitelist-reference wlr',
'--allowlist al --allowlist-reference alr',
].join(' '),
rootDir,
);
@ -369,8 +379,8 @@ describe('Providence CLI', () => {
},
outputFolder: '/outp',
extensions: ['.bla'],
whitelist: [`${rootDir}/wl`],
whitelistReference: [`${rootDir}/wlr`],
allowlist: [`${rootDir}/al`],
allowlistReference: [`${rootDir}/alr`],
});
});
});

73
packages/providence-analytics/test-node/program/services/InputDataService.test.js
View file

@ -54,10 +54,10 @@ describe('InputDataService', () => {
'/test-helpers/project-mocks/importing-target-project',
),
).to.equal(true);
expect(inputDataPerProject[0].entries.length).to.equal(11);
expect(inputDataPerProject[0].entries.length).to.equal(6);
expect(inputDataPerProject[0].entries[0].context.code).to.not.be.undefined;
expect(inputDataPerProject[0].entries[0].file).to.equal(
'./node_modules/exporting-ref-project/index.js',
'./target-src/find-customelements/multiple.js',
);
});
@ -215,7 +215,7 @@ describe('InputDataService', () => {
expect(globOutput).to.eql(['/fictional/project/index.js']);
});
it('filters npm files entries', async () => {
it('filters npm "files" entries when allowlistMode is "npm"', async () => {
mockProject({
'./docs/x.js': '',
'./src/y.js': '',
@ -225,7 +225,9 @@ describe('InputDataService', () => {
files: ['*.add.js', 'docs', 'src'],
}),
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'npm',
});
expect(globOutput).to.eql([
'/fictional/project/docs/x.js',
'/fictional/project/file.add.js',
@ -233,21 +235,80 @@ describe('InputDataService', () => {
]);
});
it('filters .gitignore entries', async () => {
it('filters .gitignore entries when allowlistMode is "git"', async () => {
mockProject({
'./coverage/file.js': '',
'./storybook-static/index.js': '',
'./build/index.js': '',
'./shall/pass.js': '',
'./keep/it.js': '',
'.gitignore': `
/coverage
# comment
/storybook-static/
build/
!keep/
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'git',
});
expect(globOutput).to.eql([
'/fictional/project/keep/it.js',
'/fictional/project/shall/pass.js',
]);
});
it('filters no entries when allowlistMode is "all"', async () => {
mockProject({
'./dist/bundle.js': '',
'./src/file.js': '',
'./package.json': JSON.stringify({
files: ['dist', 'src'],
}),
'.gitignore': `
/dist
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project', {
allowlistMode: 'all',
});
expect(globOutput).to.eql([
'/fictional/project/dist/bundle.js',
'/fictional/project/src/file.js',
]);
});
it('autodetects allowlistMode', async () => {
mockProject({
'./dist/bundle.js': '',
'./package.json': JSON.stringify({
files: ['dist'],
}),
'.gitignore': `
/dist
`,
});
const globOutput = InputDataService.gatherFilesFromDir('/fictional/project');
expect(globOutput).to.eql([]);
expect(globOutput).to.eql([
// This means allowlistMode is 'git'
]);
restoreOriginalInputDataPaths();
restoreMockedProjects();
mockProject({
'./dist/bundle.js': '',
'./package.json': JSON.stringify({
files: ['dist'],
}),
});
const globOutput2 = InputDataService.gatherFilesFromDir('/fictional/project');
expect(globOutput2).to.eql([
// This means allowlistMode is 'npm'
'/fictional/project/dist/bundle.js',
]);
});
describe('Default filter', () => {