feat(providence-analytics): monorepo support extend-docs

This commit is contained in:
Thijs Louisse 2020-10-13 15:24:24 +02:00 committed by GitHub
parent f0d11fee4a
commit 2dc85b14d3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 552 additions and 148 deletions

View file

@ -0,0 +1,13 @@
---
'providence-analytics': minor
---
Monorepo support for extend-docs
### Features
- add monorepo support for extend-docs
### Fixes
- allow custom element and class definitions to be in same file for 'match-paths'

View file

@ -10,7 +10,7 @@ const { QueryService } = require('../program/services/QueryService.js');
const { InputDataService } = require('../program/services/InputDataService.js');
const promptModule = require('./prompt-analyzer-menu.js');
const cliHelpers = require('./cli-helpers.js');
const extendDocsModule = require('./generate-extend-docs-data.js');
const extendDocsModule = require('./launch-providence-with-extend-docs.js');
const { toPosixPath } = require('../program/utils/to-posix-path.js');
const { extensionsFromCs, setQueryMethod, targetDefault, installDeps } = cliHelpers;
@ -304,6 +304,7 @@ async function cli({ cwd } = {}) {
extensions: commander.extensions,
allowlist: commander.allowlist,
allowlistReference: commander.allowlistReference,
cwd,
})
.then(resolveCli)
.catch(rejectCli);

View file

@ -1,56 +0,0 @@
/* eslint-disable import/no-extraneous-dependencies */
const fs = require('fs');
const pathLib = require('path');
const { performance } = require('perf_hooks');
const { providence } = require('../program/providence.js');
const { QueryService } = require('../program/services/QueryService.js');
const { LogService } = require('../program/services/LogService.js');
const { flatten } = require('./cli-helpers.js');
async function launchProvidenceWithExtendDocs({
referenceProjectPaths,
prefixCfg,
outputFolder,
extensions,
allowlist,
allowlistReference,
}) {
const t0 = performance.now();
const results = await providence(
QueryService.getQueryConfigFromAnalyzer('match-paths', { prefix: prefixCfg }),
{
gatherFilesConfig: {
extensions: extensions || ['.js'],
allowlist: allowlist || ['!coverage', '!test'],
},
gatherFilesConfigReference: {
extensions: extensions || ['.js'],
allowlist: allowlistReference || ['!coverage', '!test'],
},
queryMethod: 'ast',
report: false,
targetProjectPaths: [pathLib.resolve(process.cwd())],
referenceProjectPaths,
},
);
const outputFilePath = pathLib.join(outputFolder, 'providence-extend-docs-data.json');
const queryOutputs = flatten(
results.map(result => result.queryOutput).filter(o => typeof o !== 'string'), // filter out '[no-dependency]' etc.
);
if (fs.existsSync(outputFilePath)) {
fs.unlinkSync(outputFilePath);
}
fs.writeFile(outputFilePath, JSON.stringify(queryOutputs, null, 2), err => {
if (err) {
throw err;
}
});
const t1 = performance.now();
LogService.info(`"extend-docs" completed in ${Math.round((t1 - t0) / 1000)} seconds`);
}
module.exports = {
launchProvidenceWithExtendDocs,
};

View file

@ -0,0 +1,121 @@
/* eslint-disable import/no-extraneous-dependencies */
const fs = require('fs');
const pathLib = require('path');
const { performance } = require('perf_hooks');
const { providence } = require('../program/providence.js');
const { QueryService } = require('../program/services/QueryService.js');
const { InputDataService } = require('../program/services/InputDataService.js');
const { LogService } = require('../program/services/LogService.js');
const { flatten } = require('./cli-helpers.js');
async function getExtendDocsResults({
referenceProjectPaths,
prefixCfg,
extensions,
allowlist,
allowlistReference,
cwd,
}) {
const results = await providence(
QueryService.getQueryConfigFromAnalyzer('match-paths', { prefix: prefixCfg }),
{
gatherFilesConfig: {
extensions: extensions || ['.js'],
allowlist: allowlist || ['!coverage', '!test'],
},
gatherFilesConfigReference: {
extensions: extensions || ['.js'],
allowlist: allowlistReference || ['!coverage', '!test'],
},
queryMethod: 'ast',
report: false,
targetProjectPaths: [pathLib.resolve(cwd)],
referenceProjectPaths,
},
);
const queryOutputs = flatten(
results.map(result => result.queryOutput).filter(o => typeof o !== 'string'), // filter out '[no-dependency]' etc.
);
/**
* @param {string} pathStr ./packages/lea-tabs/lea-tabs.js
* @param {string[]} pkgs ['packages/lea-tabs', ...]
*/
function replaceToMonoRepoPath(pathStr, pkgs) {
let result = pathStr;
pkgs.some(({ path: p, name }) => {
// for instance ./packages/lea-tabs/lea-tabs.js starts with 'packages/lea-tabs'
const normalizedP = `./${p}`;
if (pathStr.startsWith(normalizedP)) {
const localPath = pathStr.replace(normalizedP, ''); // 'lea-tabs.js'
result = `${name}/${localPath}`; // 'lea-tabs/lea-tabs.js'
return true;
}
return false;
});
return result;
}
const pkgs = InputDataService.getMonoRepoPackages(cwd);
if (pkgs) {
queryOutputs.forEach(resultObj => {
if (resultObj.variable) {
resultObj.variable.paths.forEach(pathObj => {
// eslint-disable-next-line no-param-reassign
pathObj.to = replaceToMonoRepoPath(pathObj.to, pkgs);
});
}
if (resultObj.tag) {
resultObj.tag.paths.forEach(pathObj => {
// eslint-disable-next-line no-param-reassign
pathObj.to = replaceToMonoRepoPath(pathObj.to, pkgs);
});
}
});
}
return queryOutputs;
}
async function launchProvidenceWithExtendDocs({
referenceProjectPaths,
prefixCfg,
outputFolder,
extensions,
allowlist,
allowlistReference,
cwd = process.cwd(),
}) {
const t0 = performance.now();
const queryOutputs = await getExtendDocsResults({
referenceProjectPaths,
prefixCfg,
extensions,
allowlist,
allowlistReference,
cwd,
});
// Write results
const outputFilePath = pathLib.join(outputFolder, 'providence-extend-docs-data.json');
if (fs.existsSync(outputFilePath)) {
fs.unlinkSync(outputFilePath);
}
fs.writeFile(outputFilePath, JSON.stringify(queryOutputs, null, 2), err => {
if (err) {
throw err;
}
});
const t1 = performance.now();
LogService.info(`"extend-docs" completed in ${Math.round((t1 - t0) / 1000)} seconds`);
}
module.exports = {
launchProvidenceWithExtendDocs,
getExtendDocsResults,
};

View file

@ -216,7 +216,8 @@ function getTagPaths(
let targetResult;
targetFindCustomelementsResult.queryOutput.some(({ file, result }) => {
const targetPathMatch = result.find(entry => {
const sameRoot = entry.rootFile.file === targetMatchedFile;
const sameRoot =
entry.rootFile.file === targetMatchedFile || entry.rootFile.file === '[current]';
const sameIdentifier = entry.rootFile.specifier === toClass;
return sameRoot && sameIdentifier;
});

View file

@ -1,6 +1,4 @@
import { ClassMethod } from "@babel/types";
import { ProjectReference } from "typescript";
import { ProjectReference } from 'typescript';
export interface RootFile {
/** the file path containing declaration, for instance './target-src/direct-imports.js'. Can also contain keyword '[current]' */
@ -9,7 +7,6 @@ export interface RootFile {
specifier: string;
}
export interface AnalyzerResult {
/** meta info object */
meta: Meta;
@ -24,7 +21,6 @@ export interface AnalyzerOutputFile {
result: array;
}
// TODO: make sure that data structures of JSON output (generated in ReportService)
// and data structure generated in Analyzer.prototype._finalize match exactly (move logic from ReportSerivce to _finalize)
// so that these type definitions can be used to generate a json schema: https://www.npmjs.com/package/typescript-json-schema
@ -109,7 +105,6 @@ export interface MatchedExportSpecifier extends AnalyzerResult {
id: string;
}
// "find-customelements"
export interface FindCustomelementsAnalyzerResult extends AnalyzerResult {
@ -181,7 +176,6 @@ export interface FindExportsAnalyzerEntry {
rootFileMap: RootFileMapEntry[];
}
export interface RootFileMapEntry {
/** This is the local name in the file we track from */
currentFileSpecifier: string;
@ -287,8 +281,6 @@ export interface SuperClass {
rootFile: RootFile;
}
export interface FindClassesConfig {
/** search target paths */
targetProjectPath: string;
@ -300,9 +292,6 @@ export interface AnalyzerConfig {
gatherFilesConfig: GatherFilesConfig;
}
export interface MatchAnalyzerConfig extends AnalyzerConfig {
/** reference project path, used to match reference against target */
referenceProjectPath: string;

View file

@ -14,6 +14,57 @@ const { AstService } = require('./AstService.js');
const { getFilePathRelativeFromRoot } = require('../utils/get-file-path-relative-from-root.js');
const { toPosixPath } = require('../utils/to-posix-path.js');
// TODO: memoize
function getPackageJson(rootPath) {
try {
const fileContent = fs.readFileSync(`${rootPath}/package.json`, 'utf8');
return JSON.parse(fileContent);
} catch (_) {
return undefined;
}
}
function getLernaJson(rootPath) {
try {
const fileContent = fs.readFileSync(`${rootPath}/lerna.json`, 'utf8');
return JSON.parse(fileContent);
} catch (_) {
return undefined;
}
}
/**
*
* @param {string[]} list
* @param {string} rootPath
* @returns {{path:string, name:string}[]}
*/
function getPathsFromGlobList(list, rootPath) {
const results = [];
list.forEach(pathOrGlob => {
if (!pathOrGlob.endsWith('/')) {
// eslint-disable-next-line no-param-reassign
pathOrGlob = `${pathOrGlob}/`;
}
if (pathOrGlob.includes('*')) {
const globResults = glob.sync(pathOrGlob, { cwd: rootPath, absolute: false });
globResults.forEach(r => {
results.push(r);
});
} else {
results.push(pathOrGlob);
}
});
return results.map(path => {
const packageRoot = pathLib.resolve(rootPath, path);
const basename = pathLib.basename(path);
const pkgJson = getPackageJson(packageRoot);
const name = (pkgJson && pkgJson.name) || basename;
return { name, path };
});
}
function getGitignoreFile(rootPath) {
try {
return fs.readFileSync(`${rootPath}/.gitignore`, 'utf8');
@ -61,11 +112,8 @@ function getGitIgnorePaths(rootPath) {
* Gives back all files and folders that need to be added to npm artifact
*/
function getNpmPackagePaths(rootPath) {
let pkgJson;
try {
const fileContent = fs.readFileSync(`${rootPath}/package.json`, 'utf8');
pkgJson = JSON.parse(fileContent);
} catch (_) {
const pkgJson = getPackageJson(rootPath);
if (!pkgJson) {
return [];
}
if (pkgJson.files) {
@ -154,6 +202,7 @@ class InputDataService {
/**
* @param {string} projectPath
* @returns { { path:string, name?:string, mainEntry?:string, version?: string, commitHash?:string }}
*/
static getProjectMeta(projectPath) {
const project = { path: projectPath };
@ -414,6 +463,24 @@ class InputDataService {
return null;
}
}
/**
* Gives back all monorepo package paths
*/
static getMonoRepoPackages(rootPath) {
// [1] Look for yarn workspaces
const pkgJson = getPackageJson(rootPath);
if (pkgJson && pkgJson.workspaces) {
return getPathsFromGlobList(pkgJson.workspaces, rootPath);
}
// [2] Look for lerna packages
const lernaJson = getLernaJson(rootPath);
if (lernaJson && lernaJson.packages) {
return getPathsFromGlobList(lernaJson.packages, rootPath);
}
// TODO: support forward compatibility for npm?
return undefined;
}
}
InputDataService.cacheDisabled = false;

View file

@ -2,7 +2,7 @@
* @desc Readable way to do an async forEach
* Since predictability matters, all array items will be handled in a queue,
* one after another
* @param {array} array
* @param {any[]} array
* @param {function} callback
*/
async function aForEach(array, callback) {
@ -15,8 +15,8 @@ async function aForEach(array, callback) {
* @desc Readable way to do an async forEach
* If predictability does not matter, this method will traverse array items concurrently,
* leading to a better performance
* @param {array} array
* @param {function} callback
* @param {any[]} array
* @param {(value:any, index:number) => {}} callback
*/
async function aForEachNonSequential(array, callback) {
return Promise.all(array.map(callback));
@ -25,7 +25,7 @@ async function aForEachNonSequential(array, callback) {
* @desc Readable way to do an async map
* Since predictability is crucial for a map, all array items will be handled in a queue,
* one after anotoher
* @param {array} array
* @param {any[]} array
* @param {function} callback
*/
async function aMap(array, callback) {

View file

@ -1,5 +1,4 @@
/**
*
* @param {string|object} inputValue
* @returns {number}
*/

View file

@ -1,3 +1,4 @@
// @ts-nocheck
/* eslint-disable */
/**

View file

@ -1,5 +1,9 @@
const { InputDataService } = require('../services/InputDataService.js');
/**
* @param {function} func
* @param {{}} externalStorage
*/
function memoize(func, externalStorage) {
const storage = externalStorage || {};
// eslint-disable-next-line func-names
@ -7,15 +11,23 @@ function memoize(func, externalStorage) {
// eslint-disable-next-line prefer-rest-params
const args = [...arguments];
// Allow disabling of cache for testing purposes
// @ts-ignore
if (!InputDataService.cacheDisabled && args in storage) {
// @ts-ignore
return storage[args];
}
// @ts-ignore
const outcome = func.apply(this, args);
// @ts-ignore
storage[args] = outcome;
return outcome;
};
}
/**
* @param {function} func
* @param {{}} externalStorage
*/
function memoizeAsync(func, externalStorage) {
const storage = externalStorage || {};
// eslint-disable-next-line func-names
@ -23,10 +35,14 @@ function memoizeAsync(func, externalStorage) {
// eslint-disable-next-line prefer-rest-params
const args = [...arguments];
// Allow disabling of cache for testing purposes
// @ts-ignore
if (!InputDataService.cacheDisabled && args in storage) {
// @ts-ignore
return storage[args];
}
// @ts-ignore
const outcome = await func.apply(this, args);
// @ts-ignore
storage[args] = outcome;
return outcome;
};

View file

@ -1,3 +1,4 @@
// @ts-nocheck
/* eslint-disable */
/**
* This is a modified version of https://github.com/npm/read-package-tree/blob/master/rpt.js

View file

@ -7,9 +7,9 @@ const path = require('path');
* "InputDataService.createDataObject", it gives back a mocked response.
* @param {string[]|object} files all the code that will be run trhough AST
* @param {object} [cfg]
* @param {string} [cfg.project='fictional-project']
* @param {string} [cfg.projectName='fictional-project']
* @param {string} [cfg.projectPath='/fictional/project']
* @param {string[]} [cfg.filePath=`/fictional/project/test-file-${i}.js`] The indexes of the file
* @param {string[]} [cfg.filePaths=`[/fictional/project/test-file-${i}.js]`] The indexes of the file
* paths match with the indexes of the files
* @param {object} existingMock config for mock-fs, so the previous config is not overridden
*/
@ -18,6 +18,9 @@ function mockProject(files, cfg = {}, existingMock = {}) {
const projPath = cfg.projectPath || '/fictional/project';
// Create obj structure for mock-fs
/**
* @param {object} files
*/
// eslint-disable-next-line no-shadow
function createFilesObjForFolder(files) {
let projFilesObj = {};

View file

@ -5,6 +5,7 @@ const commander = require('commander');
const {
mockProject,
restoreMockedProjects,
mockTargetAndReferenceProject,
} = require('../../test-helpers/mock-project-helpers.js');
const {
mockWriteToJson,
@ -17,11 +18,12 @@ const {
const { InputDataService } = require('../../src/program/services/InputDataService.js');
const { QueryService } = require('../../src/program/services/QueryService.js');
const providenceModule = require('../../src/program/providence.js');
const extendDocsModule = require('../../src/cli/generate-extend-docs-data.js');
const extendDocsModule = require('../../src/cli/launch-providence-with-extend-docs.js');
const cliHelpersModule = require('../../src/cli/cli-helpers.js');
const { cli } = require('../../src/cli/cli.js');
const promptAnalyzerModule = require('../../src/cli/prompt-analyzer-menu.js');
const { toPosixPath } = require('../../src/program/utils/to-posix-path.js');
const { getExtendDocsResults } = require('../../src/cli/launch-providence-with-extend-docs.js');
const {
pathsArrayFromCs,
@ -383,6 +385,7 @@ describe('Providence CLI', () => {
extensions: ['.bla'],
allowlist: [`${rootDir}/al`],
allowlistReference: [`${rootDir}/alr`],
cwd: undefined,
});
});
});
@ -507,4 +510,136 @@ describe('CLI helpers', () => {
]);
});
});
describe('Extend docs', () => {
afterEach(() => {
restoreMockedProjects();
});
it('rewrites monorepo package paths when analysis is run from monorepo root', async () => {
const theirProjectFiles = {
'./package.json': JSON.stringify({
name: 'their-components',
version: '1.0.0',
}),
'./src/TheirButton.js': `export class TheirButton extends HTMLElement {}`,
'./src/TheirTooltip.js': `export class TheirTooltip extends HTMLElement {}`,
'./their-button.js': `
import { TheirButton } from './src/TheirButton.js';
customElements.define('their-button', TheirButton);
`,
'./demo.js': `
import { TheirTooltip } from './src/TheirTooltip.js';
import './their-button.js';
`,
};
const myProjectFiles = {
'./package.json': JSON.stringify({
name: '@my/root',
workspaces: ['packages/*', 'another-folder/my-tooltip'],
dependencies: {
'their-components': '1.0.0',
},
}),
// Package 1: @my/button
'./packages/button/package.json': JSON.stringify({
name: '@my/button',
}),
'./packages/button/src/MyButton.js': `
import { TheirButton } from 'their-components/src/TheirButton.js';
export class MyButton extends TheirButton {}
`,
'./packages/button/src/my-button.js': `
import { MyButton } from './MyButton.js';
customElements.define('my-button', MyButton);
`,
// Package 2: @my/tooltip
'./packages/tooltip/package.json': JSON.stringify({
name: '@my/tooltip',
}),
'./packages/tooltip/src/MyTooltip.js': `
import { TheirTooltip } from 'their-components/src/TheirTooltip.js';
export class MyTooltip extends TheirTooltip {}
`,
};
const theirProject = {
path: '/their-components',
name: 'their-components',
files: Object.entries(theirProjectFiles).map(([file, code]) => ({ file, code })),
};
const myProject = {
path: '/my-components',
name: 'my-components',
files: Object.entries(myProjectFiles).map(([file, code]) => ({ file, code })),
};
mockTargetAndReferenceProject(theirProject, myProject);
const result = await getExtendDocsResults({
referenceProjectPaths: ['/their-components'],
prefixCfg: { from: 'their', to: 'my' },
extensions: ['.js'],
cwd: '/my-components',
});
expect(result).to.eql([
{
name: 'TheirButton',
variable: {
from: 'TheirButton',
to: 'MyButton',
paths: [
{
from: './src/TheirButton.js',
to: '@my/button/src/MyButton.js', // rewritten from './packages/button/src/MyButton.js',
},
{
from: 'their-components/src/TheirButton.js',
to: '@my/button/src/MyButton.js', // rewritten from './packages/button/src/MyButton.js',
},
],
},
tag: {
from: 'their-button',
to: 'my-button',
paths: [
{
from: './their-button.js',
to: '@my/button/src/my-button.js', // rewritten from './packages/button/src/MyButton.js',
},
{
from: 'their-components/their-button.js',
to: '@my/button/src/my-button.js', // rewritten from './packages/button/src/MyButton.js',
},
],
},
},
{
name: 'TheirTooltip',
variable: {
from: 'TheirTooltip',
to: 'MyTooltip',
paths: [
{
from: './src/TheirTooltip.js',
to: '@my/tooltip/src/MyTooltip.js', // './packages/tooltip/src/MyTooltip.js',
},
{
from: 'their-components/src/TheirTooltip.js',
to: '@my/tooltip/src/MyTooltip.js', // './packages/tooltip/src/MyTooltip.js',
},
],
},
},
]);
});
});
});

View file

@ -543,6 +543,81 @@ describe('Analyzer "match-paths"', () => {
expect(queryResult.queryOutput[1].tag).to.eql(expectedMatches[1]);
});
// TODO: test works in isolation, but some side effects occur when run in suite
it.skip(`allows class definition and customElement to be in same file`, async () => {
const theirProjectFiles = {
'./package.json': JSON.stringify({
name: 'their-components',
version: '1.0.0',
}),
'./src/TheirButton.js': `export class TheirButton extends HTMLElement {}`,
'./src/TheirTooltip.js': `export class TheirTooltip extends HTMLElement {}`,
'./their-button.js': `
import { TheirButton } from './src/TheirButton.js';
customElements.define('their-button', TheirButton);
`,
'./demo.js': `
import { TheirTooltip } from './src/TheirTooltip.js';
import './their-button.js';
`,
};
const myProjectFiles = {
'./package.json': JSON.stringify({
name: 'my-components',
dependencies: {
'their-components': '1.0.0',
},
}),
'./src/button/MyButton.js': `
import { TheirButton } from 'their-components/src/TheirButton.js';
export class MyButton extends TheirButton {}
customElements.define('my-button', MyButton);
`,
};
const theirProject = {
path: '/their-components',
name: 'their-components',
files: Object.entries(theirProjectFiles).map(([file, code]) => ({ file, code })),
};
const myProject = {
path: '/my-components',
name: 'my-components',
files: Object.entries(myProjectFiles).map(([file, code]) => ({ file, code })),
};
mockTargetAndReferenceProject(theirProject, myProject);
const providenceCfg = {
targetProjectPaths: ['/my-components'],
referenceProjectPaths: ['/their-components'],
};
await providence(
{ ...matchPathsQueryConfig, prefix: { from: 'their', to: 'my' } },
providenceCfg,
);
const queryResult = queryResults[0];
expect(queryResult.queryOutput[0].tag).to.eql({
from: 'their-button',
to: 'my-button',
paths: [
{
from: './their-button.js',
to: './src/button/MyButton.js',
},
{
from: 'their-components/their-button.js',
to: './src/button/MyButton.js',
},
],
});
});
describe('Features', () => {
it(`identifies all "from" and "to" tagnames`, async () => {
mockTargetAndReferenceProject(searchTargetProject, referenceProject);

View file

@ -61,15 +61,53 @@ describe('InputDataService', () => {
);
});
it('mocked "createDataObject"', async () => {
// By testing the output of our mocked method against the data of the real method, we
// make sure the tests don't run sucessfully undeserved
});
it('"getTargetProjectPaths"', async () => {});
it('"getReferenceProjectPaths"', async () => {});
describe('"getMonoRepoPackages"', async () => {
it('supports yarn workspaces', async () => {
mockProject({
'./package.json': JSON.stringify({
workspaces: ['packages/*', 'another-folder/another-package'],
}),
'./packages/pkg1/package.json': '{ "name": "package1" }',
'./packages/pkg2/package.json': '',
'./packages/pkg3/package.json': '{ "name": "@scope/pkg3" }',
'./another-folder/another-package/package.json':
'{ "name": "@another-scope/another-package" }',
});
expect(InputDataService.getMonoRepoPackages('/fictional/project')).to.eql([
{ path: 'packages/pkg1/', name: 'package1' },
{ path: 'packages/pkg2/', name: 'pkg2' }, // fallback when no package.json
{ path: 'packages/pkg3/', name: '@scope/pkg3' },
{ path: 'another-folder/another-package/', name: '@another-scope/another-package' },
]);
});
it('supports lerna', async () => {
mockProject({
'./package.json': JSON.stringify({}),
'./lerna.json': JSON.stringify({
packages: ['packages/*', 'another-folder/another-package'],
}),
'./packages/pkg1/package.json': '{ "name": "package1" }',
'./packages/pkg2/package.json': '',
'./packages/pkg3/package.json': '{ "name": "@scope/pkg3" }',
'./another-folder/another-package/package.json':
'{ "name": "@another-scope/another-package" }',
});
expect(InputDataService.getMonoRepoPackages('/fictional/project')).to.eql([
{ path: 'packages/pkg1/', name: 'package1' },
{ path: 'packages/pkg2/', name: 'pkg2' }, // fallback when no package.json
{ path: 'packages/pkg3/', name: '@scope/pkg3' },
{ path: 'another-folder/another-package/', name: '@another-scope/another-package' },
]);
});
});
describe('"gatherFilesFromDir"', async () => {
beforeEach(() => {
mockProject({