Merge pull request #2292 from ing-bank/fix/providence-glob-perf

Fix/providence glob perf
This commit is contained in:
Thijs Louisse 2024-05-21 16:07:46 +02:00 committed by GitHub
commit def3df4c08
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 334 additions and 194 deletions

View file

@ -0,0 +1,8 @@
---
'providence-analytics': patch
---
- expose types; improve logging and formatting
- apply memoisation in optimised-glob
- add perf logs to LogService
- allow to clear cache of memoized function

View file

@ -24,7 +24,8 @@
}, },
"files": [ "files": [
"inlined-swc-to-babel", "inlined-swc-to-babel",
"src" "src",
"types"
], ],
"scripts": { "scripts": {
"dashboard": "node ./src/dashboard/server.js --run-server --serve-from-package-root", "dashboard": "node ./src/dashboard/server.js --run-server --serve-from-package-root",

View file

@ -1,10 +1,11 @@
/* eslint-disable no-shadow, no-param-reassign */ /* eslint-disable no-shadow, no-param-reassign */
import path from 'path'; import path from 'path';
import t from '@babel/types';
import babelTraverse from '@babel/traverse'; import babelTraverse from '@babel/traverse';
import { Analyzer } from '../core/Analyzer.js'; import t from '@babel/types';
import { trackDownIdentifierFromScope } from '../utils/track-down-identifier--legacy.js'; import { trackDownIdentifierFromScope } from '../utils/track-down-identifier--legacy.js';
import { Analyzer } from '../core/Analyzer.js';
/** /**
* @typedef {import('@babel/types').File} File * @typedef {import('@babel/types').File} File

View file

@ -1,5 +1,5 @@
/* eslint-disable no-continue */ /* eslint-disable no-continue */
import pathLib from 'path'; import path from 'path';
/* eslint-disable no-shadow, no-param-reassign */ /* eslint-disable no-shadow, no-param-reassign */
import FindImportsAnalyzer from './find-imports.js'; import FindImportsAnalyzer from './find-imports.js';
import FindExportsAnalyzer from './find-exports.js'; import FindExportsAnalyzer from './find-exports.js';
@ -9,14 +9,15 @@ import { transformIntoIterableFindExportsOutput } from './helpers/transform-into
import { transformIntoIterableFindImportsOutput } from './helpers/transform-into-iterable-find-imports-output.js'; import { transformIntoIterableFindImportsOutput } from './helpers/transform-into-iterable-find-imports-output.js';
/** /**
* @typedef {import('../../../types/index.js').FindImportsAnalyzerResult} FindImportsAnalyzerResult * @typedef {import('../../../types/index.js').ConciseMatchImportsAnalyzerResult} ConciseMatchImportsAnalyzerResult
* @typedef {import('../../../types/index.js').FindExportsAnalyzerResult} FindExportsAnalyzerResult
* @typedef {import('../../../types/index.js').IterableFindExportsAnalyzerEntry} IterableFindExportsAnalyzerEntry * @typedef {import('../../../types/index.js').IterableFindExportsAnalyzerEntry} IterableFindExportsAnalyzerEntry
* @typedef {import('../../../types/index.js').IterableFindImportsAnalyzerEntry} IterableFindImportsAnalyzerEntry * @typedef {import('../../../types/index.js').IterableFindImportsAnalyzerEntry} IterableFindImportsAnalyzerEntry
* @typedef {import('../../../types/index.js').ConciseMatchImportsAnalyzerResult} ConciseMatchImportsAnalyzerResult
* @typedef {import('../../../types/index.js').MatchImportsConfig} MatchImportsConfig
* @typedef {import('../../../types/index.js').MatchImportsAnalyzerResult} MatchImportsAnalyzerResult
* @typedef {import('../../../types/index.js').PathRelativeFromProjectRoot} PathRelativeFromProjectRoot * @typedef {import('../../../types/index.js').PathRelativeFromProjectRoot} PathRelativeFromProjectRoot
* @typedef {import('../../../types/index.js').MatchImportsAnalyzerResult} MatchImportsAnalyzerResult
* @typedef {import('../../../types/index.js').FindImportsAnalyzerResult} FindImportsAnalyzerResult
* @typedef {import('../../../types/index.js').FindExportsAnalyzerResult} FindExportsAnalyzerResult
* @typedef {import('../../../types/index.js').AnalyzerQueryResult} AnalyzerQueryResult
* @typedef {import('../../../types/index.js').MatchImportsConfig} MatchImportsConfig
* @typedef {import('../../../types/index.js').PathFromSystemRoot} PathFromSystemRoot * @typedef {import('../../../types/index.js').PathFromSystemRoot} PathFromSystemRoot
* @typedef {import('../../../types/index.js').AnalyzerName} AnalyzerName * @typedef {import('../../../types/index.js').AnalyzerName} AnalyzerName
* @typedef {import('../../../types/index.js').AnalyzerAst} AnalyzerAst * @typedef {import('../../../types/index.js').AnalyzerAst} AnalyzerAst
@ -117,7 +118,7 @@ async function matchImportsPostprocess(exportsAnalyzerResult, importsAnalyzerRes
const fromImportToExport = await fromImportToExportPerspective({ const fromImportToExport = await fromImportToExportPerspective({
importee: importEntry.normalizedSource, importee: importEntry.normalizedSource,
importer: /** @type {PathFromSystemRoot} */ ( importer: /** @type {PathFromSystemRoot} */ (
pathLib.resolve(importProjectPath, importEntry.file) path.resolve(importProjectPath, importEntry.file)
), ),
importeeProjectPath: cfg.referenceProjectPath, importeeProjectPath: cfg.referenceProjectPath,
}); });
@ -193,6 +194,7 @@ export default class MatchImportsAnalyzer extends Analyzer {
* Prepare * Prepare
*/ */
const cachedAnalyzerResult = await this._prepare(cfg); const cachedAnalyzerResult = await this._prepare(cfg);
if (cachedAnalyzerResult) { if (cachedAnalyzerResult) {
return cachedAnalyzerResult; return cachedAnalyzerResult;
} }
@ -205,6 +207,7 @@ export default class MatchImportsAnalyzer extends Analyzer {
targetProjectPath: cfg.referenceProjectPath, targetProjectPath: cfg.referenceProjectPath,
skipCheckMatchCompatibility: cfg.skipCheckMatchCompatibility, skipCheckMatchCompatibility: cfg.skipCheckMatchCompatibility,
suppressNonCriticalLogs: true, suppressNonCriticalLogs: true,
gatherFilesConfig: cfg.gatherFilesConfigReference,
}); });
} }
@ -216,6 +219,7 @@ export default class MatchImportsAnalyzer extends Analyzer {
targetProjectPath: cfg.targetProjectPath, targetProjectPath: cfg.targetProjectPath,
skipCheckMatchCompatibility: cfg.skipCheckMatchCompatibility, skipCheckMatchCompatibility: cfg.skipCheckMatchCompatibility,
suppressNonCriticalLogs: true, suppressNonCriticalLogs: true,
gatherFilesConfig: cfg.gatherFilesConfig,
}); });
} }

View file

@ -1,27 +1,39 @@
/* eslint-disable no-param-reassign */ /* eslint-disable no-param-reassign */
import semver from 'semver';
import path from 'path'; import path from 'path';
import { LogService } from './LogService.js';
import { QueryService } from './QueryService.js'; import semver from 'semver';
import { ReportService } from './ReportService.js';
import { getFilePathRelativeFromRoot } from '../utils/get-file-path-relative-from-root.js';
import { InputDataService } from './InputDataService.js'; import { InputDataService } from './InputDataService.js';
import { toPosixPath } from '../utils/to-posix-path.js'; import { toPosixPath } from '../utils/to-posix-path.js';
import { getFilePathRelativeFromRoot } from '../utils/get-file-path-relative-from-root.js'; import { ReportService } from './ReportService.js';
import { QueryService } from './QueryService.js';
import { LogService } from './LogService.js';
/** /**
* @typedef {import("@swc/core").Module} SwcAstModule * @typedef {(ast: File, astContext: {code:string; relativePath:string; projectData: ProjectInputDataWithMeta}) => object} FileAstTraverseFn
* @typedef {import('../../../types/index.js').AnalyzerName} AnalyzerName
* @typedef {import('../../../types/index.js').AnalyzerAst} AnalyzerAst
* @typedef {import('../../../types/index.js').PathFromSystemRoot} PathFromSystemRoot
* @typedef {import('../../../types/index.js').QueryOutput} QueryOutput
* @typedef {import('../../../types/index.js').ProjectInputData} ProjectInputData
* @typedef {import('../../../types/index.js').ProjectInputDataWithMeta} ProjectInputDataWithMeta * @typedef {import('../../../types/index.js').ProjectInputDataWithMeta} ProjectInputDataWithMeta
* @typedef {import('../../../types/index.js').AnalyzerQueryResult} AnalyzerQueryResult * @typedef {import('../../../types/index.js').AnalyzerQueryResult} AnalyzerQueryResult
* @typedef {import('../../../types/index.js').MatchAnalyzerConfig} MatchAnalyzerConfig * @typedef {import('../../../types/index.js').MatchAnalyzerConfig} MatchAnalyzerConfig
* @typedef {import('../../../types/index.js').PathFromSystemRoot} PathFromSystemRoot
* @typedef {import('../../../types/index.js').ProjectInputData} ProjectInputData
* @typedef {import('../../../types/index.js').AnalyzerName} AnalyzerName
* @typedef {import('../../../types/index.js').AnalyzerAst} AnalyzerAst
* @typedef {import('../../../types/index.js').QueryOutput} QueryOutput
* @typedef {import("@swc/core").Module} SwcAstModule
* @typedef {import('@babel/types').File} File * @typedef {import('@babel/types').File} File
* @typedef {(ast: File, astContext: {code:string; relativePath:string; projectData: ProjectInputDataWithMeta}) => object} FileAstTraverseFn
*/ */
/**
* @param {string} identifier
*/
function displayProjectsInLog(identifier) {
const [target, targetV, , reference, referenceV] = identifier.split('_');
return decodeURIComponent(
`${target}@${targetV} ${reference ? `- ${reference}@${referenceV}` : ''}`,
);
}
/** /**
* Analyzes one entry: the callback can traverse a given ast for each entry * Analyzes one entry: the callback can traverse a given ast for each entry
* @param {ProjectInputDataWithMeta} projectData * @param {ProjectInputDataWithMeta} projectData
@ -247,12 +259,7 @@ export class Analyzer {
if (!compatible) { if (!compatible) {
if (!cfg.suppressNonCriticalLogs) { if (!cfg.suppressNonCriticalLogs) {
LogService.info( LogService.info(
`skipping ${LogService.pad(this.name, 16)} for ${ `${LogService.pad(`skipping ${this.name} (${reason})`)}${displayProjectsInLog(this.identifier)}`,
this.identifier
}: (${reason})\n${cfg.targetProjectPath.replace(
`${process.cwd()}/providence-input-data/search-targets/`,
'',
)}`,
); );
} }
return ensureAnalyzerResultFormat(`[${reason}]`, cfg, this); return ensureAnalyzerResultFormat(`[${reason}]`, cfg, this);
@ -273,24 +280,43 @@ export class Analyzer {
} }
if (!cfg.suppressNonCriticalLogs) { if (!cfg.suppressNonCriticalLogs) {
LogService.info(`starting ${LogService.pad(this.name, 16)} for ${this.identifier}`); LogService.info(
`${LogService.pad(`starting ${this.name}`)}${displayProjectsInLog(this.identifier)}`,
);
} }
/** /**
* Get reference and search-target data * Get reference and search-target data
*/ */
if (!cfg.targetProjectResult) { if (!cfg.targetProjectResult) {
performance.mark('analyzer--prepare--createDTarg-start');
this.targetData = await InputDataService.createDataObject( this.targetData = await InputDataService.createDataObject(
[cfg.targetProjectPath], [cfg.targetProjectPath],
cfg.gatherFilesConfig, cfg.gatherFilesConfig,
); );
performance.mark('analyzer--prepare--createDTarg-end');
const m1 = performance.measure(
'analyzer--prepare--createDTarg',
'analyzer--prepare--createDTarg-start',
'analyzer--prepare--createDTarg-end',
);
LogService.perf(m1);
} }
if (cfg.referenceProjectPath) { if (cfg.referenceProjectPath) {
performance.mark('analyzer--prepare--createDRef-start');
this.referenceData = await InputDataService.createDataObject( this.referenceData = await InputDataService.createDataObject(
[cfg.referenceProjectPath], [cfg.referenceProjectPath],
cfg.gatherFilesConfigReference || cfg.gatherFilesConfig, cfg.gatherFilesConfigReference || cfg.gatherFilesConfig,
); );
performance.mark('analyzer--prepare--createDRef-end');
const m2 = performance.measure(
'analyzer--prepare--createDRef',
'analyzer--prepare--createDRef-start',
'analyzer--prepare--createDRef-end',
);
LogService.perf(m2);
} }
return undefined; return undefined;
@ -304,10 +330,21 @@ export class Analyzer {
_finalize(queryOutput, cfg) { _finalize(queryOutput, cfg) {
LogService.debug(`Analyzer "${this.name}": started _finalize method`); LogService.debug(`Analyzer "${this.name}": started _finalize method`);
performance.mark('analyzer--finalize-start');
const analyzerResult = ensureAnalyzerResultFormat(queryOutput, cfg, this); const analyzerResult = ensureAnalyzerResultFormat(queryOutput, cfg, this);
if (!cfg.suppressNonCriticalLogs) { if (!cfg.suppressNonCriticalLogs) {
LogService.success(`finished ${LogService.pad(this.name, 16)} for ${this.identifier}`); LogService.success(
`${LogService.pad(`finished ${this.name}`)}${displayProjectsInLog(this.identifier)}`,
);
} }
performance.mark('analyzer--finalize-end');
const measurementFinalize = performance.measure(
'analyzer--finalize',
'analyzer--finalize-start',
'analyzer--finalize-end',
);
LogService.perf(measurementFinalize);
return analyzerResult; return analyzerResult;
} }

View file

@ -97,11 +97,25 @@ export class LogService {
log(colors.fgBlue, ` info${printTitle(title)}`, colors.reset, text); log(colors.fgBlue, ` info${printTitle(title)}`, colors.reset, text);
} }
/**
* @param {PerformanceMeasure} measurement
* @param {string} [title]
*/
static perf(measurement, title) {
const text = `${this.pad(`[${measurement.name}]`)} ${measurement.duration}ms`;
// @ts-ignore
this._logHistory.push(`- perf -${printTitle(title)} ${text}`);
if (this.allMuted || !this.perfEnabled) {
return;
}
log(colors.fgGray, ` perf${printTitle(title)}`, colors.reset, text);
}
/** /**
* @param {string} text * @param {string} text
* @param {number} minChars * @param {number} minChars
*/ */
static pad(text, minChars = 20) { static pad(text, minChars = 40) {
let result = text; let result = text;
const padding = minChars - text.length; const padding = minChars - text.length;
if (padding > 0) { if (padding > 0) {
@ -127,6 +141,7 @@ export class LogService {
LogService.debugEnabled = false; LogService.debugEnabled = false;
LogService.allMuted = false; LogService.allMuted = false;
LogService.throwsOnError = false; LogService.throwsOnError = false;
LogService.perfEnabled = false;
/** @type {string[]} */ /** @type {string[]} */
LogService._logHistory = []; LogService._logHistory = [];

View file

@ -9,13 +9,13 @@ import { LogService } from './core/LogService.js';
import { AstService } from './core/AstService.js'; import { AstService } from './core/AstService.js';
/** /**
* @typedef {import('../../types/index.js').ProvidenceConfig} ProvidenceConfig
* @typedef {import('../../types/index.js').PathFromSystemRoot} PathFromSystemRoot
* @typedef {import('../../types/index.js').QueryResult} QueryResult
* @typedef {import('../../types/index.js').AnalyzerQueryResult} AnalyzerQueryResult * @typedef {import('../../types/index.js').AnalyzerQueryResult} AnalyzerQueryResult
* @typedef {import('../../types/index.js').QueryConfig} QueryConfig
* @typedef {import('../../types/index.js').AnalyzerQueryConfig} AnalyzerQueryConfig * @typedef {import('../../types/index.js').AnalyzerQueryConfig} AnalyzerQueryConfig
* @typedef {import('../../types/index.js').PathFromSystemRoot} PathFromSystemRoot
* @typedef {import('../../types/index.js').GatherFilesConfig} GatherFilesConfig * @typedef {import('../../types/index.js').GatherFilesConfig} GatherFilesConfig
* @typedef {import('../../types/index.js').ProvidenceConfig} ProvidenceConfig
* @typedef {import('../../types/index.js').QueryResult} QueryResult
* @typedef {import('../../types/index.js').QueryConfig} QueryConfig
*/ */
/** /**
@ -81,6 +81,8 @@ function getSlicedQueryConfig(queryConfig, targetProjectPath, referenceProjectPa
* @param {{ gatherFilesConfig:GatherFilesConfig, gatherFilesConfigReference:GatherFilesConfig, skipCheckMatchCompatibility:boolean }} cfg * @param {{ gatherFilesConfig:GatherFilesConfig, gatherFilesConfigReference:GatherFilesConfig, skipCheckMatchCompatibility:boolean }} cfg
*/ */
async function handleAnalyzerForProjectCombo(slicedQConfig, cfg) { async function handleAnalyzerForProjectCombo(slicedQConfig, cfg) {
performance.mark(`${slicedQConfig.analyzerName}-start`);
const queryResult = await QueryService.astSearch(slicedQConfig, { const queryResult = await QueryService.astSearch(slicedQConfig, {
gatherFilesConfig: cfg.gatherFilesConfig, gatherFilesConfig: cfg.gatherFilesConfig,
gatherFilesConfigReference: cfg.gatherFilesConfigReference, gatherFilesConfigReference: cfg.gatherFilesConfigReference,
@ -88,6 +90,17 @@ async function handleAnalyzerForProjectCombo(slicedQConfig, cfg) {
addSystemPathsInResult: cfg.addSystemPathsInResult, addSystemPathsInResult: cfg.addSystemPathsInResult,
...slicedQConfig.analyzerConfig, ...slicedQConfig.analyzerConfig,
}); });
performance.mark(`${slicedQConfig.analyzerName}-end`);
const measurement = /** @type {* & PerformanceMeasure} */ (
performance.measure(
slicedQConfig.analyzerName,
`${slicedQConfig.analyzerName}-start`,
`${slicedQConfig.analyzerName}-end`,
)
);
LogService.perf(measurement);
if (queryResult) { if (queryResult) {
report(queryResult, cfg); report(queryResult, cfg);
} }

View file

@ -24,10 +24,9 @@ function createCachableArg(arg) {
/** /**
* @template T * @template T
* @type {<T extends Function>(functionToMemoize:T, opts?:{ storage?:object; }) => T} * @type {<T extends Function>(functionToMemoize:T, opts?:{ cacheStorage?:object; }) => T & {clearCache:() => void}}
*/ */
export function memoize(functionToMemoize, { storage = {} } = {}) { export function memoize(functionToMemoize, { cacheStorage = {} } = {}) {
return /** @type {* & T} */ (
function memoizedFn() { function memoizedFn() {
// eslint-disable-next-line prefer-rest-params // eslint-disable-next-line prefer-rest-params
const args = [...arguments]; const args = [...arguments];
@ -36,18 +35,22 @@ export function memoize(functionToMemoize, { storage = {} } = {}) {
const cachableArgs = shouldSerialize ? args.map(createCachableArg) : args; const cachableArgs = shouldSerialize ? args.map(createCachableArg) : args;
// Allow disabling of cache for testing purposes // Allow disabling of cache for testing purposes
// @ts-expect-error // @ts-expect-error
if (shouldCache && cachableArgs in storage) { if (shouldCache && cachableArgs in cacheStorage) {
// @ts-expect-error // @ts-expect-error
return storage[cachableArgs]; return cacheStorage[cachableArgs];
} }
// @ts-expect-error // @ts-expect-error
const outcome = functionToMemoize.apply(this, args); const outcome = functionToMemoize.apply(this, args);
// @ts-expect-error // @ts-expect-error
// eslint-disable-next-line no-param-reassign // eslint-disable-next-line no-param-reassign
storage[cachableArgs] = outcome; cacheStorage[cachableArgs] = outcome;
return outcome; return outcome;
} }
); memoizedFn.clearCache = () => {
// eslint-disable-next-line no-param-reassign
cacheStorage = {};
};
return /** @type {* & T & {clearCache:() => void}} */ (memoizedFn);
} }
/** /**

View file

@ -4,22 +4,25 @@ import nodeFs from 'fs';
import path from 'path'; import path from 'path';
import { toPosixPath } from './to-posix-path.js'; import { toPosixPath } from './to-posix-path.js';
import { memoize } from './memoize.js';
/** /**
* @typedef {nodeFs} FsLike * @typedef {nodeFs} FsLike
* @typedef {nodeFs.Dirent & {path:string;parentPath:string}} DirentWithPath
* @typedef {{onlyDirectories:boolean;onlyFiles:boolean;deep:number;suppressErrors:boolean;fs: FsLike;cwd:string;absolute:boolean;extglob:boolean;}} FastGlobtions * @typedef {{onlyDirectories:boolean;onlyFiles:boolean;deep:number;suppressErrors:boolean;fs: FsLike;cwd:string;absolute:boolean;extglob:boolean;}} FastGlobtions
*/ */
const [nodeMajor] = process.versions.node.split('.').map(Number); const [nodeMajor] = process.versions.node.split('.').map(Number);
/** export const parseGlobToRegex = memoize(
/**
* @param {string} glob * @param {string} glob
* @param {object} [providedOpts] * @param {object} [providedOpts]
* @param {boolean} [providedOpts.globstar=true] if true, '/foo/*' => '^\/foo\/[^/]*$' (not allowing folders inside *), else '/foo/*' => '^\/foo\/.*$' * @param {boolean} [providedOpts.globstar=true] if true, '/foo/*' => '^\/foo\/[^/]*$' (not allowing folders inside *), else '/foo/*' => '^\/foo\/.*$'
* @param {boolean} [providedOpts.extglob=true] if true, supports so called "extended" globs (like bash) and single character matching, matching ranges of characters, group matching etc. * @param {boolean} [providedOpts.extglob=true] if true, supports so called "extended" globs (like bash) and single character matching, matching ranges of characters, group matching etc.
* @returns {RegExp} * @returns {RegExp}
*/ */
export function parseGlobToRegex(glob, providedOpts) { (glob, providedOpts) => {
if (typeof glob !== 'string') throw new TypeError('Expected a string'); if (typeof glob !== 'string') throw new TypeError('Expected a string');
const options = { const options = {
@ -84,7 +87,9 @@ export function parseGlobToRegex(glob, providedOpts) {
regexResultStr += '.*'; regexResultStr += '.*';
} else { } else {
const isGlobstarSegment = const isGlobstarSegment =
isMultiStar && ['/', undefined].includes(prevChar) && ['/', undefined].includes(nextChar); isMultiStar &&
['/', undefined].includes(prevChar) &&
['/', undefined].includes(nextChar);
if (isGlobstarSegment) { if (isGlobstarSegment) {
// Match zero or more path segments // Match zero or more path segments
regexResultStr += '((?:[^/]*(?:/|$))*)'; regexResultStr += '((?:[^/]*(?:/|$))*)';
@ -101,12 +106,14 @@ export function parseGlobToRegex(glob, providedOpts) {
} }
return new RegExp(`^${regexResultStr}$`); return new RegExp(`^${regexResultStr}$`);
} },
);
/** const getStartPath = memoize(
/**
* @param {string} glob * @param {string} glob
*/ */
function getStartPath(glob) { glob => {
const reservedChars = ['?', '[', ']', '{', '}', ',', '.', '*']; const reservedChars = ['?', '[', ']', '{', '}', ',', '.', '*'];
let hasFoundReservedChar = false; let hasFoundReservedChar = false;
return glob return glob
@ -118,45 +125,68 @@ function getStartPath(glob) {
}) })
.filter(Boolean) .filter(Boolean)
.join('/'); .join('/');
} },
);
let isCacheEnabled = false; let isCacheEnabled = false;
/** @type {{[path:string]:nodeFs.Dirent[]}} */ /** @type {{[path:string]:DirentWithPath[]}} */
const cache = {}; const cache = {};
/** const getAllDirentsFromStartPath = memoize(
/**
* @param {string} startPath * @param {string} startPath
* @param {{fs?:FsLike, dirents?:nodeFs.Dirent[]}} providedOptions * @param {{fs?:FsLike, dirents?:DirentWithPath[]}} providedOptions
* @returns {Promise<nodeFs.Dirent[]>} * @returns {Promise<DirentWithPath[]>}
*/ */
async function getAllFilesFromStartPath( async (startPath, { fs = /** @type {* & FsLike} */ (nodeFs), dirents = [] } = {}) => {
startPath,
{ fs = /** @type {* & FsLike} */ (nodeFs), dirents = [] } = {},
) {
if (isCacheEnabled && cache[startPath]) return cache[startPath]; if (isCacheEnabled && cache[startPath]) return cache[startPath];
// Older node doesn't support recursive option // Older node doesn't support recursive option
if (nodeMajor < 18) { if (nodeMajor < 18) {
/** @type {nodeFs.Dirent[]} */ /** @type {nodeFs.Dirent[]} */
const direntsForLvl = await fs.promises.readdir(startPath, { withFileTypes: true }); const direntsForLvl = await fs.promises.readdir(startPath, { withFileTypes: true });
for (const dirent of direntsForLvl) { for (const _dirent of direntsForLvl) {
// @ts-expect-error const dirent = /** @type {DirentWithPath} */ (_dirent);
dirent.parentPath = dirent.path = startPath; // eslint-disable-line no-multi-assign dirent.parentPath = dirent.path = startPath; // eslint-disable-line no-multi-assign
dirents.push(dirent); dirents.push(/** @type {DirentWithPath} */ (dirent));
if (dirent.isDirectory()) { if (dirent.isDirectory()) {
const subDir = path.join(startPath, dirent.name); const subDir = path.join(startPath, dirent.name);
await getAllFilesFromStartPath(subDir, { fs, dirents }); await getAllDirentsFromStartPath(subDir, { fs, dirents });
} }
} }
return /** @type {nodeFs.Dirent[]} */ (dirents); return dirents;
} }
dirents.push(
// @ts-expect-error // @ts-expect-error
dirents.push(...(await fs.promises.readdir(startPath, { withFileTypes: true, recursive: true }))); ...(await fs.promises.readdir(startPath, { withFileTypes: true, recursive: true })),
);
cache[startPath] = dirents; cache[startPath] = dirents;
return dirents; return dirents;
} },
);
const getAllDirentsRelativeToCwd = memoize(
/**
* @param {string} fullStartPath
* @param {{fs?:FsLike, cwd:string}} options
* @returns {Promise<{relativeToCwdPath:string;dirent:DirentWithPath}[]>}
*/
async (fullStartPath, options) => {
const allDirentsRelativeToStartPath = await getAllDirentsFromStartPath(fullStartPath, {
fs: options.fs,
});
const allDirEntsRelativeToCwd = allDirentsRelativeToStartPath.map(dirent => ({
relativeToCwdPath: toPosixPath(
path.join(dirent.parentPath || dirent.path, dirent.name),
).replace(`${toPosixPath(options.cwd)}/`, ''),
dirent,
}));
return allDirEntsRelativeToCwd;
},
);
/** /**
* Lightweight glob implementation. * Lightweight glob implementation.
@ -219,18 +249,11 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
const fullStartPath = path.join(options.cwd, startPath); const fullStartPath = path.join(options.cwd, startPath);
try { try {
const allDirentsRelativeToStartPath = await getAllFilesFromStartPath(fullStartPath, { const allDirEntsRelativeToCwd = await getAllDirentsRelativeToCwd(fullStartPath, {
cwd: options.cwd,
fs: options.fs, fs: options.fs,
}); });
const allDirEntsRelativeToCwd = allDirentsRelativeToStartPath.map(dirent => ({
relativeToCwdPath: toPosixPath(
// @ts-expect-error
path.join(dirent.parentPath || dirent.path, dirent.name),
).replace(`${toPosixPath(options.cwd)}/`, ''),
dirent,
}));
globEntries.push(...allDirEntsRelativeToCwd); globEntries.push(...allDirEntsRelativeToCwd);
} catch (e) { } catch (e) {
if (!options.suppressErrors) { if (!options.suppressErrors) {
@ -262,8 +285,8 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
filteredPaths = filteredPaths.map(f => toPosixPath(path.join(options.cwd, f))); filteredPaths = filteredPaths.map(f => toPosixPath(path.join(options.cwd, f)));
if (process.platform === 'win32') { if (process.platform === 'win32') {
const driveLetter = path.win32.resolve(options.cwd).slice(0, 1).toUpperCase(); const driveChar = path.win32.resolve(options.cwd).slice(0, 1).toUpperCase();
filteredPaths = filteredPaths.map(f => `${driveLetter}:${f}`); filteredPaths = filteredPaths.map(f => `${driveChar}:${f}`);
} }
} }
@ -273,10 +296,16 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
const result = options.unique ? Array.from(new Set(filteredPaths)) : filteredPaths; const result = options.unique ? Array.from(new Set(filteredPaths)) : filteredPaths;
return result.sort((a, b) => { const res = result.sort((a, b) => {
const pathDiff = a.split('/').length - b.split('/').length; const pathDiff = a.split('/').length - b.split('/').length;
return pathDiff !== 0 ? pathDiff : a.localeCompare(b); return pathDiff !== 0 ? pathDiff : a.localeCompare(b);
}); });
// It could happen the fs changes with the next call, so we clear the cache
getAllDirentsRelativeToCwd.clearCache();
getAllDirentsFromStartPath.clearCache();
return res;
} }
optimisedGlob.disableCache = () => { optimisedGlob.disableCache = () => {

View file

@ -201,13 +201,13 @@ describe('Memoize', () => {
sumCalled += 1; sumCalled += 1;
return { ...a, ...b }; return { ...a, ...b };
} }
const sumMemoized = memoize(sum, { serializeObjects: true }); const sumMemoized = memoize(sum);
let sum2Called = 0; let sum2Called = 0;
function sum2(/** @type {object} a */ a, /** @type {object} a */ b) { function sum2(/** @type {object} a */ a, /** @type {object} a */ b) {
sum2Called += 1; sum2Called += 1;
return { ...a, ...b }; return { ...a, ...b };
} }
const sum2Memoized = memoize(sum2, { serializeObjects: true }); const sum2Memoized = memoize(sum2);
expect(sumMemoized({ x: 1 }, { y: 2 })).to.deep.equal({ x: 1, y: 2 }); expect(sumMemoized({ x: 1 }, { y: 2 })).to.deep.equal({ x: 1, y: 2 });
expect(sumCalled).to.equal(1); expect(sumCalled).to.equal(1);
@ -233,7 +233,7 @@ describe('Memoize', () => {
sumCalled += 1; sumCalled += 1;
return { ...a, ...b }; return { ...a, ...b };
} }
const sumMemoized = memoize(sum, { serializeObjects: true }); const sumMemoized = memoize(sum);
// Put in cache for args combination // Put in cache for args combination
const result = sumMemoized({ x: 1 }, { y: 2 }); const result = sumMemoized({ x: 1 }, { y: 2 });
@ -313,4 +313,33 @@ describe('Memoize', () => {
expect(sum2Called).to.equal(1); expect(sum2Called).to.equal(1);
}); });
}); });
describe('Cache', () => {
it(`"memoizedFn.clearCache()" clears the cache for a memoized fn"`, async () => {
let sumCalled = 0;
function sum(/** @type {string} a */ a, /** @type {string} a */ b) {
sumCalled += 1;
return a + b;
}
const sumMemoized = memoize(sum);
// Put in cache for args combination
expect(sumMemoized('1', '2')).to.equal('12');
expect(sumCalled).to.equal(1);
// Return from cache
expect(sumMemoized('1', '2')).to.equal('12');
expect(sumCalled).to.equal(1);
sumMemoized.clearCache();
// Now the original function is called again
expect(sumMemoized('1', '2')).to.equal('12');
expect(sumCalled).to.equal(2);
// Return from new cache again
expect(sumMemoized('1', '2')).to.equal('12');
expect(sumCalled).to.equal(2);
});
});
}); });