ayo/lion
1
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(providence): experimental support for fs.glob + ignore support in optimisedGlob

Browse source
This commit is contained in:
Thijs Louisse 2024-10-20 22:22:13 +02:00 committed by Thijs Louisse
parent 370b357bd3
commit 2dbb1ca7bc
3 changed files with 416 additions and 254 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.changeset/forty-hotels-watch.md Normal file
View file

@ -0,0 +1,6 @@
---
'providence-analytics': patch
---
- support `ignore: string[]` globs in optimisedGlob
- experimental `fs.glob` support under the hood in optimisedGlob

185
packages-node/providence-analytics/src/program/utils/optimised-glob.js
View file

@ -7,9 +7,21 @@ import { toPosixPath } from './to-posix-path.js';
import { memoize } from './memoize.js';
/**
* @typedef {nodeFs.Dirent & { path:string; parentPath:string }} DirentWithPath
* @typedef {nodeFs} FsLike
* @typedef {nodeFs.Dirent & {path:string;parentPath:string}} DirentWithPath
* @typedef {{onlyDirectories:boolean;onlyFiles:boolean;deep:number;suppressErrors:boolean;fs: FsLike;cwd:string;absolute:boolean;extglob:boolean;}} FastGlobtions
* @typedef {{
* onlyDirectories: boolean;
* suppressErrors: boolean;
* onlyFiles: boolean;
* absolute: boolean;
* extglob: boolean;
* ignore: string[];
* unique: boolean;
* deep: number;
* dot: boolean;
* cwd: string;
* fs: FsLike;
* }} FastGlobtions
*/
const [nodeMajor] = process.versions.node.split('.').map(Number);
@ -109,6 +121,65 @@ export const parseGlobToRegex = memoize(
},
);
/**
* @template T
* @param {T[]} arr
* @returns {T[]}
*/
function toUniqueArray(arr) {
return Array.from(new Set(arr));
}
/**
* @param {DirentWithPath} dirent
* @param {{cwd:string}} cfg
* @returns {string}
*/
function direntToLocalPath(dirent, { cwd }) {
const folder = (dirent.parentPath || dirent.path).replace(/(^.*)\/(.*\..*$)/, '$1');
return toPosixPath(path.join(folder, dirent.name)).replace(
new RegExp(`^${toPosixPath(cwd)}/`),
'',
);
}
/**
* @param {{dirent:nodeFs.Dirent;relativeToCwdPath:string}[]} matchedEntries
* @param {FastGlobtions} options
* @returns {string[]}
*/
function postprocessOptions(matchedEntries, options) {
const allFileOrDirectoryEntries = matchedEntries.filter(({ dirent }) =>
options.onlyDirectories ? dirent.isDirectory() : dirent.isFile(),
);
let filteredPaths = allFileOrDirectoryEntries.map(({ relativeToCwdPath }) => relativeToCwdPath);
if (!options.dot) {
filteredPaths = filteredPaths.filter(
f => !f.split('/').some(folderOrFile => folderOrFile.startsWith('.')),
);
}
if (options.absolute) {
filteredPaths = filteredPaths.map(f => toPosixPath(path.join(options.cwd, f)));
if (process.platform === 'win32') {
const driveChar = path.win32.resolve(options.cwd).slice(0, 1).toUpperCase();
filteredPaths = filteredPaths.map(f => `${driveChar}:${f}`);
}
}
if (options.deep !== Infinity) {
filteredPaths = filteredPaths.filter(f => f.split('/').length <= options.deep + 2);
}
const result = options.unique ? toUniqueArray(filteredPaths) : filteredPaths;
return result.sort((a, b) => {
const pathDiff = a.split('/').length - b.split('/').length;
return pathDiff !== 0 ? pathDiff : a.localeCompare(b);
});
}
const getStartPath = memoize(
/**
* @param {string} glob
@ -129,6 +200,7 @@ const getStartPath = memoize(
);
let isCacheEnabled = false;
let isExperimentalFsGlobEnabled = false;
/** @type {{[path:string]:DirentWithPath[]}} */
const cache = {};
@ -179,15 +251,34 @@ const getAllDirentsRelativeToCwd = memoize(
});
const allDirEntsRelativeToCwd = allDirentsRelativeToStartPath.map(dirent => ({
relativeToCwdPath: toPosixPath(
path.join(dirent.parentPath || dirent.path, dirent.name),
).replace(`${toPosixPath(options.cwd)}/`, ''),
relativeToCwdPath: direntToLocalPath(dirent, { cwd: options.cwd }),
dirent,
}));
return allDirEntsRelativeToCwd;
},
);
/**
* @param {string|string[]} globOrGlobs
* @param {{ fs: FsLike; cwd: string; exclude?: Function; stats?: boolean }} cfg
* @returns {Promise<string[]|DirentWithPath[]>}
*/
async function nativeGlob(globOrGlobs, { fs, cwd, exclude, stats }) {
// @ts-expect-error
const asyncGenResult = await fs.promises.glob(globOrGlobs, {
withFileTypes: true,
cwd,
...(exclude ? { exclude } : {}),
});
const results = [];
for await (const dirent of asyncGenResult) {
if (dirent.name === '.' || dirent.isDirectory()) continue; // eslint-disable-line no-continue
results.push(stats ? dirent : direntToLocalPath(dirent, { cwd }));
}
return results;
}
/**
* Lightweight glob implementation.
* It's a drop-in replacement for globby, but it's faster, a few hundred lines of code and has no dependencies.
@ -209,7 +300,8 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
unique: true,
sync: false,
dot: false,
// TODO: ignore, throwErrorOnBrokenSymbolicLink, markDirectories, objectMode, onlyDirectories, onlyFiles, stats
ignore: [],
// Add if needed: throwErrorOnBrokenSymbolicLink, markDirectories, objectMode, stats
// https://github.com/mrmlnc/fast-glob?tab=readme-ov-file
...providedOptions,
};
@ -219,7 +311,46 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
options.onlyDirectories = true;
}
const globs = Array.isArray(globOrGlobs) ? Array.from(new Set(globOrGlobs)) : [globOrGlobs];
const regularGlobs = Array.isArray(globOrGlobs) ? globOrGlobs : [globOrGlobs];
const ignoreGlobs = options.ignore.map((/** @type {string} */ g) =>
g.startsWith('!') ? g : `!${g}`,
);
const optionsNotSupportedByNativeGlob = ['onlyDirectories', 'dot'];
const doesConfigAllowNative = !optionsNotSupportedByNativeGlob.some(opt => options[opt]);
if (isExperimentalFsGlobEnabled && options.fs.promises.glob && doesConfigAllowNative) {
const negativeGlobs = [...ignoreGlobs, ...regularGlobs.filter(r => r.startsWith('!'))].map(r =>
r.slice(1),
);
const negativeResults = negativeGlobs.length
? /** @type {string[]} */ (await nativeGlob(negativeGlobs, options))
: [];
const positiveGlobs = regularGlobs.filter(r => !r.startsWith('!'));
const result = /** @type {DirentWithPath[]} */ (
await nativeGlob(positiveGlobs, {
cwd: options.cwd,
fs: options.fs,
stats: true,
// we cannot use the exclude option here, because it's not working correctly
})
);
const direntsFiltered = result.filter(
dirent => !negativeResults.includes(direntToLocalPath(dirent, { cwd: options.cwd })),
);
return postprocessOptions(
direntsFiltered.map(dirent => ({
dirent,
relativeToCwdPath: direntToLocalPath(dirent, { cwd: options.cwd }),
})),
options,
);
}
const globs = toUniqueArray([...regularGlobs, ...ignoreGlobs]);
/** @type {RegExp[]} */
const matchRegexesNegative = [];
@ -269,37 +400,7 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
!matchRegexesNegative.some(globReNeg => globReNeg.test(globEntry.relativeToCwdPath)),
);
const allFileOrDirectoryEntries = matchedEntries.filter(({ dirent }) =>
options.onlyDirectories ? dirent.isDirectory() : dirent.isFile(),
);
let filteredPaths = allFileOrDirectoryEntries.map(({ relativeToCwdPath }) => relativeToCwdPath);
if (!options.dot) {
filteredPaths = filteredPaths.filter(
f => !f.split('/').some(folderOrFile => folderOrFile.startsWith('.')),
);
}
if (options.absolute) {
filteredPaths = filteredPaths.map(f => toPosixPath(path.join(options.cwd, f)));
if (process.platform === 'win32') {
const driveChar = path.win32.resolve(options.cwd).slice(0, 1).toUpperCase();
filteredPaths = filteredPaths.map(f => `${driveChar}:${f}`);
}
}
if (options.deep !== Infinity) {
filteredPaths = filteredPaths.filter(f => f.split('/').length <= options.deep + 2);
}
const result = options.unique ? Array.from(new Set(filteredPaths)) : filteredPaths;
const res = result.sort((a, b) => {
const pathDiff = a.split('/').length - b.split('/').length;
return pathDiff !== 0 ? pathDiff : a.localeCompare(b);
});
const res = postprocessOptions(matchedEntries, options);
// It could happen the fs changes with the next call, so we clear the cache
getAllDirentsRelativeToCwd.clearCache();
@ -311,3 +412,11 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
optimisedGlob.disableCache = () => {
isCacheEnabled = false;
};
optimisedGlob.enableExperimentalFsGlob = () => {
isExperimentalFsGlobEnabled = true;
};
optimisedGlob.disableExperimentalFsGlob = () => {
isExperimentalFsGlobEnabled = false;
};

67
packages-node/providence-analytics/test-node/program/utils/optimised-glob.test.js
View file

@ -1,9 +1,7 @@
import path from 'path';
import { globby } from 'globby';
// eslint-disable-next-line import/no-extraneous-dependencies
import { expect } from 'chai';
// eslint-disable-next-line import/no-extraneous-dependencies
// @ts-expect-error
import mockFs from 'mock-fs';
import { optimisedGlob } from '../../../src/program/utils/optimised-glob.js';
@ -11,8 +9,12 @@ import { optimisedGlob } from '../../../src/program/utils/optimised-glob.js';
const measurePerf = process.argv.includes('--measure-perf');
/**
* @param {*} patterns
* @param {*} options
* @typedef {import('../../../src/program/utils/optimised-glob.js').FastGlobtions} FastGlobtions
*/
/**
* @param {string|string[]} patterns
* @param {Partial<FastGlobtions>} options
* @returns {Promise<string[]>}
*/
async function runOptimisedGlobAndCheckGlobbyParity(patterns, options) {
@ -43,7 +45,8 @@ async function runOptimisedGlobAndCheckGlobbyParity(patterns, options) {
return optimisedGlobResult;
}
describe('optimisedGlob', () => {
function runSuiteForOptimisedGlob() {
describe('optimisedGlob', () => {
const testCfg = {
cwd: '/fakeFs',
};
@ -93,7 +96,10 @@ describe('optimisedGlob', () => {
});
it('supports single asterisk like "my/folder/*/some/file.js" ', async () => {
const files = await runOptimisedGlobAndCheckGlobbyParity('my/folder/*/some/file.js', testCfg);
const files = await runOptimisedGlobAndCheckGlobbyParity(
'my/folder/*/some/file.js',
testCfg,
);
expect(files).to.deep.equal(['my/folder/lvl1/some/file.js']);
});
@ -159,7 +165,10 @@ describe('optimisedGlob', () => {
testCfg,
);
expect(files).to.deep.equal(['my/folder/lvl1/some/file.d.ts', 'my/folder/lvl1/some/file.js']);
expect(files).to.deep.equal([
'my/folder/lvl1/some/file.d.ts',
'my/folder/lvl1/some/file.js',
]);
});
});
@ -192,10 +201,13 @@ describe('optimisedGlob', () => {
describe('Options', () => {
it('"absolute" returns full system paths', async () => {
const files = await runOptimisedGlobAndCheckGlobbyParity('my/folder/*/some/file.{js,d.ts}', {
const files = await runOptimisedGlobAndCheckGlobbyParity(
'my/folder/*/some/file.{js,d.ts}',
{
...testCfg,
absolute: true,
});
},
);
if (process.platform === 'win32') {
const driveLetter = path.win32.resolve(testCfg.cwd).slice(0, 1).toUpperCase();
@ -266,6 +278,20 @@ describe('optimisedGlob', () => {
expect(files).to.deep.equal(['my/.hiddenFile.js']);
});
it('"ignore" filters out files" ', async () => {
const files = await runOptimisedGlobAndCheckGlobbyParity('**', {
...testCfg,
ignore: ['**/lvl1/**'],
});
expect(files).to.deep.equal([
'my/folder/some/anotherFile.d.ts',
'my/folder/some/anotherFile.js',
'my/folder/some/file.d.ts',
'my/folder/some/file.js',
]);
});
it.skip('"suppressErrors" throws errors when paths do not exist', async () => {
expect(async () =>
optimisedGlob('my/folder/**/some/file.js', {
@ -276,4 +302,25 @@ describe('optimisedGlob', () => {
).to.throw();
});
});
});
}
runSuiteForOptimisedGlob();
describe('Native glob', () => {
const [nodeMajor] = process.versions.node.split('.').map(Number);
if (nodeMajor < 22) {
console.warn('Skipping native glob tests because Node.js version is too low.');
return;
}
before(() => {
optimisedGlob.enableExperimentalFsGlob();
});
after(() => {
optimisedGlob.disableExperimentalFsGlob();
});
runSuiteForOptimisedGlob();
});