From 0f0991cd62407d3479c8ee484b1942b9b17bc42f Mon Sep 17 00:00:00 2001
From: Thijs Louisse <Thijs.Louisse@ing.com>
Date: Wed, 22 Jan 2025 17:23:01 +0100
Subject: [PATCH] fix(providence): [optimised-glob] use Array.concat for memory
 efficiency

---
 .changeset/selfish-wasps-switch.md            |  5 +++++
 .../src/program/utils/optimised-glob.js       | 19 ++++++++++++-------
 2 files changed, 17 insertions(+), 7 deletions(-)
 create mode 100644 .changeset/selfish-wasps-switch.md

diff --git a/.changeset/selfish-wasps-switch.md b/.changeset/selfish-wasps-switch.md
new file mode 100644
index 000000000..c96b051fc
--- /dev/null
+++ b/.changeset/selfish-wasps-switch.md
@@ -0,0 +1,5 @@
+---
+'providence-analytics': patch
+---
+
+[optimised-glob] use Array.concat for memory efficiency
diff --git a/packages-node/providence-analytics/src/program/utils/optimised-glob.js b/packages-node/providence-analytics/src/program/utils/optimised-glob.js
index bb0d34931..211b97fcf 100644
--- a/packages-node/providence-analytics/src/program/utils/optimised-glob.js
+++ b/packages-node/providence-analytics/src/program/utils/optimised-glob.js
@@ -253,12 +253,16 @@ const getAllDirentsFromStartPath = memoize(
       return dirents;
     }
 
-    dirents.push(
-      // @ts-expect-error
-      ...(await fs.promises.readdir(startPath, { withFileTypes: true, recursive: true })),
+    const direntResult = /** @type {* & DirentWithPath[]} */ (
+      await fs.promises.readdir(startPath, {
+        withFileTypes: true,
+        // @ts-expect-error
+        recursive: true,
+      })
     );
-    cache[startPath] = dirents;
-    return dirents;
+
+    cache[startPath] = direntResult;
+    return direntResult;
   },
 );
 
@@ -401,7 +405,7 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
   /** @type {RegExp[]} */
   const matchRegexes = [];
   /** @type {{dirent:nodeFs.Dirent;relativeToCwdPath:string}[]} */
-  const globEntries = [];
+  let globEntries = [];
 
   for (const glob of globs) {
     const isNegative = glob.startsWith('!');
@@ -431,7 +435,8 @@ export async function optimisedGlob(globOrGlobs, providedOptions = {}) {
         cwd,
       });
 
-      globEntries.push(...allDirEntsRelativeToCwd);
+      // N.B. `globEntries.push(...allDirEntsRelativeToCwd);` can give memory issues for large file trees
+      globEntries = globEntries.concat(allDirEntsRelativeToCwd);
     } catch (e) {
       if (!options.suppressErrors) {
         throw e;