index.js 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333
  1. import { lstat, readdir, realpath, stat } from 'node:fs/promises';
  2. import { join as pjoin, resolve as presolve, sep as psep } from 'node:path';
  3. import { Readable } from 'node:stream';
  4. export const EntryTypes = {
  5. FILE_TYPE: 'files',
  6. DIR_TYPE: 'directories',
  7. FILE_DIR_TYPE: 'files_directories',
  8. EVERYTHING_TYPE: 'all',
  9. };
  10. const defaultOptions = {
  11. root: '.',
  12. fileFilter: (_entryInfo) => true,
  13. directoryFilter: (_entryInfo) => true,
  14. type: EntryTypes.FILE_TYPE,
  15. lstat: false,
  16. depth: 2147483648,
  17. alwaysStat: false,
  18. // Throughput is flat from 16 to 65536 (traversal is I/O-bound), but
  19. // batches of 1024+ entries survive young-gen GC and bloat RSS ~20-60%.
  20. highWaterMark: 256,
  21. };
  22. Object.freeze(defaultOptions);
  23. const RECURSIVE_ERROR_CODE = 'READDIRP_RECURSIVE_ERROR';
  24. const NORMAL_FLOW_ERRORS = new Set(['ENOENT', 'EPERM', 'EACCES', 'ELOOP', RECURSIVE_ERROR_CODE]);
  25. const ALL_TYPES = [
  26. EntryTypes.DIR_TYPE,
  27. EntryTypes.EVERYTHING_TYPE,
  28. EntryTypes.FILE_DIR_TYPE,
  29. EntryTypes.FILE_TYPE,
  30. ];
  31. const DIR_TYPES = new Set([
  32. EntryTypes.DIR_TYPE,
  33. EntryTypes.EVERYTHING_TYPE,
  34. EntryTypes.FILE_DIR_TYPE,
  35. ]);
  36. const FILE_TYPES = new Set([
  37. EntryTypes.EVERYTHING_TYPE,
  38. EntryTypes.FILE_DIR_TYPE,
  39. EntryTypes.FILE_TYPE,
  40. ]);
  41. const isNormalFlowError = (error) => NORMAL_FLOW_ERRORS.has(error.code);
  42. const wantBigintFsStats = process.platform === 'win32';
  43. const emptyFn = (_entryInfo) => true;
  44. const normalizeFilter = (filter) => {
  45. if (filter === undefined)
  46. return emptyFn;
  47. if (typeof filter === 'function')
  48. return filter;
  49. if (typeof filter === 'string') {
  50. const fl = filter.trim();
  51. return (entry) => entry.basename === fl;
  52. }
  53. if (Array.isArray(filter)) {
  54. const trItems = filter.map((item) => item.trim());
  55. return (entry) => trItems.some((f) => entry.basename === f);
  56. }
  57. return emptyFn;
  58. };
  59. export class ReaddirpStream extends Readable {
  60. /**
  61. * Directories discovered but not yet emitted from. Listings are read
  62. * lazily (on pop, plus one prefetch) instead of eagerly on discovery:
  63. * keeping whole listings for every queued dir balloons RAM on wide trees.
  64. */
  65. parents;
  66. reading;
  67. parent;
  68. _stat;
  69. _maxDepth;
  70. _wantsDir;
  71. _wantsFile;
  72. _wantsEverything;
  73. _root;
  74. _isDirent;
  75. _statsProp;
  76. _rdOptions;
  77. _fileFilter;
  78. _directoryFilter;
  79. _relStart;
  80. constructor(options = {}) {
  81. super({
  82. objectMode: true,
  83. autoDestroy: true,
  84. highWaterMark: options.highWaterMark ?? defaultOptions.highWaterMark,
  85. });
  86. const opts = { ...defaultOptions, ...options };
  87. // Use ?? so an explicit `undefined` in user options doesn't shadow defaults.
  88. const root = opts.root ?? defaultOptions.root;
  89. const type = opts.type ?? defaultOptions.type;
  90. this._fileFilter = normalizeFilter(opts.fileFilter);
  91. this._directoryFilter = normalizeFilter(opts.directoryFilter);
  92. const statMethod = opts.lstat ? lstat : stat;
  93. // Use bigint stats if it's windows and stat() supports options (node 10+).
  94. if (wantBigintFsStats) {
  95. this._stat = (path) => statMethod(path, { bigint: true });
  96. }
  97. else {
  98. this._stat = statMethod;
  99. }
  100. this._maxDepth =
  101. opts.depth != null && Number.isSafeInteger(opts.depth) ? opts.depth : defaultOptions.depth;
  102. this._wantsDir = DIR_TYPES.has(type);
  103. this._wantsFile = FILE_TYPES.has(type);
  104. this._wantsEverything = type === EntryTypes.EVERYTHING_TYPE;
  105. this._root = presolve(root);
  106. // Every fullPath is `_root + sep + relative path` (see _formatEntry), so
  107. // the relative path is a slice starting past the root and its trailing
  108. // separator (which resolved paths lack, except fs roots like '/', 'C:\').
  109. this._relStart = this._root.endsWith(psep) ? this._root.length : this._root.length + 1;
  110. this._isDirent = !opts.alwaysStat;
  111. this._statsProp = this._isDirent ? 'dirent' : 'stats';
  112. this._rdOptions = { encoding: 'utf8', withFileTypes: this._isDirent };
  113. // Launch stream with one parent, the root dir, whose readdir starts
  114. // right away. Explore the resolved root so all parent paths stay
  115. // absolute even if process.cwd() changes mid-iteration.
  116. const rootDir = { path: this._root, depth: 1 };
  117. rootDir.pending = this._exploreDir(this._root, 1);
  118. this.parents = [rootDir];
  119. this.reading = false;
  120. this.parent = undefined;
  121. }
  122. async _read(batch) {
  123. if (this.reading)
  124. return;
  125. this.reading = true;
  126. try {
  127. while (!this.destroyed && batch > 0) {
  128. const par = this.parent;
  129. const fil = par && par.files;
  130. if (fil && fil.length > 0) {
  131. const { path, depth } = par;
  132. const slice = fil.splice(0, batch).map((dirent) => this._formatEntry(dirent, path));
  133. // In dirent mode _formatEntry is synchronous: skip Promise.all and
  134. // its per-entry microtask overhead.
  135. const awaited = this._isDirent
  136. ? slice
  137. : await Promise.all(slice);
  138. for (const entry of awaited) {
  139. if (!entry)
  140. continue;
  141. if (this.destroyed)
  142. return;
  143. // Only symlinks require async work; plain files / dirs resolve synchronously.
  144. let entryType = this._getEntryType(entry);
  145. if (typeof entryType !== 'string')
  146. entryType = await entryType;
  147. if (entryType === 'directory' && this._directoryFilter(entry)) {
  148. if (depth <= this._maxDepth) {
  149. // Lazy: don't readdir until this dir is popped. Keeping whole
  150. // listings for every queued dir would balloon RAM on wide trees.
  151. this.parents.push({ path: entry.fullPath, depth: depth + 1 });
  152. }
  153. if (this._wantsDir) {
  154. this.push(entry);
  155. batch--;
  156. }
  157. }
  158. else if ((entryType === 'file' || this._includeAsFile(entry)) &&
  159. this._fileFilter(entry)) {
  160. if (this._wantsFile) {
  161. this.push(entry);
  162. batch--;
  163. }
  164. }
  165. }
  166. }
  167. else {
  168. const parent = this.parents.pop();
  169. if (!parent) {
  170. this.push(null);
  171. break;
  172. }
  173. const dir = parent.pending ?? this._exploreDir(parent.path, parent.depth);
  174. // Prefetch the next dir so its readdir overlaps with processing
  175. // this one's entries. Only the stack top is prefetched, keeping at
  176. // most a handful of listings (~tree depth) in RAM at once.
  177. const next = this.parents[this.parents.length - 1];
  178. if (next && !next.pending) {
  179. next.pending = this._exploreDir(next.path, next.depth);
  180. }
  181. this.parent = await dir;
  182. if (this.destroyed)
  183. return;
  184. }
  185. }
  186. }
  187. catch (error) {
  188. this.destroy(error);
  189. }
  190. finally {
  191. this.reading = false;
  192. }
  193. }
  194. // NOTE: native `readdir(path, { recursive: true })` was evaluated as a
  195. // replacement for this per-directory traversal and rejected:
  196. // - Not faster: node implements it in JS, walking directories sequentially
  197. // just like this loop, but with extra path bookkeeping. Benchmarks
  198. // (node 24): ~10% slower on wide trees, ~40% slower on small ones,
  199. // parity on deep ones.
  200. // - Much more RAM: it buffers the entire subtree listing in one array,
  201. // instead of one directory at a time, defeating streaming.
  202. // - Semantics diverge: it can't limit depth, can't skip directories a
  203. // directoryFilter rejects, doesn't follow symlinked dirs, and fails
  204. // wholesale (all entries lost) if anything in the subtree is unreadable,
  205. // instead of emitting a 'warn' and continuing.
  206. async _exploreDir(path, depth) {
  207. let files;
  208. try {
  209. files = await readdir(path, this._rdOptions);
  210. }
  211. catch (error) {
  212. this._onError(error);
  213. }
  214. return { files, depth, path };
  215. }
  216. // Synchronous in dirent mode; returns a promise only when stats are needed.
  217. _formatEntry(dirent, path) {
  218. const basename = this._isDirent ? dirent.name : dirent;
  219. // `path` is always an absolute, normalized parent dir (see _exploreDir
  220. // seeding in the constructor), so a plain join is enough — resolve()
  221. // would re-read cwd on every entry.
  222. const fullPath = pjoin(path, basename);
  223. // Slice instead of path.relative(): equivalent here (fullPath is always
  224. // under _root) and avoids several intermediate allocations per entry.
  225. const entry = { path: fullPath.slice(this._relStart), fullPath, basename };
  226. if (this._isDirent) {
  227. entry.dirent = dirent;
  228. return entry;
  229. }
  230. return this._stat(fullPath).then((stats) => {
  231. entry.stats = stats;
  232. return entry;
  233. }, (err) => {
  234. this._onError(err);
  235. return undefined;
  236. });
  237. }
  238. _onError(err) {
  239. if (isNormalFlowError(err) && !this.destroyed) {
  240. this.emit('warn', err);
  241. }
  242. else {
  243. this.destroy(err);
  244. }
  245. }
  246. // Synchronous for regular files and directories; returns a promise only for
  247. // symlinks, which need realpath() to be classified.
  248. _getEntryType(entry) {
  249. // entry may be undefined, because a warning or an error were emitted
  250. // and the statsProp is undefined
  251. if (!entry || !(this._statsProp in entry)) {
  252. return '';
  253. }
  254. const stats = entry[this._statsProp];
  255. if (stats.isFile())
  256. return 'file';
  257. if (stats.isDirectory())
  258. return 'directory';
  259. if (stats.isSymbolicLink())
  260. return this._getSymlinkEntryType(entry);
  261. return '';
  262. }
  263. async _getSymlinkEntryType(entry) {
  264. const full = entry.fullPath;
  265. try {
  266. const entryRealPath = await realpath(full);
  267. const entryRealPathStats = await lstat(entryRealPath);
  268. if (entryRealPathStats.isFile()) {
  269. return 'file';
  270. }
  271. if (entryRealPathStats.isDirectory()) {
  272. const len = entryRealPath.length;
  273. if (full.startsWith(entryRealPath) && full[len] === psep) {
  274. const recursiveError = new Error(`Circular symlink detected: "${full}" points to "${entryRealPath}"`);
  275. // @ts-ignore
  276. recursiveError.code = RECURSIVE_ERROR_CODE;
  277. this._onError(recursiveError);
  278. return '';
  279. }
  280. return 'directory';
  281. }
  282. }
  283. catch (error) {
  284. this._onError(error);
  285. }
  286. return '';
  287. }
  288. _includeAsFile(entry) {
  289. const stats = entry && entry[this._statsProp];
  290. return stats && this._wantsEverything && !stats.isDirectory();
  291. }
  292. }
  293. /**
  294. * Streaming version: Reads all files and directories in given root recursively.
  295. * Consumes ~constant small amount of RAM.
  296. * @param root Root directory
  297. * @param options Options to specify root (start directory), filters and recursion depth
  298. */
  299. export function readdirp(root, options = {}) {
  300. // @ts-ignore
  301. let type = options.entryType || options.type;
  302. if (type === 'both')
  303. type = EntryTypes.FILE_DIR_TYPE; // backwards-compatibility
  304. if (!root) {
  305. throw new Error('readdirp: root argument is required. Usage: readdirp(root, options)');
  306. }
  307. else if (typeof root !== 'string') {
  308. throw new TypeError('readdirp: root argument must be a string. Usage: readdirp(root, options)');
  309. }
  310. else if (type && !ALL_TYPES.includes(type)) {
  311. throw new Error(`readdirp: Invalid type passed. Use one of ${ALL_TYPES.join(', ')}`);
  312. }
  313. // Copy options instead of mutating the caller's object.
  314. const opts = { ...options, root };
  315. if (type)
  316. opts.type = type;
  317. return new ReaddirpStream(opts);
  318. }
  319. /**
  320. * Promise version: Reads all files and directories in given root recursively.
  321. * Compared to streaming version, will consume a lot of RAM e.g. when 1 million files are listed.
  322. * @returns array of paths and their entry infos
  323. */
  324. export function readdirpPromise(root, options = {}) {
  325. return new Promise((resolve, reject) => {
  326. const files = [];
  327. readdirp(root, options)
  328. .on('data', (entry) => files.push(entry))
  329. .on('end', () => resolve(files))
  330. .on('error', (error) => reject(error));
  331. });
  332. }
  333. export default readdirp;