CachedInputFileSystem.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. // eslint-disable-next-line n/prefer-global/process
  7. const { nextTick } = require("process");
  8. /** @typedef {import("./Resolver").FileSystem} FileSystem */
  9. /** @typedef {import("./Resolver").PathLike} PathLike */
  10. /** @typedef {import("./Resolver").PathOrFileDescriptor} PathOrFileDescriptor */
  11. /** @typedef {import("./Resolver").SyncFileSystem} SyncFileSystem */
  12. /** @typedef {FileSystem & SyncFileSystem} BaseFileSystem */
  13. /**
  14. * @template T
  15. * @typedef {import("./Resolver").FileSystemCallback<T>} FileSystemCallback<T>
  16. */
  17. /**
  18. * @param {string} path path
  19. * @returns {string} dirname
  20. */
  21. const dirname = (path) => {
  22. let idx = path.length - 1;
  23. while (idx >= 0) {
  24. const char = path.charCodeAt(idx);
  25. // slash or backslash
  26. if (char === 47 || char === 92) break;
  27. idx--;
  28. }
  29. if (idx < 0) return "";
  30. return path.slice(0, idx);
  31. };
  32. /**
  33. * @template T
  34. * @param {FileSystemCallback<T>[]} callbacks callbacks
  35. * @param {Error | null} err error
  36. * @param {T} result result
  37. */
  38. const runCallbacks = (callbacks, err, result) => {
  39. if (callbacks.length === 1) {
  40. callbacks[0](err, result);
  41. callbacks.length = 0;
  42. return;
  43. }
  44. let error;
  45. for (const callback of callbacks) {
  46. try {
  47. callback(err, result);
  48. } catch (err) {
  49. if (!error) error = err;
  50. }
  51. }
  52. callbacks.length = 0;
  53. if (error) throw error;
  54. };
  55. // eslint-disable-next-line jsdoc/no-restricted-syntax
  56. /** @typedef {Function} EXPECTED_FUNCTION */
  57. // eslint-disable-next-line jsdoc/no-restricted-syntax
  58. /** @typedef {any} EXPECTED_ANY */
  59. class OperationMergerBackend {
  60. /**
  61. * @param {EXPECTED_FUNCTION | undefined} provider async method in filesystem
  62. * @param {EXPECTED_FUNCTION | undefined} syncProvider sync method in filesystem
  63. * @param {BaseFileSystem} providerContext call context for the provider methods
  64. */
  65. constructor(provider, syncProvider, providerContext) {
  66. this._provider = provider;
  67. this._syncProvider = syncProvider;
  68. this._providerContext = providerContext;
  69. this._activeAsyncOperations = new Map();
  70. this.provide = this._provider
  71. ? // Comment to align jsdoc
  72. /**
  73. * @param {PathLike | PathOrFileDescriptor} path path
  74. * @param {object | FileSystemCallback<EXPECTED_ANY> | undefined} options options
  75. * @param {FileSystemCallback<EXPECTED_ANY>=} callback callback
  76. * @returns {EXPECTED_ANY} result
  77. */
  78. (path, options, callback) => {
  79. if (typeof options === "function") {
  80. callback =
  81. /** @type {FileSystemCallback<EXPECTED_ANY>} */
  82. (options);
  83. options = undefined;
  84. }
  85. if (
  86. typeof path !== "string" &&
  87. !Buffer.isBuffer(path) &&
  88. !(path instanceof URL) &&
  89. typeof path !== "number"
  90. ) {
  91. /** @type {EXPECTED_FUNCTION} */
  92. (callback)(
  93. new TypeError("path must be a string, Buffer, URL or number"),
  94. );
  95. return;
  96. }
  97. if (options) {
  98. return /** @type {EXPECTED_FUNCTION} */ (this._provider).call(
  99. this._providerContext,
  100. path,
  101. options,
  102. callback,
  103. );
  104. }
  105. let callbacks = this._activeAsyncOperations.get(path);
  106. if (callbacks) {
  107. callbacks.push(callback);
  108. return;
  109. }
  110. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  111. /** @type {EXPECTED_FUNCTION} */
  112. (provider)(
  113. path,
  114. /**
  115. * @param {Error} err error
  116. * @param {EXPECTED_ANY} result result
  117. */
  118. (err, result) => {
  119. this._activeAsyncOperations.delete(path);
  120. runCallbacks(callbacks, err, result);
  121. },
  122. );
  123. }
  124. : null;
  125. this.provideSync = this._syncProvider
  126. ? // Comment to align jsdoc
  127. /**
  128. * @param {PathLike | PathOrFileDescriptor} path path
  129. * @param {object=} options options
  130. * @returns {EXPECTED_ANY} result
  131. */
  132. (path, options) =>
  133. /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  134. this._providerContext,
  135. path,
  136. options,
  137. )
  138. : null;
  139. }
  140. purge() {}
  141. purgeParent() {}
  142. }
  143. /*
  144. IDLE:
  145. insert data: goto SYNC
  146. SYNC:
  147. before provide: run ticks
  148. event loop tick: goto ASYNC_ACTIVE
  149. ASYNC:
  150. timeout: run tick, goto ASYNC_PASSIVE
  151. ASYNC_PASSIVE:
  152. before provide: run ticks
  153. IDLE --[insert data]--> SYNC --[event loop tick]--> ASYNC_ACTIVE --[interval tick]-> ASYNC_PASSIVE
  154. ^ |
  155. +---------[insert data]-------+
  156. */
  157. const STORAGE_MODE_IDLE = 0;
  158. const STORAGE_MODE_SYNC = 1;
  159. const STORAGE_MODE_ASYNC = 2;
  160. /**
  161. * @callback Provide
  162. * @param {PathLike | PathOrFileDescriptor} path path
  163. * @param {EXPECTED_ANY} options options
  164. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  165. * @returns {void}
  166. */
  167. class CacheBackend {
  168. /**
  169. * @param {number} duration max cache duration of items
  170. * @param {EXPECTED_FUNCTION | undefined} provider async method
  171. * @param {EXPECTED_FUNCTION | undefined} syncProvider sync method
  172. * @param {BaseFileSystem} providerContext call context for the provider methods
  173. */
  174. constructor(duration, provider, syncProvider, providerContext) {
  175. this._duration = duration;
  176. this._provider = provider;
  177. this._syncProvider = syncProvider;
  178. this._providerContext = providerContext;
  179. /** @type {Map<string, FileSystemCallback<EXPECTED_ANY>[]>} */
  180. this._activeAsyncOperations = new Map();
  181. /** @type {Map<string, { err: Error | null, result?: EXPECTED_ANY, level: Set<string> }>} */
  182. this._data = new Map();
  183. /** @type {Set<string>[]} */
  184. this._levels = [];
  185. for (let i = 0; i < 10; i++) this._levels.push(new Set());
  186. for (let i = 5000; i < duration; i += 500) this._levels.push(new Set());
  187. this._currentLevel = 0;
  188. this._tickInterval = Math.floor(duration / this._levels.length);
  189. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  190. this._mode = STORAGE_MODE_IDLE;
  191. /** @type {NodeJS.Timeout | undefined} */
  192. this._timeout = undefined;
  193. /** @type {number | undefined} */
  194. this._nextDecay = undefined;
  195. // eslint-disable-next-line no-warning-comments
  196. // @ts-ignore
  197. this.provide = provider ? this.provide.bind(this) : null;
  198. // eslint-disable-next-line no-warning-comments
  199. // @ts-ignore
  200. this.provideSync = syncProvider ? this.provideSync.bind(this) : null;
  201. }
  202. /**
  203. * @param {PathLike | PathOrFileDescriptor} path path
  204. * @param {EXPECTED_ANY} options options
  205. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  206. * @returns {void}
  207. */
  208. provide(path, options, callback) {
  209. if (typeof options === "function") {
  210. callback = options;
  211. options = undefined;
  212. }
  213. if (
  214. typeof path !== "string" &&
  215. !Buffer.isBuffer(path) &&
  216. !(path instanceof URL) &&
  217. typeof path !== "number"
  218. ) {
  219. callback(new TypeError("path must be a string, Buffer, URL or number"));
  220. return;
  221. }
  222. const strPath = typeof path !== "string" ? path.toString() : path;
  223. if (options) {
  224. return /** @type {EXPECTED_FUNCTION} */ (this._provider).call(
  225. this._providerContext,
  226. path,
  227. options,
  228. callback,
  229. );
  230. }
  231. // When in sync mode we can move to async mode
  232. if (this._mode === STORAGE_MODE_SYNC) {
  233. this._enterAsyncMode();
  234. }
  235. // Check in cache
  236. const cacheEntry = this._data.get(strPath);
  237. if (cacheEntry !== undefined) {
  238. if (cacheEntry.err) return nextTick(callback, cacheEntry.err);
  239. return nextTick(callback, null, cacheEntry.result);
  240. }
  241. // Check if there is already the same operation running
  242. let callbacks = this._activeAsyncOperations.get(strPath);
  243. if (callbacks !== undefined) {
  244. callbacks.push(callback);
  245. return;
  246. }
  247. this._activeAsyncOperations.set(strPath, (callbacks = [callback]));
  248. // Run the operation
  249. /** @type {EXPECTED_FUNCTION} */
  250. (this._provider).call(
  251. this._providerContext,
  252. path,
  253. /**
  254. * @param {Error | null} err error
  255. * @param {EXPECTED_ANY=} result result
  256. */
  257. (err, result) => {
  258. this._activeAsyncOperations.delete(strPath);
  259. this._storeResult(strPath, err, result);
  260. // Enter async mode if not yet done
  261. this._enterAsyncMode();
  262. runCallbacks(
  263. /** @type {FileSystemCallback<EXPECTED_ANY>[]} */ (callbacks),
  264. err,
  265. result,
  266. );
  267. },
  268. );
  269. }
  270. /**
  271. * @param {PathLike | PathOrFileDescriptor} path path
  272. * @param {EXPECTED_ANY} options options
  273. * @returns {EXPECTED_ANY} result
  274. */
  275. provideSync(path, options) {
  276. if (
  277. typeof path !== "string" &&
  278. !Buffer.isBuffer(path) &&
  279. !(path instanceof URL) &&
  280. typeof path !== "number"
  281. ) {
  282. throw new TypeError("path must be a string");
  283. }
  284. const strPath = typeof path !== "string" ? path.toString() : path;
  285. if (options) {
  286. return /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  287. this._providerContext,
  288. path,
  289. options,
  290. );
  291. }
  292. // In sync mode we may have to decay some cache items
  293. if (this._mode === STORAGE_MODE_SYNC) {
  294. this._runDecays();
  295. }
  296. // Check in cache
  297. const cacheEntry = this._data.get(strPath);
  298. if (cacheEntry !== undefined) {
  299. if (cacheEntry.err) throw cacheEntry.err;
  300. return cacheEntry.result;
  301. }
  302. // Get all active async operations
  303. // This sync operation will also complete them
  304. const callbacks = this._activeAsyncOperations.get(strPath);
  305. this._activeAsyncOperations.delete(strPath);
  306. // Run the operation
  307. // When in idle mode, we will enter sync mode
  308. let result;
  309. try {
  310. result = /** @type {EXPECTED_FUNCTION} */ (this._syncProvider).call(
  311. this._providerContext,
  312. path,
  313. );
  314. } catch (err) {
  315. this._storeResult(strPath, /** @type {Error} */ (err), undefined);
  316. this._enterSyncModeWhenIdle();
  317. if (callbacks) {
  318. runCallbacks(callbacks, /** @type {Error} */ (err), undefined);
  319. }
  320. throw err;
  321. }
  322. this._storeResult(strPath, null, result);
  323. this._enterSyncModeWhenIdle();
  324. if (callbacks) {
  325. runCallbacks(callbacks, null, result);
  326. }
  327. return result;
  328. }
  329. /**
  330. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  331. */
  332. purge(what) {
  333. if (!what) {
  334. if (this._mode !== STORAGE_MODE_IDLE) {
  335. this._data.clear();
  336. for (const level of this._levels) {
  337. level.clear();
  338. }
  339. this._enterIdleMode();
  340. }
  341. } else if (
  342. typeof what === "string" ||
  343. Buffer.isBuffer(what) ||
  344. what instanceof URL ||
  345. typeof what === "number"
  346. ) {
  347. const strWhat = typeof what !== "string" ? what.toString() : what;
  348. for (const [key, data] of this._data) {
  349. if (key.startsWith(strWhat)) {
  350. this._data.delete(key);
  351. data.level.delete(key);
  352. }
  353. }
  354. if (this._data.size === 0) {
  355. this._enterIdleMode();
  356. }
  357. } else {
  358. for (const [key, data] of this._data) {
  359. for (const item of what) {
  360. const strItem = typeof item !== "string" ? item.toString() : item;
  361. if (key.startsWith(strItem)) {
  362. this._data.delete(key);
  363. data.level.delete(key);
  364. break;
  365. }
  366. }
  367. }
  368. if (this._data.size === 0) {
  369. this._enterIdleMode();
  370. }
  371. }
  372. }
  373. /**
  374. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  375. */
  376. purgeParent(what) {
  377. if (!what) {
  378. this.purge();
  379. } else if (
  380. typeof what === "string" ||
  381. Buffer.isBuffer(what) ||
  382. what instanceof URL ||
  383. typeof what === "number"
  384. ) {
  385. const strWhat = typeof what !== "string" ? what.toString() : what;
  386. this.purge(dirname(strWhat));
  387. } else {
  388. const set = new Set();
  389. for (const item of what) {
  390. const strItem = typeof item !== "string" ? item.toString() : item;
  391. set.add(dirname(strItem));
  392. }
  393. this.purge(set);
  394. }
  395. }
  396. /**
  397. * @param {string} path path
  398. * @param {Error | null} err error
  399. * @param {EXPECTED_ANY} result result
  400. */
  401. _storeResult(path, err, result) {
  402. if (this._data.has(path)) return;
  403. const level = this._levels[this._currentLevel];
  404. this._data.set(path, { err, result, level });
  405. level.add(path);
  406. }
  407. _decayLevel() {
  408. const nextLevel = (this._currentLevel + 1) % this._levels.length;
  409. const decay = this._levels[nextLevel];
  410. this._currentLevel = nextLevel;
  411. for (const item of decay) {
  412. this._data.delete(item);
  413. }
  414. decay.clear();
  415. if (this._data.size === 0) {
  416. this._enterIdleMode();
  417. } else {
  418. /** @type {number} */
  419. (this._nextDecay) += this._tickInterval;
  420. }
  421. }
  422. _runDecays() {
  423. while (
  424. /** @type {number} */ (this._nextDecay) <= Date.now() &&
  425. this._mode !== STORAGE_MODE_IDLE
  426. ) {
  427. this._decayLevel();
  428. }
  429. }
  430. _enterAsyncMode() {
  431. let timeout = 0;
  432. switch (this._mode) {
  433. case STORAGE_MODE_ASYNC:
  434. return;
  435. case STORAGE_MODE_IDLE:
  436. this._nextDecay = Date.now() + this._tickInterval;
  437. timeout = this._tickInterval;
  438. break;
  439. case STORAGE_MODE_SYNC:
  440. this._runDecays();
  441. // _runDecays may change the mode
  442. if (
  443. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  444. (this._mode) === STORAGE_MODE_IDLE
  445. ) {
  446. return;
  447. }
  448. timeout = Math.max(
  449. 0,
  450. /** @type {number} */ (this._nextDecay) - Date.now(),
  451. );
  452. break;
  453. }
  454. this._mode = STORAGE_MODE_ASYNC;
  455. const ref = setTimeout(() => {
  456. this._mode = STORAGE_MODE_SYNC;
  457. this._runDecays();
  458. }, timeout);
  459. if (ref.unref) ref.unref();
  460. this._timeout = ref;
  461. }
  462. _enterSyncModeWhenIdle() {
  463. if (this._mode === STORAGE_MODE_IDLE) {
  464. this._mode = STORAGE_MODE_SYNC;
  465. this._nextDecay = Date.now() + this._tickInterval;
  466. }
  467. }
  468. _enterIdleMode() {
  469. this._mode = STORAGE_MODE_IDLE;
  470. this._nextDecay = undefined;
  471. if (this._timeout) clearTimeout(this._timeout);
  472. }
  473. }
  474. /**
  475. * @template {EXPECTED_FUNCTION} Provider
  476. * @template {EXPECTED_FUNCTION} AsyncProvider
  477. * @template FileSystem
  478. * @param {number} duration duration in ms files are cached
  479. * @param {Provider | undefined} provider provider
  480. * @param {AsyncProvider | undefined} syncProvider sync provider
  481. * @param {BaseFileSystem} providerContext provider context
  482. * @returns {OperationMergerBackend | CacheBackend} backend
  483. */
  484. const createBackend = (duration, provider, syncProvider, providerContext) => {
  485. if (duration > 0) {
  486. return new CacheBackend(duration, provider, syncProvider, providerContext);
  487. }
  488. return new OperationMergerBackend(provider, syncProvider, providerContext);
  489. };
  490. module.exports = class CachedInputFileSystem {
  491. /**
  492. * @param {BaseFileSystem} fileSystem file system
  493. * @param {number} duration duration in ms files are cached
  494. */
  495. constructor(fileSystem, duration) {
  496. this.fileSystem = fileSystem;
  497. this._lstatBackend = createBackend(
  498. duration,
  499. this.fileSystem.lstat,
  500. this.fileSystem.lstatSync,
  501. this.fileSystem,
  502. );
  503. const lstat = this._lstatBackend.provide;
  504. this.lstat = /** @type {FileSystem["lstat"]} */ (lstat);
  505. const lstatSync = this._lstatBackend.provideSync;
  506. this.lstatSync = /** @type {SyncFileSystem["lstatSync"]} */ (lstatSync);
  507. this._statBackend = createBackend(
  508. duration,
  509. this.fileSystem.stat,
  510. this.fileSystem.statSync,
  511. this.fileSystem,
  512. );
  513. const stat = this._statBackend.provide;
  514. this.stat = /** @type {FileSystem["stat"]} */ (stat);
  515. const statSync = this._statBackend.provideSync;
  516. this.statSync = /** @type {SyncFileSystem["statSync"]} */ (statSync);
  517. this._readdirBackend = createBackend(
  518. duration,
  519. this.fileSystem.readdir,
  520. this.fileSystem.readdirSync,
  521. this.fileSystem,
  522. );
  523. const readdir = this._readdirBackend.provide;
  524. this.readdir = /** @type {FileSystem["readdir"]} */ (readdir);
  525. const readdirSync = this._readdirBackend.provideSync;
  526. this.readdirSync = /** @type {SyncFileSystem["readdirSync"]} */ (
  527. readdirSync
  528. );
  529. this._readFileBackend = createBackend(
  530. duration,
  531. this.fileSystem.readFile,
  532. this.fileSystem.readFileSync,
  533. this.fileSystem,
  534. );
  535. const readFile = this._readFileBackend.provide;
  536. this.readFile = /** @type {FileSystem["readFile"]} */ (readFile);
  537. const readFileSync = this._readFileBackend.provideSync;
  538. this.readFileSync = /** @type {SyncFileSystem["readFileSync"]} */ (
  539. readFileSync
  540. );
  541. this._readJsonBackend = createBackend(
  542. duration,
  543. // prettier-ignore
  544. this.fileSystem.readJson ||
  545. (this.readFile &&
  546. (
  547. /**
  548. * @param {string} path path
  549. * @param {FileSystemCallback<EXPECTED_ANY>} callback callback
  550. */
  551. (path, callback) => {
  552. this.readFile(path, (err, buffer) => {
  553. if (err) return callback(err);
  554. if (!buffer || buffer.length === 0)
  555. {return callback(new Error("No file content"));}
  556. let data;
  557. try {
  558. data = JSON.parse(buffer.toString("utf8"));
  559. } catch (err_) {
  560. return callback(/** @type {Error} */ (err_));
  561. }
  562. callback(null, data);
  563. });
  564. })
  565. ),
  566. // prettier-ignore
  567. this.fileSystem.readJsonSync ||
  568. (this.readFileSync &&
  569. (
  570. /**
  571. * @param {string} path path
  572. * @returns {EXPECTED_ANY} result
  573. */
  574. (path) => {
  575. const buffer = this.readFileSync(path);
  576. const data = JSON.parse(buffer.toString("utf8"));
  577. return data;
  578. }
  579. )),
  580. this.fileSystem,
  581. );
  582. const readJson = this._readJsonBackend.provide;
  583. this.readJson = /** @type {FileSystem["readJson"]} */ (readJson);
  584. const readJsonSync = this._readJsonBackend.provideSync;
  585. this.readJsonSync = /** @type {SyncFileSystem["readJsonSync"]} */ (
  586. readJsonSync
  587. );
  588. this._readlinkBackend = createBackend(
  589. duration,
  590. this.fileSystem.readlink,
  591. this.fileSystem.readlinkSync,
  592. this.fileSystem,
  593. );
  594. const readlink = this._readlinkBackend.provide;
  595. this.readlink = /** @type {FileSystem["readlink"]} */ (readlink);
  596. const readlinkSync = this._readlinkBackend.provideSync;
  597. this.readlinkSync = /** @type {SyncFileSystem["readlinkSync"]} */ (
  598. readlinkSync
  599. );
  600. this._realpathBackend = createBackend(
  601. duration,
  602. this.fileSystem.realpath,
  603. this.fileSystem.realpathSync,
  604. this.fileSystem,
  605. );
  606. const realpath = this._realpathBackend.provide;
  607. this.realpath = /** @type {FileSystem["realpath"]} */ (realpath);
  608. const realpathSync = this._realpathBackend.provideSync;
  609. this.realpathSync = /** @type {SyncFileSystem["realpathSync"]} */ (
  610. realpathSync
  611. );
  612. }
  613. /**
  614. * @param {(string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>)=} what what to purge
  615. */
  616. purge(what) {
  617. this._statBackend.purge(what);
  618. this._lstatBackend.purge(what);
  619. this._readdirBackend.purgeParent(what);
  620. this._readFileBackend.purge(what);
  621. this._readlinkBackend.purge(what);
  622. this._readJsonBackend.purge(what);
  623. this._realpathBackend.purge(what);
  624. }
  625. };