parse-chunked.cjs 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572
  1. 'use strict';
  2. const utils = require('./utils.cjs');
  3. const NO_VALUE = Symbol('empty');
  4. const STACK_OBJECT = 1;
  5. const STACK_ARRAY = 2;
  6. const MODE_JSON = 0;
  7. const MODE_JSONL = 1;
  8. const MODE_JSONL_AUTO = 2;
  9. const decoder = new TextDecoder();
  10. function adjustPosition(error, jsonParseOffset) {
  11. if (error.name === 'SyntaxError' && jsonParseOffset) {
  12. error.message = error.message.replace(/at position (\d+)/, (_, pos) =>
  13. 'at position ' + (Number(pos) + jsonParseOffset)
  14. );
  15. }
  16. return error;
  17. }
  18. function append(array, elements) {
  19. // Note: Avoid using array.push(...elements) since it may lead to
  20. // "RangeError: Maximum call stack size exceeded" for long arrays
  21. const initialLength = array.length;
  22. array.length += elements.length;
  23. for (let i = 0; i < elements.length; i++) {
  24. array[initialLength + i] = elements[i];
  25. }
  26. }
  27. function resolveParseMode(mode) {
  28. switch (mode) {
  29. case 'json':
  30. return MODE_JSON;
  31. case 'jsonl':
  32. return MODE_JSONL;
  33. case 'auto':
  34. return MODE_JSONL_AUTO;
  35. default:
  36. throw new TypeError('Invalid options: `mode` should be "json", "jsonl", or "auto"');
  37. }
  38. }
  39. function parseChunkedOptions(value) {
  40. const options = typeof value === 'function'
  41. ? { reviver: value }
  42. : value || {};
  43. return {
  44. mode: resolveParseMode(options.mode ?? 'json'),
  45. reviver: options.reviver ?? null,
  46. onRootValue: options.onRootValue ?? null,
  47. onChunk: options.onChunk ?? null
  48. };
  49. }
  50. function applyReviver(value, reviver) {
  51. return walk({ '': value }, '', value);
  52. function walk(holder, key, value) {
  53. if (value && typeof value === 'object') {
  54. for (const childKey of Object.keys(value)) {
  55. const childValue = value[childKey];
  56. const newValue = walk(value, childKey, childValue);
  57. if (newValue === undefined) {
  58. delete value[childKey];
  59. } else if (newValue !== childValue) {
  60. value[childKey] = newValue;
  61. }
  62. }
  63. }
  64. return reviver.call(holder, key, value);
  65. }
  66. }
  67. async function parseChunked(chunkEmitter, optionsOrReviver) {
  68. const { mode, reviver, onRootValue, onChunk } = parseChunkedOptions(optionsOrReviver);
  69. const iterable = typeof chunkEmitter === 'function'
  70. ? chunkEmitter()
  71. : chunkEmitter;
  72. if (utils.isIterable(iterable)) {
  73. const parser = createChunkParser(mode, reviver, onRootValue, onChunk);
  74. try {
  75. for await (const chunk of iterable) {
  76. if (typeof chunk !== 'string' && !ArrayBuffer.isView(chunk)) {
  77. throw new TypeError('Invalid chunk: Expected string, TypedArray or Buffer');
  78. }
  79. parser.push(chunk);
  80. }
  81. return parser.finish();
  82. } catch (e) {
  83. throw adjustPosition(e, parser.jsonParseOffset);
  84. }
  85. }
  86. throw new TypeError(
  87. 'Invalid chunk emitter: Expected an Iterable, AsyncIterable, generator, ' +
  88. 'async generator, or a function returning an Iterable or AsyncIterable'
  89. );
  90. }
  91. function createChunkParser(parseMode, reviver, onRootValue, onChunk) {
  92. let rootValues = parseMode === MODE_JSONL ? [] : null;
  93. let rootValuesCount = 0;
  94. let currentRootValue = NO_VALUE;
  95. let currentRootValueCursor = null;
  96. let consumedChunkLength = 0;
  97. let parsedChunkLength = 0;
  98. let prevArray = null;
  99. let prevArraySlices = [];
  100. let stack = new Array(100);
  101. let lastFlushDepth = 0;
  102. let flushDepth = 0;
  103. let stateString = false;
  104. let stateStringEscape = false;
  105. let seenNonWhiteSpace = false;
  106. let allowNewRootValue = true;
  107. let pendingByteSeq = null;
  108. let pendingChunk = null;
  109. let jsonParseOffset = 0;
  110. const state = Object.freeze({
  111. get mode() {
  112. return parseMode === MODE_JSONL ? 'jsonl' : 'json';
  113. },
  114. get returnValue() {
  115. return typeof onRootValue === 'function'
  116. ? rootValuesCount
  117. : rootValues !== null
  118. ? rootValues
  119. : currentRootValue !== NO_VALUE
  120. ? currentRootValue
  121. : undefined;
  122. },
  123. get currentRootValue() {
  124. return currentRootValue !== NO_VALUE ? currentRootValue : undefined;
  125. },
  126. get rootValuesCount() {
  127. return rootValuesCount;
  128. },
  129. get consumed() {
  130. return consumedChunkLength;
  131. },
  132. get parsed() {
  133. return parsedChunkLength;
  134. }
  135. });
  136. return {
  137. push,
  138. finish,
  139. state,
  140. get jsonParseOffset() {
  141. return jsonParseOffset;
  142. }
  143. };
  144. function startRootValue(fragment) {
  145. // Extra non-whitespace after complete root value should fail to parse
  146. if (!allowNewRootValue) {
  147. jsonParseOffset -= 2;
  148. JSON.parse('[]' + fragment);
  149. }
  150. // In "auto" mode, switch to JSONL when a second root value is starting after a newline
  151. if (currentRootValue !== NO_VALUE && parseMode === MODE_JSONL_AUTO) {
  152. parseMode = MODE_JSONL;
  153. rootValues = [currentRootValue];
  154. }
  155. // Block parsing of an additional root value until a newline is encountered
  156. allowNewRootValue = false;
  157. // Parse fragment as a new root value
  158. currentRootValue = JSON.parse(fragment);
  159. }
  160. function finishRootValue() {
  161. rootValuesCount++;
  162. if (typeof reviver === 'function') {
  163. currentRootValue = applyReviver(currentRootValue, reviver);
  164. }
  165. if (typeof onRootValue === 'function') {
  166. onRootValue(currentRootValue, state);
  167. } else if (parseMode === MODE_JSONL) {
  168. rootValues.push(currentRootValue);
  169. }
  170. }
  171. function mergeArraySlices() {
  172. if (prevArray === null) {
  173. return;
  174. }
  175. if (prevArraySlices.length !== 0) {
  176. const newArray = prevArraySlices.length === 1
  177. ? prevArray.concat(prevArraySlices[0])
  178. : prevArray.concat(...prevArraySlices);
  179. if (currentRootValueCursor.prev !== null) {
  180. currentRootValueCursor.prev.value[currentRootValueCursor.key] = newArray;
  181. } else {
  182. currentRootValue = newArray;
  183. }
  184. currentRootValueCursor.value = newArray;
  185. prevArraySlices = [];
  186. }
  187. prevArray = null;
  188. }
  189. function parseAndAppend(fragment, wrap) {
  190. // Append new entries or elements
  191. if (stack[lastFlushDepth - 1] === STACK_OBJECT) {
  192. if (wrap) {
  193. jsonParseOffset--;
  194. fragment = '{' + fragment + '}';
  195. }
  196. Object.assign(currentRootValueCursor.value, JSON.parse(fragment));
  197. } else {
  198. if (wrap) {
  199. jsonParseOffset--;
  200. fragment = '[' + fragment + ']';
  201. }
  202. if (prevArray === currentRootValueCursor.value) {
  203. prevArraySlices.push(JSON.parse(fragment));
  204. } else {
  205. append(currentRootValueCursor.value, JSON.parse(fragment));
  206. prevArray = currentRootValueCursor.value;
  207. }
  208. }
  209. }
  210. function prepareAddition(fragment) {
  211. const { value } = currentRootValueCursor;
  212. const expectComma = Array.isArray(value)
  213. ? value.length !== 0
  214. : Object.keys(value).length !== 0;
  215. if (expectComma) {
  216. // Skip a comma at the beginning of fragment, otherwise it would
  217. // fail to parse
  218. if (fragment[0] === ',') {
  219. jsonParseOffset++;
  220. return fragment.slice(1);
  221. }
  222. // When value (an object or array) is not empty and a fragment
  223. // doesn't start with a comma, a single valid fragment starting
  224. // is a closing bracket. If it's not, a prefix is adding to fail
  225. // parsing. Otherwise, the sequence of chunks can be successfully
  226. // parsed, although it should not, e.g. ["[{}", "{}]"]
  227. if (fragment[0] !== '}' && fragment[0] !== ']') {
  228. jsonParseOffset -= 3;
  229. return '[[]' + fragment;
  230. }
  231. }
  232. return fragment;
  233. }
  234. function flush(chunk, start, end) {
  235. let fragment = chunk.slice(start, end);
  236. // Save position correction for an error in JSON.parse() if any
  237. jsonParseOffset = consumedChunkLength + start;
  238. parsedChunkLength += end - start;
  239. // Prepend pending chunk if any
  240. if (pendingChunk !== null) {
  241. fragment = pendingChunk + fragment;
  242. jsonParseOffset -= pendingChunk.length;
  243. parsedChunkLength += pendingChunk.length;
  244. pendingChunk = null;
  245. }
  246. if (flushDepth === lastFlushDepth) {
  247. // Depth didn't change, so it's a continuation of the current value or entire value if it's a root one
  248. if (lastFlushDepth === 0) {
  249. startRootValue(fragment);
  250. } else {
  251. parseAndAppend(prepareAddition(fragment), true);
  252. }
  253. } else if (flushDepth > lastFlushDepth) {
  254. // Add missed closing brackets/parentheses
  255. for (let i = flushDepth - 1; i >= lastFlushDepth; i--) {
  256. fragment += stack[i] === STACK_OBJECT ? '}' : ']';
  257. }
  258. if (lastFlushDepth === 0) {
  259. startRootValue(fragment);
  260. currentRootValueCursor = {
  261. value: currentRootValue,
  262. key: null,
  263. prev: null
  264. };
  265. } else {
  266. parseAndAppend(prepareAddition(fragment), true);
  267. mergeArraySlices();
  268. }
  269. // Move down to the depths to the last object/array, which is current now
  270. for (let i = lastFlushDepth || 1; i < flushDepth; i++) {
  271. let { value } = currentRootValueCursor;
  272. let key = null;
  273. if (stack[i - 1] === STACK_OBJECT) {
  274. // Find last entry
  275. // eslint-disable-next-line curly
  276. for (key in value);
  277. value = value[key];
  278. } else {
  279. // Last element
  280. key = value.length - 1;
  281. value = value[key];
  282. }
  283. currentRootValueCursor = {
  284. value,
  285. key,
  286. prev: currentRootValueCursor
  287. };
  288. }
  289. } else /* flushDepth < lastFlushDepth */ {
  290. fragment = prepareAddition(fragment);
  291. // Add missed opening brackets/parentheses
  292. for (let i = lastFlushDepth - 1; i >= flushDepth; i--) {
  293. jsonParseOffset--;
  294. fragment = (stack[i] === STACK_OBJECT ? '{' : '[') + fragment;
  295. }
  296. parseAndAppend(fragment, false);
  297. mergeArraySlices();
  298. for (let i = lastFlushDepth - 1; i >= flushDepth; i--) {
  299. currentRootValueCursor = currentRootValueCursor.prev;
  300. }
  301. }
  302. if (flushDepth === 0) {
  303. finishRootValue();
  304. }
  305. lastFlushDepth = flushDepth;
  306. seenNonWhiteSpace = false;
  307. }
  308. function ensureChunkString(chunk) {
  309. if (typeof chunk !== 'string') {
  310. // Suppose chunk is Buffer or Uint8Array
  311. // Prepend uncompleted byte sequence if any
  312. if (pendingByteSeq !== null) {
  313. const origRawChunk = chunk;
  314. chunk = new Uint8Array(pendingByteSeq.length + origRawChunk.length);
  315. chunk.set(pendingByteSeq);
  316. chunk.set(origRawChunk, pendingByteSeq.length);
  317. pendingByteSeq = null;
  318. }
  319. // In case Buffer/Uint8Array, an input is encoded in UTF8
  320. // Seek for parts of uncompleted UTF8 symbol on the ending
  321. // This makes sense only if we expect more chunks and last char is not multi-bytes
  322. if (chunk[chunk.length - 1] > 127) {
  323. for (let seqLength = 0; seqLength < chunk.length; seqLength++) {
  324. const byte = chunk[chunk.length - 1 - seqLength];
  325. // 10xxxxxx - 2nd, 3rd or 4th byte
  326. // 110xxxxx – first byte of 2-byte sequence
  327. // 1110xxxx - first byte of 3-byte sequence
  328. // 11110xxx - first byte of 4-byte sequence
  329. if (byte >> 6 === 3) {
  330. seqLength++;
  331. // If the sequence is really incomplete, then preserve it
  332. // for the future chunk and cut off it from the current chunk
  333. if ((seqLength !== 4 && byte >> 3 === 0b11110) ||
  334. (seqLength !== 3 && byte >> 4 === 0b1110) ||
  335. (seqLength !== 2 && byte >> 5 === 0b110)) {
  336. pendingByteSeq = chunk.slice(chunk.length - seqLength); // use slice to avoid tying chunk
  337. chunk = chunk.subarray(0, -seqLength); // use subarray to avoid buffer copy
  338. }
  339. break;
  340. }
  341. }
  342. }
  343. // Convert chunk to a string, since single decode per chunk
  344. // is much effective than decode multiple small substrings
  345. chunk = decoder.decode(chunk);
  346. }
  347. return chunk;
  348. }
  349. function push(chunk) {
  350. chunk = ensureChunkString(chunk);
  351. const chunkLength = chunk.length;
  352. const prevParsedChunkLength = parsedChunkLength;
  353. let lastFlushPoint = 0;
  354. let flushPoint = 0;
  355. // Main scan loop
  356. scan: for (let i = 0; i < chunkLength; i++) {
  357. if (stateString) {
  358. for (; i < chunkLength; i++) {
  359. if (stateStringEscape) {
  360. stateStringEscape = false;
  361. } else {
  362. switch (chunk.charCodeAt(i)) {
  363. case 0x22: /* " */
  364. stateString = false;
  365. continue scan;
  366. case 0x5C: /* \ */
  367. stateStringEscape = true;
  368. }
  369. }
  370. }
  371. break;
  372. }
  373. switch (chunk.charCodeAt(i)) {
  374. case 0x22: /* " */
  375. stateString = true;
  376. stateStringEscape = false;
  377. seenNonWhiteSpace = true;
  378. break;
  379. case 0x2C: /* , */
  380. flushPoint = i;
  381. break;
  382. case 0x7B: /* { */
  383. // Open an object
  384. flushPoint = i + 1;
  385. stack[flushDepth++] = STACK_OBJECT;
  386. seenNonWhiteSpace = true;
  387. break;
  388. case 0x5B: /* [ */
  389. // Open an array
  390. flushPoint = i + 1;
  391. stack[flushDepth++] = STACK_ARRAY;
  392. seenNonWhiteSpace = true;
  393. break;
  394. case 0x5D: /* ] */
  395. case 0x7D: /* } */
  396. // Close an object or array
  397. flushPoint = i + 1;
  398. if (flushDepth === 0) {
  399. // Unmatched closing bracket/brace at top level, should fail to parse
  400. break scan;
  401. }
  402. flushDepth--;
  403. // Flush on depth decrease related to last flush, otherwise wait for more chunks to flush together
  404. if (flushDepth < lastFlushDepth) {
  405. flush(chunk, lastFlushPoint, flushPoint);
  406. lastFlushPoint = flushPoint;
  407. }
  408. break;
  409. case 0x09: /* \t */
  410. case 0x0A: /* \n */
  411. case 0x0D: /* \r */
  412. case 0x20: /* space */
  413. if (flushDepth === 0) {
  414. if (seenNonWhiteSpace) {
  415. flushPoint = i;
  416. flush(chunk, lastFlushPoint, flushPoint);
  417. lastFlushPoint = flushPoint;
  418. }
  419. if (parseMode !== MODE_JSON &&
  420. allowNewRootValue === false &&
  421. (chunk.charCodeAt(i) === 0x0A || chunk.charCodeAt(i) === 0x0D)
  422. ) {
  423. allowNewRootValue = true;
  424. }
  425. if (flushPoint === i) {
  426. parsedChunkLength++;
  427. }
  428. }
  429. // Move points forward when they point to current position and it's a whitespace
  430. if (lastFlushPoint === i) {
  431. lastFlushPoint++;
  432. }
  433. if (flushPoint === i) {
  434. flushPoint++;
  435. }
  436. break;
  437. default:
  438. seenNonWhiteSpace = true;
  439. }
  440. }
  441. if (flushPoint > lastFlushPoint) {
  442. flush(chunk, lastFlushPoint, flushPoint);
  443. }
  444. // Produce pendingChunk if something left
  445. if (flushPoint < chunkLength) {
  446. if (pendingChunk !== null) {
  447. // When there is already a pending chunk then no flush happened,
  448. // appending entire chunk to pending one
  449. pendingChunk += chunk;
  450. } else {
  451. // Create a pending chunk, it will start with non-whitespace since
  452. // flushPoint was moved forward away from whitespaces on scan
  453. pendingChunk = chunk.slice(flushPoint, chunkLength);
  454. }
  455. }
  456. consumedChunkLength += chunkLength;
  457. if (typeof onChunk === 'function') {
  458. onChunk(parsedChunkLength - prevParsedChunkLength, chunk, pendingChunk, state);
  459. }
  460. }
  461. function finish() {
  462. if (pendingChunk !== null || (currentRootValue === NO_VALUE && parseMode !== MODE_JSONL)) {
  463. // Force the `flushDepth < lastFlushDepth` branch in flush() to prepend missed
  464. // opening brackets/parentheses and produce a natural JSON.parse() EOF error
  465. flushDepth = 0;
  466. flush('', 0, 0);
  467. }
  468. if (typeof onChunk === 'function') {
  469. parsedChunkLength = consumedChunkLength;
  470. onChunk(0, null, null, state);
  471. }
  472. const result = state.returnValue;
  473. rootValues = null;
  474. currentRootValue = NO_VALUE;
  475. return result;
  476. }
  477. }
  478. exports.parseChunked = parseChunked;