tokenize.js 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301
  1. "use strict";
  2. var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
  3. if (k2 === undefined) k2 = k;
  4. var desc = Object.getOwnPropertyDescriptor(m, k);
  5. if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
  6. desc = { enumerable: true, get: function() { return m[k]; } };
  7. }
  8. Object.defineProperty(o, k2, desc);
  9. }) : (function(o, m, k, k2) {
  10. if (k2 === undefined) k2 = k;
  11. o[k2] = m[k];
  12. }));
  13. var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
  14. Object.defineProperty(o, "default", { enumerable: true, value: v });
  15. }) : function(o, v) {
  16. o["default"] = v;
  17. });
  18. var __importStar = (this && this.__importStar) || (function () {
  19. var ownKeys = function(o) {
  20. ownKeys = Object.getOwnPropertyNames || function (o) {
  21. var ar = [];
  22. for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
  23. return ar;
  24. };
  25. return ownKeys(o);
  26. };
  27. return function (mod) {
  28. if (mod && mod.__esModule) return mod;
  29. var result = {};
  30. if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
  31. __setModuleDefault(result, mod);
  32. return result;
  33. };
  34. })();
  35. var _a, _b;
  36. Object.defineProperty(exports, "__esModule", { value: true });
  37. exports.FIELDS = void 0;
  38. exports.default = tokenize;
  39. var t = __importStar(require("./tokenTypes"));
  40. var unescapable = (_a = {},
  41. _a[t.tab] = true,
  42. _a[t.newline] = true,
  43. _a[t.cr] = true,
  44. _a[t.feed] = true,
  45. _a);
  46. var wordDelimiters = (_b = {},
  47. _b[t.space] = true,
  48. _b[t.tab] = true,
  49. _b[t.newline] = true,
  50. _b[t.cr] = true,
  51. _b[t.feed] = true,
  52. _b[t.ampersand] = true,
  53. _b[t.asterisk] = true,
  54. _b[t.bang] = true,
  55. _b[t.comma] = true,
  56. _b[t.colon] = true,
  57. _b[t.semicolon] = true,
  58. _b[t.openParenthesis] = true,
  59. _b[t.closeParenthesis] = true,
  60. _b[t.openSquare] = true,
  61. _b[t.closeSquare] = true,
  62. _b[t.singleQuote] = true,
  63. _b[t.doubleQuote] = true,
  64. _b[t.plus] = true,
  65. _b[t.pipe] = true,
  66. _b[t.tilde] = true,
  67. _b[t.greaterThan] = true,
  68. _b[t.equals] = true,
  69. _b[t.dollar] = true,
  70. _b[t.caret] = true,
  71. _b[t.slash] = true,
  72. _b);
  73. var hex = {};
  74. var hexChars = "0123456789abcdefABCDEF";
  75. for (var i = 0; i < hexChars.length; i++) {
  76. hex[hexChars.charCodeAt(i)] = true;
  77. }
  78. /**
  79. * Returns the last index of the bar css word
  80. * @param {string} css The string in which the word begins
  81. * @param {number} start The index into the string where word's first letter occurs
  82. */
  83. function consumeWord(css, start) {
  84. var next = start;
  85. var code;
  86. do {
  87. code = css.charCodeAt(next);
  88. if (wordDelimiters[code]) {
  89. return next - 1;
  90. }
  91. else if (code === t.backslash) {
  92. next = consumeEscape(css, next) + 1;
  93. }
  94. else {
  95. // All other characters are part of the word
  96. next++;
  97. }
  98. } while (next < css.length);
  99. return next - 1;
  100. }
  101. /**
  102. * Returns the last index of the escape sequence
  103. * @param {string} css The string in which the sequence begins
  104. * @param {number} start The index into the string where escape character (`\`) occurs.
  105. */
  106. function consumeEscape(css, start) {
  107. var next = start;
  108. var code = css.charCodeAt(next + 1);
  109. if (unescapable[code]) {
  110. // just consume the escape char
  111. }
  112. else if (hex[code]) {
  113. var hexDigits = 0;
  114. // consume up to 6 hex chars
  115. do {
  116. next++;
  117. hexDigits++;
  118. code = css.charCodeAt(next + 1);
  119. } while (hex[code] && hexDigits < 6);
  120. // if fewer than 6 hex chars, a trailing space ends the escape
  121. if (hexDigits < 6 && code === t.space) {
  122. next++;
  123. }
  124. }
  125. else {
  126. // the next char is part of the current word
  127. next++;
  128. }
  129. return next;
  130. }
  131. exports.FIELDS = {
  132. TYPE: 0,
  133. START_LINE: 1,
  134. START_COL: 2,
  135. END_LINE: 3,
  136. END_COL: 4,
  137. START_POS: 5,
  138. END_POS: 6,
  139. };
  140. function tokenize(input) {
  141. var tokens = [];
  142. var css = input.css.valueOf();
  143. var length = css.length;
  144. var offset = -1;
  145. var line = 1;
  146. var start = 0;
  147. var end = 0;
  148. var code, content, endColumn, endLine, escaped, escapePos, last, lines, next, nextLine, nextOffset, quote, tokenType;
  149. function unclosed(what, fix) {
  150. if (input.safe) {
  151. // fyi: this is never set to true.
  152. css += fix;
  153. next = css.length - 1;
  154. }
  155. else {
  156. throw input.error("Unclosed " + what, line, start - offset, start);
  157. }
  158. }
  159. while (start < length) {
  160. code = css.charCodeAt(start);
  161. if (code === t.newline) {
  162. offset = start;
  163. line += 1;
  164. }
  165. switch (code) {
  166. case t.space:
  167. case t.tab:
  168. case t.newline:
  169. case t.cr:
  170. case t.feed:
  171. next = start;
  172. do {
  173. next += 1;
  174. code = css.charCodeAt(next);
  175. if (code === t.newline) {
  176. offset = next;
  177. line += 1;
  178. }
  179. } while (code === t.space ||
  180. code === t.newline ||
  181. code === t.tab ||
  182. code === t.cr ||
  183. code === t.feed);
  184. tokenType = t.space;
  185. endLine = line;
  186. endColumn = next - offset - 1;
  187. end = next;
  188. break;
  189. case t.plus:
  190. case t.greaterThan:
  191. case t.tilde:
  192. case t.pipe:
  193. next = start;
  194. do {
  195. next += 1;
  196. code = css.charCodeAt(next);
  197. } while (code === t.plus || code === t.greaterThan || code === t.tilde || code === t.pipe);
  198. tokenType = t.combinator;
  199. endLine = line;
  200. endColumn = start - offset;
  201. end = next;
  202. break;
  203. // Consume these characters as single tokens.
  204. case t.asterisk:
  205. case t.ampersand:
  206. case t.bang:
  207. case t.comma:
  208. case t.equals:
  209. case t.dollar:
  210. case t.caret:
  211. case t.openSquare:
  212. case t.closeSquare:
  213. case t.colon:
  214. case t.semicolon:
  215. case t.openParenthesis:
  216. case t.closeParenthesis:
  217. next = start;
  218. tokenType = code;
  219. endLine = line;
  220. endColumn = start - offset;
  221. end = next + 1;
  222. break;
  223. case t.singleQuote:
  224. case t.doubleQuote:
  225. quote = code === t.singleQuote ? "'" : '"';
  226. next = start;
  227. do {
  228. escaped = false;
  229. next = css.indexOf(quote, next + 1);
  230. if (next === -1) {
  231. unclosed("quote", quote);
  232. }
  233. escapePos = next;
  234. while (css.charCodeAt(escapePos - 1) === t.backslash) {
  235. escapePos -= 1;
  236. escaped = !escaped;
  237. }
  238. } while (escaped);
  239. tokenType = t.str;
  240. endLine = line;
  241. endColumn = start - offset;
  242. end = next + 1;
  243. break;
  244. default:
  245. if (code === t.slash && css.charCodeAt(start + 1) === t.asterisk) {
  246. next = css.indexOf("*/", start + 2) + 1;
  247. if (next === 0) {
  248. unclosed("comment", "*/");
  249. }
  250. content = css.slice(start, next + 1);
  251. lines = content.split("\n");
  252. last = lines.length - 1;
  253. if (last > 0) {
  254. nextLine = line + last;
  255. nextOffset = next - lines[last].length;
  256. }
  257. else {
  258. nextLine = line;
  259. nextOffset = offset;
  260. }
  261. tokenType = t.comment;
  262. line = nextLine;
  263. endLine = nextLine;
  264. endColumn = next - nextOffset;
  265. }
  266. else if (code === t.slash) {
  267. next = start;
  268. tokenType = code;
  269. endLine = line;
  270. endColumn = start - offset;
  271. end = next + 1;
  272. }
  273. else {
  274. next = consumeWord(css, start);
  275. tokenType = t.word;
  276. endLine = line;
  277. endColumn = next - offset;
  278. }
  279. end = next + 1;
  280. break;
  281. }
  282. // Ensure that the token structure remains consistent
  283. tokens.push([
  284. tokenType, // [0] Token type
  285. line, // [1] Starting line
  286. start - offset, // [2] Starting column
  287. endLine, // [3] Ending line
  288. endColumn, // [4] Ending column
  289. start, // [5] Start position / Source index
  290. end, // [6] End position
  291. ]);
  292. // Reset offset for the next token
  293. if (nextOffset) {
  294. offset = nextOffset;
  295. nextOffset = null;
  296. }
  297. start = end;
  298. }
  299. return tokens;
  300. }
  301. //# sourceMappingURL=tokenize.js.map