tokenize.js 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231
  1. 'use strict';
  2. exports.__esModule = true;
  3. exports.default = tokenize;
  4. var singleQuote = 39,
  5. doubleQuote = 34,
  6. backslash = 92,
  7. slash = 47,
  8. newline = 10,
  9. space = 32,
  10. feed = 12,
  11. tab = 9,
  12. cr = 13,
  13. plus = 43,
  14. gt = 62,
  15. tilde = 126,
  16. pipe = 124,
  17. comma = 44,
  18. openBracket = 40,
  19. closeBracket = 41,
  20. openSq = 91,
  21. closeSq = 93,
  22. semicolon = 59,
  23. asterisk = 42,
  24. colon = 58,
  25. ampersand = 38,
  26. at = 64,
  27. atEnd = /[ \n\t\r\{\(\)'"\\;/]/g,
  28. wordEnd = /[ \n\t\r\(\)\*:;@!&'"\+\|~>,\[\]\\]|\/(?=\*)/g;
  29. function tokenize(input) {
  30. var tokens = [];
  31. var css = input.css.valueOf();
  32. var code = void 0,
  33. next = void 0,
  34. quote = void 0,
  35. lines = void 0,
  36. last = void 0,
  37. content = void 0,
  38. escape = void 0,
  39. nextLine = void 0,
  40. nextOffset = void 0,
  41. escaped = void 0,
  42. escapePos = void 0;
  43. var length = css.length;
  44. var offset = -1;
  45. var line = 1;
  46. var pos = 0;
  47. var unclosed = function unclosed(what, end) {
  48. if (input.safe) {
  49. css += end;
  50. next = css.length - 1;
  51. } else {
  52. throw input.error('Unclosed ' + what, line, pos - offset, pos);
  53. }
  54. };
  55. while (pos < length) {
  56. code = css.charCodeAt(pos);
  57. if (code === newline) {
  58. offset = pos;
  59. line += 1;
  60. }
  61. switch (code) {
  62. case newline:
  63. case space:
  64. case tab:
  65. case cr:
  66. case feed:
  67. next = pos;
  68. do {
  69. next += 1;
  70. code = css.charCodeAt(next);
  71. if (code === newline) {
  72. offset = next;
  73. line += 1;
  74. }
  75. } while (code === space || code === newline || code === tab || code === cr || code === feed);
  76. tokens.push(['space', css.slice(pos, next), line, pos - offset, pos]);
  77. pos = next - 1;
  78. break;
  79. case plus:
  80. case gt:
  81. case tilde:
  82. case pipe:
  83. next = pos;
  84. do {
  85. next += 1;
  86. code = css.charCodeAt(next);
  87. } while (code === plus || code === gt || code === tilde || code === pipe);
  88. tokens.push(['combinator', css.slice(pos, next), line, pos - offset, pos]);
  89. pos = next - 1;
  90. break;
  91. case asterisk:
  92. tokens.push(['*', '*', line, pos - offset, pos]);
  93. break;
  94. case ampersand:
  95. tokens.push(['&', '&', line, pos - offset, pos]);
  96. break;
  97. case comma:
  98. tokens.push([',', ',', line, pos - offset, pos]);
  99. break;
  100. case openSq:
  101. tokens.push(['[', '[', line, pos - offset, pos]);
  102. break;
  103. case closeSq:
  104. tokens.push([']', ']', line, pos - offset, pos]);
  105. break;
  106. case colon:
  107. tokens.push([':', ':', line, pos - offset, pos]);
  108. break;
  109. case semicolon:
  110. tokens.push([';', ';', line, pos - offset, pos]);
  111. break;
  112. case openBracket:
  113. tokens.push(['(', '(', line, pos - offset, pos]);
  114. break;
  115. case closeBracket:
  116. tokens.push([')', ')', line, pos - offset, pos]);
  117. break;
  118. case singleQuote:
  119. case doubleQuote:
  120. quote = code === singleQuote ? "'" : '"';
  121. next = pos;
  122. do {
  123. escaped = false;
  124. next = css.indexOf(quote, next + 1);
  125. if (next === -1) {
  126. unclosed('quote', quote);
  127. }
  128. escapePos = next;
  129. while (css.charCodeAt(escapePos - 1) === backslash) {
  130. escapePos -= 1;
  131. escaped = !escaped;
  132. }
  133. } while (escaped);
  134. tokens.push(['string', css.slice(pos, next + 1), line, pos - offset, line, next - offset, pos]);
  135. pos = next;
  136. break;
  137. case at:
  138. atEnd.lastIndex = pos + 1;
  139. atEnd.test(css);
  140. if (atEnd.lastIndex === 0) {
  141. next = css.length - 1;
  142. } else {
  143. next = atEnd.lastIndex - 2;
  144. }
  145. tokens.push(['at-word', css.slice(pos, next + 1), line, pos - offset, line, next - offset, pos]);
  146. pos = next;
  147. break;
  148. case backslash:
  149. next = pos;
  150. escape = true;
  151. while (css.charCodeAt(next + 1) === backslash) {
  152. next += 1;
  153. escape = !escape;
  154. }
  155. code = css.charCodeAt(next + 1);
  156. if (escape && code !== slash && code !== space && code !== newline && code !== tab && code !== cr && code !== feed) {
  157. next += 1;
  158. }
  159. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset, pos]);
  160. pos = next;
  161. break;
  162. default:
  163. if (code === slash && css.charCodeAt(pos + 1) === asterisk) {
  164. next = css.indexOf('*/', pos + 2) + 1;
  165. if (next === 0) {
  166. unclosed('comment', '*/');
  167. }
  168. content = css.slice(pos, next + 1);
  169. lines = content.split('\n');
  170. last = lines.length - 1;
  171. if (last > 0) {
  172. nextLine = line + last;
  173. nextOffset = next - lines[last].length;
  174. } else {
  175. nextLine = line;
  176. nextOffset = offset;
  177. }
  178. tokens.push(['comment', content, line, pos - offset, nextLine, next - nextOffset, pos]);
  179. offset = nextOffset;
  180. line = nextLine;
  181. pos = next;
  182. } else {
  183. wordEnd.lastIndex = pos + 1;
  184. wordEnd.test(css);
  185. if (wordEnd.lastIndex === 0) {
  186. next = css.length - 1;
  187. } else {
  188. next = wordEnd.lastIndex - 2;
  189. }
  190. tokens.push(['word', css.slice(pos, next + 1), line, pos - offset, line, next - offset, pos]);
  191. pos = next;
  192. }
  193. break;
  194. }
  195. pos++;
  196. }
  197. return tokens;
  198. }
  199. module.exports = exports['default'];