BVB Source Codes

reactide Show tokenize.js Source code

Return Download reactide: download tokenize.js Source code - Download reactide Source code - Type:.js
  1. import {isIdentifierStart, isIdentifierChar} from "./identifier"
  2. import {types as tt, keywords as keywordTypes} from "./tokentype"
  3. import {Parser} from "./state"
  4. import {SourceLocation} from "./locutil"
  5. import {lineBreak, lineBreakG, isNewLine, nonASCIIwhitespace} from "./whitespace"
  6.  
  7. // Object type used to represent tokens. Note that normally, tokens
  8. // simply exist as properties on the parser object. This is only
  9. // used for the onToken callback and the external tokenizer.
  10.  
  11. export class Token {
  12.   constructor(p) {
  13.     this.type = p.type
  14.     this.value = p.value
  15.     this.start = p.start
  16.     this.end = p.end
  17.     if (p.options.locations)
  18.       this.loc = new SourceLocation(p, p.startLoc, p.endLoc)
  19.     if (p.options.ranges)
  20.       this.range = [p.start, p.end]
  21.   }
  22. }
  23.  
  24. // ## Tokenizer
  25.  
  26. const pp = Parser.prototype
  27.  
  28. // Are we running under Rhino?
  29. const isRhino = typeof Packages == "object" && Object.prototype.toString.call(Packages) == "[object JavaPackage]"
  30.  
  31. // Move to the next token
  32.  
  33. pp.next = function() {
  34.   if (this.options.onToken)
  35.     this.options.onToken(new Token(this))
  36.  
  37.   this.lastTokEnd = this.end
  38.   this.lastTokStart = this.start
  39.   this.lastTokEndLoc = this.endLoc
  40.   this.lastTokStartLoc = this.startLoc
  41.   this.nextToken()
  42. }
  43.  
  44. pp.getToken = function() {
  45.   this.next()
  46.   return new Token(this)
  47. }
  48.  
  49. // If we're in an ES6 environment, make parsers iterable
  50. if (typeof Symbol !== "undefined")
  51.   pp[Symbol.iterator] = function () {
  52.     let self = this
  53.     return {next: function () {
  54.       let token = self.getToken()
  55.       return {
  56.         done: token.type === tt.eof,
  57.         value: token
  58.       }
  59.     }}
  60.   }
  61.  
  62. // Toggle strict mode. Re-reads the next number or string to please
  63. // pedantic tests (`"use strict"; 010;` should fail).
  64.  
  65. pp.setStrict = function(strict) {
  66.   this.strict = strict
  67.   if (this.type !== tt.num && this.type !== tt.string) return
  68.   this.pos = this.start
  69.   if (this.options.locations) {
  70.     while (this.pos < this.lineStart) {
  71.       this.lineStart = this.input.lastIndexOf("\n", this.lineStart - 2) + 1
  72.       --this.curLine
  73.     }
  74.   }
  75.   this.nextToken()
  76. }
  77.  
  78. pp.curContext = function() {
  79.   return this.context[this.context.length - 1]
  80. }
  81.  
  82. // Read a single token, updating the parser object's token-related
  83. // properties.
  84.  
  85. pp.nextToken = function() {
  86.   let curContext = this.curContext()
  87.   if (!curContext || !curContext.preserveSpace) this.skipSpace()
  88.  
  89.   this.start = this.pos
  90.   if (this.options.locations) this.startLoc = this.curPosition()
  91.   if (this.pos >= this.input.length) return this.finishToken(tt.eof)
  92.  
  93.   if (curContext.override) return curContext.override(this)
  94.   else this.readToken(this.fullCharCodeAtPos())
  95. }
  96.  
  97. pp.readToken = function(code) {
  98.   // Identifier or keyword. '\uXXXX' sequences are allowed in
  99.   // identifiers, so '\' also dispatches to that.
  100.   if (isIdentifierStart(code, this.options.ecmaVersion >= 6) || code === 92 /* '\' */)
  101.     return this.readWord()
  102.  
  103.   return this.getTokenFromCode(code)
  104. }
  105.  
  106. pp.fullCharCodeAtPos = function() {
  107.   let code = this.input.charCodeAt(this.pos)
  108.   if (code <= 0xd7ff || code >= 0xe000) return code
  109.   let next = this.input.charCodeAt(this.pos + 1)
  110.   return (code << 10) + next - 0x35fdc00
  111. }
  112.  
  113. pp.skipBlockComment = function() {
  114.   let startLoc = this.options.onComment && this.curPosition()
  115.   let start = this.pos, end = this.input.indexOf("*/", this.pos += 2)
  116.   if (end === -1) this.raise(this.pos - 2, "Unterminated comment")
  117.   this.pos = end + 2
  118.   if (this.options.locations) {
  119.     lineBreakG.lastIndex = start
  120.     let match
  121.     while ((match = lineBreakG.exec(this.input)) && match.index < this.pos) {
  122.       ++this.curLine
  123.       this.lineStart = match.index + match[0].length
  124.     }
  125.   }
  126.   if (this.options.onComment)
  127.     this.options.onComment(true, this.input.slice(start + 2, end), start, this.pos,
  128.                            startLoc, this.curPosition())
  129. }
  130.  
  131. pp.skipLineComment = function(startSkip) {
  132.   let start = this.pos
  133.   let startLoc = this.options.onComment && this.curPosition()
  134.   let ch = this.input.charCodeAt(this.pos+=startSkip)
  135.   while (this.pos < this.input.length && ch !== 10 && ch !== 13 && ch !== 8232 && ch !== 8233) {
  136.     ++this.pos
  137.     ch = this.input.charCodeAt(this.pos)
  138.   }
  139.   if (this.options.onComment)
  140.     this.options.onComment(false, this.input.slice(start + startSkip, this.pos), start, this.pos,
  141.                            startLoc, this.curPosition())
  142. }
  143.  
  144. // Called at the start of the parse and after every token. Skips
  145. // whitespace and comments, and.
  146.  
  147. pp.skipSpace = function() {
  148.   loop: while (this.pos < this.input.length) {
  149.     let ch = this.input.charCodeAt(this.pos)
  150.     switch (ch) {
  151.       case 32: case 160: // ' '
  152.         ++this.pos
  153.         break
  154.       case 13:
  155.         if (this.input.charCodeAt(this.pos + 1) === 10) {
  156.           ++this.pos
  157.         }
  158.       case 10: case 8232: case 8233:
  159.         ++this.pos
  160.         if (this.options.locations) {
  161.           ++this.curLine
  162.           this.lineStart = this.pos
  163.         }
  164.         break
  165.       case 47: // '/'
  166.         switch (this.input.charCodeAt(this.pos + 1)) {
  167.           case 42: // '*'
  168.             this.skipBlockComment()
  169.             break
  170.           case 47:
  171.             this.skipLineComment(2)
  172.             break
  173.           default:
  174.             break loop
  175.         }
  176.         break
  177.       default:
  178.         if (ch > 8 && ch < 14 || ch >= 5760 && nonASCIIwhitespace.test(String.fromCharCode(ch))) {
  179.           ++this.pos
  180.         } else {
  181.           break loop
  182.         }
  183.     }
  184.   }
  185. }
  186.  
  187. // Called at the end of every token. Sets `end`, `val`, and
  188. // maintains `context` and `exprAllowed`, and skips the space after
  189. // the token, so that the next one's `start` will point at the
  190. // right position.
  191.  
  192. pp.finishToken = function(type, val) {
  193.   this.end = this.pos
  194.   if (this.options.locations) this.endLoc = this.curPosition()
  195.   let prevType = this.type
  196.   this.type = type
  197.   this.value = val
  198.  
  199.   this.updateContext(prevType)
  200. }
  201.  
  202. // ### Token reading
  203.  
  204. // This is the function that is called to fetch the next token. It
  205. // is somewhat obscure, because it works in character codes rather
  206. // than characters, and because operator parsing has been inlined
  207. // into it.
  208. //
  209. // All in the name of speed.
  210. //
  211. pp.readToken_dot = function() {
  212.   let next = this.input.charCodeAt(this.pos + 1)
  213.   if (next >= 48 && next <= 57) return this.readNumber(true)
  214.   let next2 = this.input.charCodeAt(this.pos + 2)
  215.   if (this.options.ecmaVersion >= 6 && next === 46 && next2 === 46) { // 46 = dot '.'
  216.     this.pos += 3
  217.     return this.finishToken(tt.ellipsis)
  218.   } else {
  219.     ++this.pos
  220.     return this.finishToken(tt.dot)
  221.   }
  222. }
  223.  
  224. pp.readToken_slash = function() { // '/'
  225.   let next = this.input.charCodeAt(this.pos + 1)
  226.   if (this.exprAllowed) {++this.pos; return this.readRegexp()}
  227.   if (next === 61) return this.finishOp(tt.assign, 2)
  228.   return this.finishOp(tt.slash, 1)
  229. }
  230.  
  231. pp.readToken_mult_modulo_exp = function(code) { // '%*'
  232.   let next = this.input.charCodeAt(this.pos + 1)
  233.   let size = 1
  234.   let tokentype = code === 42 ? tt.star : tt.modulo
  235.  
  236.   // exponentiation operator ** and **=
  237.   if (this.options.ecmaVersion >= 7 && next === 42) {
  238.     ++size
  239.     tokentype = tt.starstar
  240.     next = this.input.charCodeAt(this.pos + 2)
  241.   }
  242.  
  243.   if (next === 61) return this.finishOp(tt.assign, size + 1)
  244.   return this.finishOp(tokentype, size)
  245. }
  246.  
  247. pp.readToken_pipe_amp = function(code) { // '|&'
  248.   let next = this.input.charCodeAt(this.pos + 1)
  249.   if (next === code) return this.finishOp(code === 124 ? tt.logicalOR : tt.logicalAND, 2)
  250.   if (next === 61) return this.finishOp(tt.assign, 2)
  251.   return this.finishOp(code === 124 ? tt.bitwiseOR : tt.bitwiseAND, 1)
  252. }
  253.  
  254. pp.readToken_caret = function() { // '^'
  255.   let next = this.input.charCodeAt(this.pos + 1)
  256.   if (next === 61) return this.finishOp(tt.assign, 2)
  257.   return this.finishOp(tt.bitwiseXOR, 1)
  258. }
  259.  
  260. pp.readToken_plus_min = function(code) { // '+-'
  261.   let next = this.input.charCodeAt(this.pos + 1)
  262.   if (next === code) {
  263.     if (next == 45 && this.input.charCodeAt(this.pos + 2) == 62 &&
  264.         lineBreak.test(this.input.slice(this.lastTokEnd, this.pos))) {
  265.       // A `-->` line comment
  266.       this.skipLineComment(3)
  267.       this.skipSpace()
  268.       return this.nextToken()
  269.     }
  270.     return this.finishOp(tt.incDec, 2)
  271.   }
  272.   if (next === 61) return this.finishOp(tt.assign, 2)
  273.   return this.finishOp(tt.plusMin, 1)
  274. }
  275.  
  276. pp.readToken_lt_gt = function(code) { // '<>'
  277.   let next = this.input.charCodeAt(this.pos + 1)
  278.   let size = 1
  279.   if (next === code) {
  280.     size = code === 62 && this.input.charCodeAt(this.pos + 2) === 62 ? 3 : 2
  281.     if (this.input.charCodeAt(this.pos + size) === 61) return this.finishOp(tt.assign, size + 1)
  282.     return this.finishOp(tt.bitShift, size)
  283.   }
  284.   if (next == 33 && code == 60 && this.input.charCodeAt(this.pos + 2) == 45 &&
  285.       this.input.charCodeAt(this.pos + 3) == 45) {
  286.     if (this.inModule) this.unexpected()
  287.     // `<!--`, an XML-style comment that should be interpreted as a line comment
  288.     this.skipLineComment(4)
  289.     this.skipSpace()
  290.     return this.nextToken()
  291.   }
  292.   if (next === 61) size = 2
  293.   return this.finishOp(tt.relational, size)
  294. }
  295.  
  296. pp.readToken_eq_excl = function(code) { // '=!'
  297.   let next = this.input.charCodeAt(this.pos + 1)
  298.   if (next === 61) return this.finishOp(tt.equality, this.input.charCodeAt(this.pos + 2) === 61 ? 3 : 2)
  299.   if (code === 61 && next === 62 && this.options.ecmaVersion >= 6) { // '=>'
  300.     this.pos += 2
  301.     return this.finishToken(tt.arrow)
  302.   }
  303.   return this.finishOp(code === 61 ? tt.eq : tt.prefix, 1)
  304. }
  305.  
  306. pp.getTokenFromCode = function(code) {
  307.   switch (code) {
  308.     // The interpretation of a dot depends on whether it is followed
  309.     // by a digit or another two dots.
  310.   case 46: // '.'
  311.     return this.readToken_dot()
  312.  
  313.     // Punctuation tokens.
  314.   case 40: ++this.pos; return this.finishToken(tt.parenL)
  315.   case 41: ++this.pos; return this.finishToken(tt.parenR)
  316.   case 59: ++this.pos; return this.finishToken(tt.semi)
  317.   case 44: ++this.pos; return this.finishToken(tt.comma)
  318.   case 91: ++this.pos; return this.finishToken(tt.bracketL)
  319.   case 93: ++this.pos; return this.finishToken(tt.bracketR)
  320.   case 123: ++this.pos; return this.finishToken(tt.braceL)
  321.   case 125: ++this.pos; return this.finishToken(tt.braceR)
  322.   case 58: ++this.pos; return this.finishToken(tt.colon)
  323.   case 63: ++this.pos; return this.finishToken(tt.question)
  324.  
  325.   case 96: // '`'
  326.     if (this.options.ecmaVersion < 6) break
  327.     ++this.pos
  328.     return this.finishToken(tt.backQuote)
  329.  
  330.   case 48: // '0'
  331.     let next = this.input.charCodeAt(this.pos + 1)
  332.     if (next === 120 || next === 88) return this.readRadixNumber(16) // '0x', '0X' - hex number
  333.     if (this.options.ecmaVersion >= 6) {
  334.       if (next === 111 || next === 79) return this.readRadixNumber(8) // '0o', '0O' - octal number
  335.       if (next === 98 || next === 66) return this.readRadixNumber(2) // '0b', '0B' - binary number
  336.     }
  337.     // Anything else beginning with a digit is an integer, octal
  338.     // number, or float.
  339.   case 49: case 50: case 51: case 52: case 53: case 54: case 55: case 56: case 57: // 1-9
  340.     return this.readNumber(false)
  341.  
  342.     // Quotes produce strings.
  343.   case 34: case 39: // '"', "'"
  344.     return this.readString(code)
  345.  
  346.     // Operators are parsed inline in tiny state machines. '=' (61) is
  347.     // often referred to. `finishOp` simply skips the amount of
  348.     // characters it is given as second argument, and returns a token
  349.     // of the type given by its first argument.
  350.  
  351.   case 47: // '/'
  352.     return this.readToken_slash()
  353.  
  354.   case 37: case 42: // '%*'
  355.     return this.readToken_mult_modulo_exp(code)
  356.  
  357.   case 124: case 38: // '|&'
  358.     return this.readToken_pipe_amp(code)
  359.  
  360.   case 94: // '^'
  361.     return this.readToken_caret()
  362.  
  363.   case 43: case 45: // '+-'
  364.     return this.readToken_plus_min(code)
  365.  
  366.   case 60: case 62: // '<>'
  367.     return this.readToken_lt_gt(code)
  368.  
  369.   case 61: case 33: // '=!'
  370.     return this.readToken_eq_excl(code)
  371.  
  372.   case 126: // '~'
  373.     return this.finishOp(tt.prefix, 1)
  374.   }
  375.  
  376.   this.raise(this.pos, "Unexpected character '" + codePointToString(code) + "'")
  377. }
  378.  
  379. pp.finishOp = function(type, size) {
  380.   let str = this.input.slice(this.pos, this.pos + size)
  381.   this.pos += size
  382.   return this.finishToken(type, str)
  383. }
  384.  
  385. // Parse a regular expression. Some context-awareness is necessary,
  386. // since a '/' inside a '[]' set does not end the expression.
  387.  
  388. function tryCreateRegexp(src, flags, throwErrorAt, parser) {
  389.   try {
  390.     return new RegExp(src, flags)
  391.   } catch (e) {
  392.     if (throwErrorAt !== undefined) {
  393.       if (e instanceof SyntaxError) parser.raise(throwErrorAt, "Error parsing regular expression: " + e.message)
  394.       throw e
  395.     }
  396.   }
  397. }
  398.  
  399. var regexpUnicodeSupport = !!tryCreateRegexp("\uffff", "u")
  400.  
  401. pp.readRegexp = function() {
  402.   let escaped, inClass, start = this.pos
  403.   for (;;) {
  404.     if (this.pos >= this.input.length) this.raise(start, "Unterminated regular expression")
  405.     let ch = this.input.charAt(this.pos)
  406.     if (lineBreak.test(ch)) this.raise(start, "Unterminated regular expression")
  407.     if (!escaped) {
  408.       if (ch === "[") inClass = true
  409.       else if (ch === "]" && inClass) inClass = false
  410.       else if (ch === "/" && !inClass) break
  411.       escaped = ch === "\\"
  412.     } else escaped = false
  413.     ++this.pos
  414.   }
  415.   let content = this.input.slice(start, this.pos)
  416.   ++this.pos
  417.   // Need to use `readWord1` because '\uXXXX' sequences are allowed
  418.   // here (don't ask).
  419.   let mods = this.readWord1()
  420.   let tmp = content, tmpFlags = ""
  421.   if (mods) {
  422.     let validFlags = /^[gim]*$/
  423.     if (this.options.ecmaVersion >= 6) validFlags = /^[gimuy]*$/
  424.     if (!validFlags.test(mods)) this.raise(start, "Invalid regular expression flag")
  425.     if (mods.indexOf("u") >= 0) {
  426.       if (regexpUnicodeSupport) {
  427.         tmpFlags = "u"
  428.       } else {
  429.         // Replace each astral symbol and every Unicode escape sequence that
  430.         // possibly represents an astral symbol or a paired surrogate with a
  431.         // single ASCII symbol to avoid throwing on regular expressions that
  432.         // are only valid in combination with the `/u` flag.
  433.         // Note: replacing with the ASCII symbol `x` might cause false
  434.         // negatives in unlikely scenarios. For example, `[\u{61}-b]` is a
  435.         // perfectly valid pattern that is equivalent to `[a-b]`, but it would
  436.         // be replaced by `[x-b]` which throws an error.
  437.         tmp = tmp.replace(/\\u\{([0-9a-fA-F]+)\}/g, (_match, code, offset) => {
  438.           code = Number("0x" + code)
  439.           if (code > 0x10FFFF) this.raise(start + offset + 3, "Code point out of bounds")
  440.           return "x"
  441.         })
  442.         tmp = tmp.replace(/\\u([a-fA-F0-9]{4})|[\uD800-\uDBFF][\uDC00-\uDFFF]/g, "x")
  443.         tmpFlags = tmpFlags.replace("u", "")
  444.       }
  445.     }
  446.   }
  447.   // Detect invalid regular expressions.
  448.   let value = null
  449.   // Rhino's regular expression parser is flaky and throws uncatchable exceptions,
  450.   // so don't do detection if we are running under Rhino
  451.   if (!isRhino) {
  452.     tryCreateRegexp(tmp, tmpFlags, start, this)
  453.     // Get a regular expression object for this pattern-flag pair, or `null` in
  454.     // case the current environment doesn't support the flags it uses.
  455.     value = tryCreateRegexp(content, mods)
  456.   }
  457.   return this.finishToken(tt.regexp, {pattern: content, flags: mods, value: value})
  458. }
  459.  
  460. // Read an integer in the given radix. Return null if zero digits
  461. // were read, the integer value otherwise. When `len` is given, this
  462. // will return `null` unless the integer has exactly `len` digits.
  463.  
  464. pp.readInt = function(radix, len) {
  465.   let start = this.pos, total = 0
  466.   for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
  467.     let code = this.input.charCodeAt(this.pos), val
  468.     if (code >= 97) val = code - 97 + 10 // a
  469.     else if (code >= 65) val = code - 65 + 10 // A
  470.     else if (code >= 48 && code <= 57) val = code - 48 // 0-9
  471.     else val = Infinity
  472.     if (val >= radix) break
  473.     ++this.pos
  474.     total = total * radix + val
  475.   }
  476.   if (this.pos === start || len != null && this.pos - start !== len) return null
  477.  
  478.   return total
  479. }
  480.  
  481. pp.readRadixNumber = function(radix) {
  482.   this.pos += 2 // 0x
  483.   let val = this.readInt(radix)
  484.   if (val == null) this.raise(this.start + 2, "Expected number in radix " + radix)
  485.   if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
  486.   return this.finishToken(tt.num, val)
  487. }
  488.  
  489. // Read an integer, octal integer, or floating-point number.
  490.  
  491. pp.readNumber = function(startsWithDot) {
  492.   let start = this.pos, isFloat = false, octal = this.input.charCodeAt(this.pos) === 48
  493.   if (!startsWithDot && this.readInt(10) === null) this.raise(start, "Invalid number")
  494.   let next = this.input.charCodeAt(this.pos)
  495.   if (next === 46) { // '.'
  496.     ++this.pos
  497.     this.readInt(10)
  498.     isFloat = true
  499.     next = this.input.charCodeAt(this.pos)
  500.   }
  501.   if (next === 69 || next === 101) { // 'eE'
  502.     next = this.input.charCodeAt(++this.pos)
  503.     if (next === 43 || next === 45) ++this.pos // '+-'
  504.     if (this.readInt(10) === null) this.raise(start, "Invalid number")
  505.     isFloat = true
  506.   }
  507.   if (isIdentifierStart(this.fullCharCodeAtPos())) this.raise(this.pos, "Identifier directly after number")
  508.  
  509.   let str = this.input.slice(start, this.pos), val
  510.   if (isFloat) val = parseFloat(str)
  511.   else if (!octal || str.length === 1) val = parseInt(str, 10)
  512.   else if (/[89]/.test(str) || this.strict) this.raise(start, "Invalid number")
  513.   else val = parseInt(str, 8)
  514.   return this.finishToken(tt.num, val)
  515. }
  516.  
  517. // Read a string value, interpreting backslash-escapes.
  518.  
  519. pp.readCodePoint = function() {
  520.   let ch = this.input.charCodeAt(this.pos), code
  521.  
  522.   if (ch === 123) {
  523.     if (this.options.ecmaVersion < 6) this.unexpected()
  524.     let codePos = ++this.pos
  525.     code = this.readHexChar(this.input.indexOf('}', this.pos) - this.pos)
  526.     ++this.pos
  527.     if (code > 0x10FFFF) this.raise(codePos, "Code point out of bounds")
  528.   } else {
  529.     code = this.readHexChar(4)
  530.   }
  531.   return code
  532. }
  533.  
  534. function codePointToString(code) {
  535.   // UTF-16 Decoding
  536.   if (code <= 0xFFFF) return String.fromCharCode(code)
  537.   code -= 0x10000
  538.   return String.fromCharCode((code >> 10) + 0xD800, (code & 1023) + 0xDC00)
  539. }
  540.  
  541. pp.readString = function(quote) {
  542.   let out = "", chunkStart = ++this.pos
  543.   for (;;) {
  544.     if (this.pos >= this.input.length) this.raise(this.start, "Unterminated string constant")
  545.     let ch = this.input.charCodeAt(this.pos)
  546.     if (ch === quote) break
  547.     if (ch === 92) { // '\'
  548.       out += this.input.slice(chunkStart, this.pos)
  549.       out += this.readEscapedChar(false)
  550.       chunkStart = this.pos
  551.     } else {
  552.       if (isNewLine(ch)) this.raise(this.start, "Unterminated string constant")
  553.       ++this.pos
  554.     }
  555.   }
  556.   out += this.input.slice(chunkStart, this.pos++)
  557.   return this.finishToken(tt.string, out)
  558. }
  559.  
  560. // Reads template string tokens.
  561.  
  562. pp.readTmplToken = function() {
  563.   let out = "", chunkStart = this.pos
  564.   for (;;) {
  565.     if (this.pos >= this.input.length) this.raise(this.start, "Unterminated template")
  566.     let ch = this.input.charCodeAt(this.pos)
  567.     if (ch === 96 || ch === 36 && this.input.charCodeAt(this.pos + 1) === 123) { // '`', '${'
  568.       if (this.pos === this.start && this.type === tt.template) {
  569.         if (ch === 36) {
  570.           this.pos += 2
  571.           return this.finishToken(tt.dollarBraceL)
  572.         } else {
  573.           ++this.pos
  574.           return this.finishToken(tt.backQuote)
  575.         }
  576.       }
  577.       out += this.input.slice(chunkStart, this.pos)
  578.       return this.finishToken(tt.template, out)
  579.     }
  580.     if (ch === 92) { // '\'
  581.       out += this.input.slice(chunkStart, this.pos)
  582.       out += this.readEscapedChar(true)
  583.       chunkStart = this.pos
  584.     } else if (isNewLine(ch)) {
  585.       out += this.input.slice(chunkStart, this.pos)
  586.       ++this.pos
  587.       switch (ch) {
  588.         case 13:
  589.           if (this.input.charCodeAt(this.pos) === 10) ++this.pos
  590.         case 10:
  591.           out += "\n"
  592.           break
  593.         default:
  594.           out += String.fromCharCode(ch)
  595.           break
  596.       }
  597.       if (this.options.locations) {
  598.         ++this.curLine
  599.         this.lineStart = this.pos
  600.       }
  601.       chunkStart = this.pos
  602.     } else {
  603.       ++this.pos
  604.     }
  605.   }
  606. }
  607.  
  608. // Used to read escaped characters
  609.  
  610. pp.readEscapedChar = function(inTemplate) {
  611.   let ch = this.input.charCodeAt(++this.pos)
  612.   ++this.pos
  613.   switch (ch) {
  614.   case 110: return "\n" // 'n' -> '\n'
  615.   case 114: return "\r" // 'r' -> '\r'
  616.   case 120: return String.fromCharCode(this.readHexChar(2)) // 'x'
  617.   case 117: return codePointToString(this.readCodePoint()) // 'u'
  618.   case 116: return "\t" // 't' -> '\t'
  619.   case 98: return "\b" // 'b' -> '\b'
  620.   case 118: return "\u000b" // 'v' -> '\u000b'
  621.   case 102: return "\f" // 'f' -> '\f'
  622.   case 13: if (this.input.charCodeAt(this.pos) === 10) ++this.pos // '\r\n'
  623.   case 10: // ' \n'
  624.     if (this.options.locations) { this.lineStart = this.pos; ++this.curLine }
  625.     return ""
  626.   default:
  627.     if (ch >= 48 && ch <= 55) {
  628.       let octalStr = this.input.substr(this.pos - 1, 3).match(/^[0-7]+/)[0]
  629.       let octal = parseInt(octalStr, 8)
  630.       if (octal > 255) {
  631.         octalStr = octalStr.slice(0, -1)
  632.         octal = parseInt(octalStr, 8)
  633.       }
  634.       if (octalStr !== "0" && (this.strict || inTemplate)) {
  635.         this.raise(this.pos - 2, "Octal literal in strict mode")
  636.       }
  637.       this.pos += octalStr.length - 1
  638.       return String.fromCharCode(octal)
  639.     }
  640.     return String.fromCharCode(ch)
  641.   }
  642. }
  643.  
  644. // Used to read character escape sequences ('\x', '\u', '\U').
  645.  
  646. pp.readHexChar = function(len) {
  647.   let codePos = this.pos
  648.   let n = this.readInt(16, len)
  649.   if (n === null) this.raise(codePos, "Bad character escape sequence")
  650.   return n
  651. }
  652.  
  653. // Read an identifier, and return it as a string. Sets `this.containsEsc`
  654. // to whether the word contained a '\u' escape.
  655. //
  656. // Incrementally adds only escaped chars, adding other chunks as-is
  657. // as a micro-optimization.
  658.  
  659. pp.readWord1 = function() {
  660.   this.containsEsc = false
  661.   let word = "", first = true, chunkStart = this.pos
  662.   let astral = this.options.ecmaVersion >= 6
  663.   while (this.pos < this.input.length) {
  664.     let ch = this.fullCharCodeAtPos()
  665.     if (isIdentifierChar(ch, astral)) {
  666.       this.pos += ch <= 0xffff ? 1 : 2
  667.     } else if (ch === 92) { // "\"
  668.       this.containsEsc = true
  669.       word += this.input.slice(chunkStart, this.pos)
  670.       let escStart = this.pos
  671.       if (this.input.charCodeAt(++this.pos) != 117) // "u"
  672.         this.raise(this.pos, "Expecting Unicode escape sequence \\uXXXX")
  673.       ++this.pos
  674.       let esc = this.readCodePoint()
  675.       if (!(first ? isIdentifierStart : isIdentifierChar)(esc, astral))
  676.         this.raise(escStart, "Invalid Unicode escape")
  677.       word += codePointToString(esc)
  678.       chunkStart = this.pos
  679.     } else {
  680.       break
  681.     }
  682.     first = false
  683.   }
  684.   return word + this.input.slice(chunkStart, this.pos)
  685. }
  686.  
  687. // Read an identifier or keyword token. Will check for reserved
  688. // words when necessary.
  689.  
  690. pp.readWord = function() {
  691.   let word = this.readWord1()
  692.   let type = tt.name
  693.   if ((this.options.ecmaVersion >= 6 || !this.containsEsc) && this.keywords.test(word))
  694.     type = keywordTypes[word]
  695.   return this.finishToken(type, word)
  696. }
  697.  
downloadtokenize.js Source code - Download reactide Source code
Related Source Codes/Software:
rkt - rkt is a pod-native container engine for Linux. It... 2017-06-11
uWebSockets - Tiny WebSockets https://for... 2017-06-11
realworld - TodoMVC for the RealWorld - Exemplary fullstack Me... 2017-06-11
goreplay - GoReplay is an open-source tool for capturing and ... 2017-06-10
pyenv - Simple Python version management 2017-06-10
postal - 2017-06-11
CRYENGINE - CRYENGINE is a powerful real-time game development... 2017-06-11
reactide - Reactide is the first dedicated IDE for React web ... 2017-06-11
redux-saga - An alternative side effect model for Redux apps ... 2017-06-10
angular-starter - 2017-06-10

 Back to top