From 6dee98d1b9709419bcd57fc24a24c4e61b2e202b Mon Sep 17 00:00:00 2001 From: Ingvar Stepanyan Date: Thu, 15 Jan 2015 12:21:28 +0200 Subject: [PATCH] Adapt ES6 template handling to new tokenizer. Avoid need for: * extra `templates` array in favor of new `tokContext`; * special location handling for first & last template elements; * separate `_templateContinued` token in favor of same `_template`. Adds: * token types for backQuote and dollarBraceL instead of skipping them so they can be handled (i.e. highlighted differently). --- acorn.js | 121 ++++++++++++++++++++++-------------------- acorn_loose.js | 39 +++++++------- test/tests-harmony.js | 2 +- 3 files changed, 84 insertions(+), 78 deletions(-) diff --git a/acorn.js b/acorn.js index bdb026566e..4945981d59 100644 --- a/acorn.js +++ b/acorn.js @@ -320,13 +320,6 @@ var metParenL; - // This is used by the tokenizer to track the template strings it is - // inside, and count the amount of open braces seen inside them, to - // be able to switch back to a template token when the } to match ${ - // is encountered. It will hold an array of integers. - - var templates; - function initParserState() { lastStart = lastEnd = tokPos; if (options.locations) lastEndLoc = curPosition(); @@ -428,8 +421,9 @@ var _braceR = {type: "}"}, _parenL = {type: "(", beforeExpr: true}, _parenR = {type: ")"}; var _comma = {type: ",", beforeExpr: true}, _semi = {type: ";", beforeExpr: true}; var _colon = {type: ":", beforeExpr: true}, _dot = {type: "."}, _question = {type: "?", beforeExpr: true}; - var _arrow = {type: "=>", beforeExpr: true}, _template = {type: "template"}, _templateContinued = {type: "templateContinued"}; + var _arrow = {type: "=>", beforeExpr: true}, _template = {type: "template"}; var _ellipsis = {type: "...", prefix: true, beforeExpr: true}; + var _backQuote = {type: "`"}, _dollarBraceL = {type: "${", beforeExpr: true}; // Operators. These carry several kinds of properties to help the // parser use them properly (the presence of these properties is @@ -471,8 +465,8 @@ parenL: _parenL, parenR: _parenR, comma: _comma, semi: _semi, colon: _colon, dot: _dot, ellipsis: _ellipsis, question: _question, slash: _slash, eq: _eq, name: _name, eof: _eof, num: _num, regexp: _regexp, string: _string, - arrow: _arrow, template: _template, templateContinued: _templateContinued, star: _star, - assign: _assign}; + arrow: _arrow, template: _template, star: _star, assign: _assign, + backQuote: _backQuote, dollarBraceL: _dollarBraceL}; for (var kw in keywordTypes) exports.tokTypes["_" + kw] = keywordTypes[kw]; // This is a trick taken from Esprima. It turns out that, on @@ -631,7 +625,6 @@ tokContext = []; tokExprAllowed = true; metParenL = 0; - templates = []; if (tokPos === 0 && options.allowHashBang && input.slice(0, 2) === '#!') { skipLineComment(2); } @@ -641,8 +634,9 @@ // given point in the program is loosely based on sweet.js' approach. // See https://github.com/mozilla/sweet.js/wiki/design - var b_stat = {token: "{", isExpr: false}, b_expr = {token: "{", isExpr: true}; + var b_stat = {token: "{", isExpr: false}, b_expr = {token: "{", isExpr: true}, b_tmpl = {token: "${", isExpr: true}; var p_stat = {token: "(", isExpr: false}, p_expr = {token: "(", isExpr: true}; + var q_tmpl = {token: "`", isExpr: true}; function braceIsBlock(prevType) { var parent; @@ -665,18 +659,21 @@ function finishToken(type, val) { tokEnd = tokPos; if (options.locations) tokEndLoc = curPosition(); - var prevType = tokType; + var prevType = tokType, preserveSpace = false; tokType = type; - skipSpace(); tokVal = val; // Update context info if (type === _parenR || type === _braceR) { var out = tokContext.pop(); tokExprAllowed = !(out && out.isExpr); + preserveSpace = out === b_tmpl; } else if (type === _braceL) { tokContext.push(braceIsBlock(prevType) ? b_stat : b_expr); tokExprAllowed = true; + } else if (type === _dollarBraceL) { + tokContext.push(b_tmpl); + tokExprAllowed = true; } else if (type == _parenL) { var statementParens = prevType === _if || prevType === _for || prevType === _with || prevType === _while; tokContext.push(statementParens ? p_stat : p_expr); @@ -687,9 +684,19 @@ tokExprAllowed = false; } else if (tokExprAllowed && type == _function) { tokExprAllowed = false; + } else if (type === _backQuote) { + if (tokContext[tokContext.length - 1] === q_tmpl) { + tokContext.pop(); + } else { + tokContext.push(q_tmpl); + preserveSpace = true; + } + tokExprAllowed = false; } else { tokExprAllowed = type.beforeExpr; } + + if (!preserveSpace) skipSpace(); } function skipBlockComment() { @@ -874,23 +881,17 @@ case 44: ++tokPos; return finishToken(_comma); case 91: ++tokPos; return finishToken(_bracketL); case 93: ++tokPos; return finishToken(_bracketR); - case 123: - ++tokPos; - if (templates.length) ++templates[templates.length - 1]; - return finishToken(_braceL); - case 125: - ++tokPos; - if (templates.length && --templates[templates.length - 1] === 0) - return readTemplateString(_templateContinued); - else - return finishToken(_braceR); + case 123: ++tokPos; return finishToken(_braceL); + case 125: ++tokPos; return finishToken(_braceR); case 58: ++tokPos; return finishToken(_colon); case 63: ++tokPos; return finishToken(_question); case 96: // '`' if (options.ecmaVersion >= 6) { ++tokPos; - return readTemplateString(_template); + return finishToken(_backQuote); + } else { + return false; } case 48: // '0' @@ -947,6 +948,10 @@ if (options.locations) tokStartLoc = curPosition(); if (tokPos >= inputLen) return finishToken(_eof); + if (tokContext[tokContext.length - 1] === q_tmpl) { + return readTmplToken(); + } + var code = input.charCodeAt(tokPos); // Identifier or keyword. '\uXXXX' sequences are allowed in @@ -1131,34 +1136,40 @@ } } - function readTemplateString(type) { - if (type == _templateContinued) templates.pop(); - var out = "", start = tokPos;; + // Reads template string tokens. + + function readTmplToken() { + var out = "", start = tokPos; for (;;) { if (tokPos >= inputLen) raise(tokStart, "Unterminated template"); - var ch = input.charAt(tokPos); - if (ch === "`" || ch === "$" && input.charCodeAt(tokPos + 1) === 123) { // '`', '${' - var raw = input.slice(start, tokPos); - ++tokPos; - if (ch == "$") { ++tokPos; templates.push(1); } - return finishToken(type, {cooked: out, raw: raw}); + var ch = input.charCodeAt(tokPos); + if (ch === 96 || ch === 36 && input.charCodeAt(tokPos + 1) === 123) { // '`', '${' + if (tokPos === start && tokType === _template) { + if (ch === 36) { + tokPos += 2; + return finishToken(_dollarBraceL); + } else { + ++tokPos; + return finishToken(_backQuote); + } + } + return finishToken(_template, out); } - - if (ch === "\\") { // '\' + if (ch === 92) { // '\' out += readEscapedChar(); } else { ++tokPos; - if (newline.test(ch)) { - if (ch === "\r" && input.charCodeAt(tokPos) === 10) { + if (isNewLine(ch)) { + if (ch === 13 && input.charCodeAt(tokPos) === 10) { ++tokPos; - ch = "\n"; + ch = 10; } if (options.locations) { ++tokCurLine; tokLineStart = tokPos; } } - out += ch; + out += String.fromCharCode(ch); } } } @@ -1369,15 +1380,6 @@ return node; } - function finishNodeAt(node, type, pos) { - if (options.locations) { node.loc.end = pos[1]; pos = pos[0]; } - node.type = type; - node.end = pos; - if (options.ranges) - node.range[1] = pos; - return node; - } - // Test whether a statement node is the string literal `"use strict"`. function isUseStrict(stmt) { @@ -2137,7 +2139,7 @@ node.callee = base; node.arguments = parseExprList(_parenR, false); return parseSubscripts(finishNode(node, "CallExpression"), start, noCalls); - } else if (tokType === _template) { + } else if (tokType === _backQuote) { var node = startNodeAt(start); node.tag = base; node.quasi = parseTemplate(); @@ -2252,7 +2254,7 @@ case _new: return parseNew(); - case _template: + case _backQuote: return parseTemplate(); default: @@ -2277,24 +2279,29 @@ // Parse template expression. function parseTemplateElement() { - var elem = startNodeAt(options.locations ? [tokStart + 1, tokStartLoc.offset(1)] : tokStart + 1); - elem.value = tokVal; - elem.tail = input.charCodeAt(tokEnd - 1) !== 123; // '{' + var elem = startNode(); + elem.value = { + raw: input.slice(tokStart, tokEnd), + cooked: tokVal + }; next(); - var endOff = elem.tail ? 1 : 2; - return finishNodeAt(elem, "TemplateElement", options.locations ? [lastEnd - endOff, lastEndLoc.offset(-endOff)] : lastEnd - endOff); + elem.tail = tokType === _backQuote; + return finishNode(elem, "TemplateElement"); } function parseTemplate() { var node = startNode(); + next(); node.expressions = []; var curElt = parseTemplateElement(); node.quasis = [curElt]; while (!curElt.tail) { + expect(_dollarBraceL); node.expressions.push(parseExpression()); - if (tokType !== _templateContinued) unexpected(); + expect(_braceR); node.quasis.push(curElt = parseTemplateElement()); } + next(); return finishNode(node, "TemplateLiteral"); } diff --git a/acorn_loose.js b/acorn_loose.js index 62626fe97f..b1b461360d 100644 --- a/acorn_loose.js +++ b/acorn_loose.js @@ -103,8 +103,8 @@ replace = {start: e.pos, end: pos, type: tt.regexp, value: re}; } else if (/template/.test(msg)) { replace = {start: e.pos, end: pos, - type: input.charAt(e.pos) == "`" ? tt.template : tt.templateContinued, - value: input.slice(e.pos + 1, pos)}; + type: tt.template, + value: input.slice(e.pos, pos)}; } else if (/comment/.test(msg)) { replace = fetchToken.current(); } else { @@ -697,7 +697,7 @@ node.callee = base; node.arguments = parseExprList(tt.parenR); base = finishNode(node, "CallExpression"); - } else if (token.type == tt.template) { + } else if (token.type == tt.backQuote) { var node = startNodeAt(start); node.tag = base; node.quasi = parseTemplate(); @@ -790,7 +790,7 @@ } return finishNode(node, "YieldExpression"); - case tt.template: + case tt.backQuote: return parseTemplate(); default: @@ -813,36 +813,35 @@ } function parseTemplateElement() { - var elem = startNodeAt(options.locations ? [token.start + 1, token.startLoc.offset(1)] : token.start + 1); - elem.value = token.value; - elem.tail = input.charCodeAt(token.end - 1) !== 123; // '{' - var endOff = elem.tail ? 1 : 2; - var endPos = options.locations ? [token.end - endOff, token.endLoc.offset(-endOff)] : token.end - endOff; + var elem = startNode(); + elem.value = { + raw: input.slice(token.start, token.end), + cooked: token.value + }; next(); - return finishNodeAt(elem, "TemplateElement", endPos); + elem.tail = token.type === tt.backQuote; + return finishNode(elem, "TemplateElement"); } function parseTemplate() { var node = startNode(); + next(); node.expressions = []; var curElt = parseTemplateElement(); node.quasis = [curElt]; while (!curElt.tail) { - var next = parseExpression(); - if (isDummy(next)) { - node.quasis[node.quasis.length - 1].tail = true; - break; - } - node.expressions.push(next); - if (token.type === tt.templateContinued) { - node.quasis.push(curElt = parseTemplateElement()); + next(); + node.expressions.push(parseExpression()); + if (expect(tt.braceR)) { + curElt = parseTemplateElement(); } else { curElt = startNode(); - curElt.value = {cooked: "", raw: ""}; + curElt.value = {cooked: '', raw: ''}; curElt.tail = true; - node.quasis.push(curElt); } + node.quasis.push(curElt); } + expect(tt.backQuote); return finishNode(node, "TemplateLiteral"); } diff --git a/test/tests-harmony.js b/test/tests-harmony.js index df983c7ccd..7f45357f78 100644 --- a/test/tests-harmony.js +++ b/test/tests-harmony.js @@ -14077,7 +14077,7 @@ testFail("class A extends yield B { }", "Unexpected token (1:22)", {ecmaVersion: testFail("class default", "Unexpected token (1:6)", {ecmaVersion: 6}); -testFail("`test", "Unterminated template (1:0)", {ecmaVersion: 6}); +testFail("`test", "Unterminated template (1:1)", {ecmaVersion: 6}); testFail("switch `test`", "Unexpected token (1:7)", {ecmaVersion: 6});