From d8a53298345b0f2814f2964280890cd6b4a0752f Mon Sep 17 00:00:00 2001 From: Daniel Tschinder Date: Wed, 6 Mar 2019 14:30:04 -0800 Subject: [PATCH] Reorganize token types and use a map for them (#9645) --- .../babel-parser/src/plugins/jsx/index.js | 2 + packages/babel-parser/src/tokenizer/index.js | 2 +- packages/babel-parser/src/tokenizer/types.js | 128 ++++++++---------- packages/babel-parser/src/util/identifier.js | 39 +----- 4 files changed, 63 insertions(+), 108 deletions(-) diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js index a129c6d27a..378bba9286 100644 --- a/packages/babel-parser/src/plugins/jsx/index.js +++ b/packages/babel-parser/src/plugins/jsx/index.js @@ -14,6 +14,8 @@ import { isNewLine } from "../../util/whitespace"; const HEX_NUMBER = /^[\da-fA-F]+$/; const DECIMAL_NUMBER = /^\d+$/; +// Be aware that this file is always executed and not only when the plugin is enabled. +// Therefore this contexts and tokens do always exist. tc.j_oTag = new TokContext("...", true, true); diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index f576698130..5d86682bcd 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1329,7 +1329,7 @@ export default class Tokenizer extends LocationParser { readWord(): void { const word = this.readWord1(); - const type = keywordTypes[word] || tt.name; + const type = keywordTypes.get(word) || tt.name; if (type.keyword && this.state.containsEsc) { this.raise(this.state.pos, `Escape sequence in keyword ${word}`); diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index ea9f15c708..2301759f38 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -32,7 +32,6 @@ const postfix = true; type TokenOptions = { keyword?: string, - beforeExpr?: boolean, startsExpr?: boolean, rightAssociative?: boolean, @@ -66,16 +65,21 @@ export class TokenType { this.isAssign = !!conf.isAssign; this.prefix = !!conf.prefix; this.postfix = !!conf.postfix; - this.binop = conf.binop === 0 ? 0 : conf.binop || null; + this.binop = conf.binop != null ? conf.binop : null; this.updateContext = null; } } -function KeywordTokenType(keyword: string, options: TokenOptions = {}) { - return new TokenType(keyword, { ...options, keyword }); +export const keywords = new Map(); + +function createKeyword(name: string, options: TokenOptions = {}): TokenType { + options.keyword = name; + const token = new TokenType(name, options); + keywords.set(name, token); + return token; } -function BinopTokenType(name: string, binop: number) { +function createBinop(name: string, binop: number) { return new TokenType(name, { beforeExpr, binop }); } @@ -133,74 +137,60 @@ export const types: { [name: string]: TokenType } = { incDec: new TokenType("++/--", { prefix, postfix, startsExpr }), bang: new TokenType("!", { beforeExpr, prefix, startsExpr }), tilde: new TokenType("~", { beforeExpr, prefix, startsExpr }), - pipeline: BinopTokenType("|>", 0), - nullishCoalescing: BinopTokenType("??", 1), - logicalOR: BinopTokenType("||", 1), - logicalAND: BinopTokenType("&&", 2), - bitwiseOR: BinopTokenType("|", 3), - bitwiseXOR: BinopTokenType("^", 4), - bitwiseAND: BinopTokenType("&", 5), - equality: BinopTokenType("==/!=", 6), - relational: BinopTokenType("", 7), - bitShift: BinopTokenType("<>", 8), + pipeline: createBinop("|>", 0), + nullishCoalescing: createBinop("??", 1), + logicalOR: createBinop("||", 1), + logicalAND: createBinop("&&", 2), + bitwiseOR: createBinop("|", 3), + bitwiseXOR: createBinop("^", 4), + bitwiseAND: createBinop("&", 5), + equality: createBinop("==/!=", 6), + relational: createBinop("", 7), + bitShift: createBinop("<>", 8), plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }), - modulo: BinopTokenType("%", 10), - star: BinopTokenType("*", 10), - slash: BinopTokenType("/", 10), + modulo: createBinop("%", 10), + star: createBinop("*", 10), + slash: createBinop("/", 10), exponent: new TokenType("**", { beforeExpr, binop: 11, rightAssociative: true, }), + + // Keywords + _break: createKeyword("break"), + _case: createKeyword("case", { beforeExpr }), + _catch: createKeyword("catch"), + _continue: createKeyword("continue"), + _debugger: createKeyword("debugger"), + _default: createKeyword("default", { beforeExpr }), + _do: createKeyword("do", { isLoop, beforeExpr }), + _else: createKeyword("else", { beforeExpr }), + _finally: createKeyword("finally"), + _for: createKeyword("for", { isLoop }), + _function: createKeyword("function", { startsExpr }), + _if: createKeyword("if"), + _return: createKeyword("return", { beforeExpr }), + _switch: createKeyword("switch"), + _throw: createKeyword("throw", { beforeExpr, prefix, startsExpr }), + _try: createKeyword("try"), + _var: createKeyword("var"), + _const: createKeyword("const"), + _while: createKeyword("while", { isLoop }), + _with: createKeyword("with"), + _new: createKeyword("new", { beforeExpr, startsExpr }), + _this: createKeyword("this", { startsExpr }), + _super: createKeyword("super", { startsExpr }), + _class: createKeyword("class", { startsExpr }), + _extends: createKeyword("extends", { beforeExpr }), + _export: createKeyword("export"), + _import: createKeyword("import", { startsExpr }), + _null: createKeyword("null", { startsExpr }), + _true: createKeyword("true", { startsExpr }), + _false: createKeyword("false", { startsExpr }), + _in: createKeyword("in", { beforeExpr, binop: 7 }), + _instanceof: createKeyword("instanceof", { beforeExpr, binop: 7 }), + _typeof: createKeyword("typeof", { beforeExpr, prefix, startsExpr }), + _void: createKeyword("void", { beforeExpr, prefix, startsExpr }), + _delete: createKeyword("delete", { beforeExpr, prefix, startsExpr }), }; - -function makeKeywordProps( - name: string, - conf: any, -): PropertyDescriptor { - return { value: KeywordTokenType(name, conf), enumerable: true }; -} - -// $FlowIssue -export const keywords = Object.create(null, { - break: makeKeywordProps("break"), - case: makeKeywordProps("case", { beforeExpr }), - catch: makeKeywordProps("catch"), - continue: makeKeywordProps("continue"), - debugger: makeKeywordProps("debugger"), - default: makeKeywordProps("default", { beforeExpr }), - do: makeKeywordProps("do", { isLoop, beforeExpr }), - else: makeKeywordProps("else", { beforeExpr }), - finally: makeKeywordProps("finally"), - for: makeKeywordProps("for", { isLoop }), - function: makeKeywordProps("function", { startsExpr }), - if: makeKeywordProps("if"), - return: makeKeywordProps("return", { beforeExpr }), - switch: makeKeywordProps("switch"), - throw: makeKeywordProps("throw", { beforeExpr, prefix, startsExpr }), - try: makeKeywordProps("try"), - var: makeKeywordProps("var"), - const: makeKeywordProps("const"), - while: makeKeywordProps("while", { isLoop }), - with: makeKeywordProps("with"), - new: makeKeywordProps("new", { beforeExpr, startsExpr }), - this: makeKeywordProps("this", { startsExpr }), - super: makeKeywordProps("super", { startsExpr }), - class: makeKeywordProps("class", { startsExpr }), - extends: makeKeywordProps("extends", { beforeExpr }), - export: makeKeywordProps("export"), - import: makeKeywordProps("import", { startsExpr }), - null: makeKeywordProps("null", { startsExpr }), - true: makeKeywordProps("true", { startsExpr }), - false: makeKeywordProps("false", { startsExpr }), - in: makeKeywordProps("in", { beforeExpr, binop: 7 }), - instanceof: makeKeywordProps("instanceof", { beforeExpr, binop: 7 }), - typeof: makeKeywordProps("typeof", { beforeExpr, prefix, startsExpr }), - void: makeKeywordProps("void", { beforeExpr, prefix, startsExpr }), - delete: makeKeywordProps("delete", { beforeExpr, prefix, startsExpr }), -}); - -// Map keyword names to token types. -Object.keys(keywords).forEach(name => { - types["_" + name] = keywords[name]; -}); diff --git a/packages/babel-parser/src/util/identifier.js b/packages/babel-parser/src/util/identifier.js index bfd2ca748f..7fda0e27de 100644 --- a/packages/babel-parser/src/util/identifier.js +++ b/packages/babel-parser/src/util/identifier.js @@ -3,6 +3,7 @@ // @flow import * as charCodes from "charcodes"; +import { keywords } from "../tokenizer/types"; const reservedWords = { strict: [ @@ -52,44 +53,6 @@ export function isStrictBindReservedWord( return isReservedWord(word, inModule) || reservedWordsStrictBindSet.has(word); } -const keywords = new Set([ - "break", - "case", - "catch", - "continue", - "debugger", - "default", - "do", - "else", - "finally", - "for", - "function", - "if", - "return", - "switch", - "throw", - "try", - "var", - "while", - "with", - "null", - "true", - "false", - "instanceof", - "typeof", - "void", - "delete", - "new", - "in", - "this", - "const", - "class", - "extends", - "export", - "import", - "super", -]); - export function isKeyword(word: string): boolean { return keywords.has(word); }