diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 860762f688..2b6cc06db3 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -110,18 +110,6 @@ export class Token { // ## Tokenizer -function codePointToString(code: number): string { - // UTF-16 Decoding - if (code <= 0xffff) { - return String.fromCharCode(code); - } else { - return String.fromCharCode( - ((code - 0x10000) >> 10) + 0xd800, - ((code - 0x10000) & 1023) + 0xdc00, - ); - } -} - export default class Tokenizer extends LocationParser { // Forward-declarations // parser/util.js @@ -226,7 +214,7 @@ export default class Tokenizer extends LocationParser { if (curContext.override) { curContext.override(this); } else { - this.readToken(this.fullCharCodeAtPos()); + this.readToken(this.input.codePointAt(this.state.pos)); } } @@ -240,14 +228,6 @@ export default class Tokenizer extends LocationParser { } } - fullCharCodeAtPos(): number { - const code = this.input.charCodeAt(this.state.pos); - if (code <= 0xd7ff || code >= 0xe000) return code; - - const next = this.input.charCodeAt(this.state.pos + 1); - return (code << 10) + next - 0x35fdc00; - } - pushComment( block: boolean, text: string, @@ -658,7 +638,7 @@ export default class Tokenizer extends LocationParser { } else { this.raise( this.state.pos, - `Unexpected character '${codePointToString(code)}'`, + `Unexpected character '${String.fromCodePoint(code)}'`, ); } @@ -820,7 +800,7 @@ export default class Tokenizer extends LocationParser { this.raise( this.state.pos, - `Unexpected character '${codePointToString(code)}'`, + `Unexpected character '${String.fromCodePoint(code)}'`, ); } @@ -862,7 +842,7 @@ export default class Tokenizer extends LocationParser { while (this.state.pos < this.input.length) { const char = this.input[this.state.pos]; - const charCode = this.fullCharCodeAtPos(); + const charCode = this.input.codePointAt(this.state.pos); if (VALID_REGEX_FLAGS.indexOf(char) > -1) { if (mods.indexOf(char) > -1) { @@ -974,7 +954,7 @@ export default class Tokenizer extends LocationParser { } } - if (isIdentifierStart(this.fullCharCodeAtPos())) { + if (isIdentifierStart(this.input.codePointAt(this.state.pos))) { this.raise(this.state.pos, "Identifier directly after number"); } @@ -1030,7 +1010,7 @@ export default class Tokenizer extends LocationParser { } } - if (isIdentifierStart(this.fullCharCodeAtPos())) { + if (isIdentifierStart(this.input.codePointAt(this.state.pos))) { this.raise(this.state.pos, "Identifier directly after number"); } @@ -1199,7 +1179,7 @@ export default class Tokenizer extends LocationParser { } case charCodes.lowercaseU: { const code = this.readCodePoint(throwOnInvalid); - return code === null ? null : codePointToString(code); + return code === null ? null : String.fromCodePoint(code); } case charCodes.lowercaseT: return "\t"; @@ -1277,7 +1257,7 @@ export default class Tokenizer extends LocationParser { first = true, chunkStart = this.state.pos; while (this.state.pos < this.input.length) { - const ch = this.fullCharCodeAtPos(); + const ch = this.input.codePointAt(this.state.pos); if (isIdentifierChar(ch)) { this.state.pos += ch <= 0xffff ? 1 : 2; } else if (this.state.isIterator && ch === charCodes.atSign) { @@ -1303,7 +1283,7 @@ export default class Tokenizer extends LocationParser { } // $FlowFixMe - word += codePointToString(esc); + word += String.fromCodePoint(esc); chunkStart = this.state.pos; } else { break;