Add decimal parsing support (#11640)

* docs: add DecimalLiteral to AST spec

* add decimal support

* fix: throw invalid decimal on start

* add DecimalLiteral type definitions

* update parser typings

* add generator support

* add syntax-decimal plugin

* Add syntax-decimal to babel-standalone

* add syntax-decimal to missing plugin helpers

* fix incorrect test macro

packages/babel-parser/ast/spec.md

@@ -11,6 +11,7 @@ These are the core @babel/parser (babylon) AST node types.
   - [BooleanLiteral](#booleanliteral)
   - [NumericLiteral](#numericliteral)
   - [BigIntLiteral](#bigintliteral)
+  - [DecimalLiteral](#decimalliteral)
 - [Programs](#programs)
 - [Functions](#functions)
 - [Statements](#statements)
@@ -253,6 +254,17 @@ interface BigIntLiteral <: Literal {
 
 The `value` property is the string representation of the `BigInt` value. It doesn't include the suffix `n`.
 
+## DecimalLiteral
+
+```js
+interface DecimalLiteral <: Literal {
+  type: "DecimalLiteral";
+  value: string;
+}
+```
+
+The `value` property is the string representation of the `BigDecimal` value. It doesn't include the suffix `m`.
+
 # Programs
 
 ```js

packages/babel-parser/src/parser/error-message.js

@@ -61,6 +61,7 @@ export const ErrorMessages = Object.freeze({
   ImportOutsideModule: `'import' and 'export' may appear only with 'sourceType: "module"'`,
   InvalidBigIntLiteral: "Invalid BigIntLiteral",
   InvalidCodePoint: "Code point out of bounds",
+  InvalidDecimal: "Invalid decimal",
   InvalidDigit: "Expected number in radix %0",
   InvalidEscapeSequence: "Bad character escape sequence",
   InvalidEscapeSequenceTemplate: "Invalid escape sequence in template",

packages/babel-parser/src/parser/expression.js

@@ -1032,6 +1032,9 @@ export default class ExpressionParser extends LValParser {
       case tt.bigint:
         return this.parseLiteral(this.state.value, "BigIntLiteral");
 
+      case tt.decimal:
+        return this.parseLiteral(this.state.value, "DecimalLiteral");
+
       case tt.string:
         return this.parseLiteral(this.state.value, "StringLiteral");
 
@@ -1859,7 +1862,10 @@ export default class ExpressionParser extends LValParser {
       this.state.inPropertyName = true;
       // We check if it's valid for it to be a private name when we push it.
       (prop: $FlowFixMe).key =
-        this.match(tt.num) || this.match(tt.string) || this.match(tt.bigint)
+        this.match(tt.num) ||
+        this.match(tt.string) ||
+        this.match(tt.bigint) ||
+        this.match(tt.decimal)
           ? this.parseExprAtom()
           : this.parseMaybePrivateName(isPrivateNameAllowed);
 

packages/babel-parser/src/parser/util.js

@@ -273,7 +273,8 @@ export default class UtilParser extends Tokenizer {
       !!this.state.type.keyword ||
       this.match(tt.string) ||
       this.match(tt.num) ||
-      this.match(tt.bigint)
+      this.match(tt.bigint) ||
+      this.match(tt.decimal)
     );
   }
 }

packages/babel-parser/src/plugins/estree.js

@@ -43,6 +43,17 @@ export default (superClass: Class<Parser>): Class<Parser> =>
       return node;
     }
 
+    estreeParseDecimalLiteral(value: any): N.Node {
+      // https://github.com/estree/estree/blob/master/experimental/decimal.md
+      // $FlowIgnore
+      // todo: use BigDecimal when node supports it.
+      const decimal = null;
+      const node = this.estreeParseLiteral(decimal);
+      node.decimal = String(node.value || value);
+
+      return node;
+    }
+
     estreeParseLiteral(value: any): N.Node {
       return this.parseLiteral(value, "Literal");
     }
@@ -229,6 +240,9 @@ export default (superClass: Class<Parser>): Class<Parser> =>
         case tt.bigint:
           return this.estreeParseBigIntLiteral(this.state.value);
 
+        case tt.decimal:
+          return this.estreeParseDecimalLiteral(this.state.value);
+
         case tt._null:
           return this.estreeParseLiteral(null);
 

packages/babel-parser/src/tokenizer/index.js

@@ -1095,6 +1095,8 @@ export default class Tokenizer extends ParserErrors {
     if (next === charCodes.lowercaseN) {
       ++this.state.pos;
       isBigInt = true;
+    } else if (next === charCodes.lowercaseM) {
+      throw this.raise(start, Errors.InvalidDecimal);
     }
 
     if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
@@ -1116,6 +1118,8 @@ export default class Tokenizer extends ParserErrors {
     const start = this.state.pos;
     let isFloat = false;
     let isBigInt = false;
+    let isDecimal = false;
+    let hasExponent = false;
     let isOctal = false;
 
     if (!startsWithDot && this.readInt(10) === null) {
@@ -1157,6 +1161,7 @@ export default class Tokenizer extends ParserErrors {
       }
       if (this.readInt(10) === null) this.raise(start, Errors.InvalidNumber);
       isFloat = true;
+      hasExponent = true;
       next = this.input.charCodeAt(this.state.pos);
     }
 
@@ -1174,18 +1179,32 @@ export default class Tokenizer extends ParserErrors {
       isBigInt = true;
     }
 
+    if (next === charCodes.lowercaseM) {
+      this.expectPlugin("decimal", this.state.pos);
+      if (hasExponent || hasLeadingZero) {
+        this.raise(start, Errors.InvalidDecimal);
+      }
+      ++this.state.pos;
+      isDecimal = true;
+    }
+
     if (isIdentifierStart(this.input.codePointAt(this.state.pos))) {
       throw this.raise(this.state.pos, Errors.NumberIdentifier);
     }
 
-    // remove "_" for numeric literal separator, and "n" for BigInts
-    const str = this.input.slice(start, this.state.pos).replace(/[_n]/g, "");
+    // remove "_" for numeric literal separator, and trailing `m` or `n`
+    const str = this.input.slice(start, this.state.pos).replace(/[_mn]/g, "");
 
     if (isBigInt) {
       this.finishToken(tt.bigint, str);
       return;
     }
 
+    if (isDecimal) {
+      this.finishToken(tt.decimal, str);
+      return;
+    }
+
     const val = isOctal ? parseInt(str, 8) : parseFloat(str);
     this.finishToken(tt.num, val);
   }

packages/babel-parser/src/tokenizer/types.js

@@ -86,6 +86,7 @@ function createBinop(name: string, binop: number) {
 export const types: { [name: string]: TokenType } = {
   num: new TokenType("num", { startsExpr }),
   bigint: new TokenType("bigint", { startsExpr }),
+  decimal: new TokenType("decimal", { startsExpr }),
   regexp: new TokenType("regexp", { startsExpr }),
   string: new TokenType("string", { startsExpr }),
   name: new TokenType("name", { startsExpr }),

packages/babel-parser/test/fixtures/experimental/_no-plugin/decimal/input.js

@@ -0,0 +1 @@
+.1m;

packages/babel-parser/test/fixtures/experimental/_no-plugin/decimal/options.json

@@ -0,0 +1,3 @@
+{
+  "throws": "This experimental syntax requires enabling the parser plugin: 'decimal' (1:2)"
+}

packages/babel-parser/test/fixtures/experimental/decimal/decimal-as-property-name/input.js

@@ -0,0 +1 @@
+({ 0m: 0, .1m() {}, get 0.2m(){}, set 3m(_){}, async 4m() {}, *.5m() {} });

packages/babel-parser/test/fixtures/experimental/decimal/decimal-as-property-name/output.json

@@ -0,0 +1,188 @@
+{
+  "type": "File",
+  "start":0,"end":75,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":75}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":75,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":75}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":75,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":75}},
+        "expression": {
+          "type": "ObjectExpression",
+          "start":1,"end":73,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":73}},
+          "properties": [
+            {
+              "type": "ObjectProperty",
+              "start":3,"end":8,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":8}},
+              "method": false,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":3,"end":5,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":5}},
+                "extra": {
+                  "rawValue": "0",
+                  "raw": "0m"
+                },
+                "value": "0"
+              },
+              "computed": false,
+              "shorthand": false,
+              "value": {
+                "type": "NumericLiteral",
+                "start":7,"end":8,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":8}},
+                "extra": {
+                  "rawValue": 0,
+                  "raw": "0"
+                },
+                "value": 0
+              }
+            },
+            {
+              "type": "ObjectMethod",
+              "start":10,"end":18,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":18}},
+              "method": true,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":10,"end":13,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":13}},
+                "extra": {
+                  "rawValue": ".1",
+                  "raw": ".1m"
+                },
+                "value": ".1"
+              },
+              "computed": false,
+              "kind": "method",
+              "id": null,
+              "generator": false,
+              "async": false,
+              "params": [],
+              "body": {
+                "type": "BlockStatement",
+                "start":16,"end":18,"loc":{"start":{"line":1,"column":16},"end":{"line":1,"column":18}},
+                "body": [],
+                "directives": []
+              }
+            },
+            {
+              "type": "ObjectMethod",
+              "start":20,"end":32,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":32}},
+              "method": false,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":24,"end":28,"loc":{"start":{"line":1,"column":24},"end":{"line":1,"column":28}},
+                "extra": {
+                  "rawValue": "0.2",
+                  "raw": "0.2m"
+                },
+                "value": "0.2"
+              },
+              "computed": false,
+              "kind": "get",
+              "id": null,
+              "generator": false,
+              "async": false,
+              "params": [],
+              "body": {
+                "type": "BlockStatement",
+                "start":30,"end":32,"loc":{"start":{"line":1,"column":30},"end":{"line":1,"column":32}},
+                "body": [],
+                "directives": []
+              }
+            },
+            {
+              "type": "ObjectMethod",
+              "start":34,"end":45,"loc":{"start":{"line":1,"column":34},"end":{"line":1,"column":45}},
+              "method": false,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":38,"end":40,"loc":{"start":{"line":1,"column":38},"end":{"line":1,"column":40}},
+                "extra": {
+                  "rawValue": "3",
+                  "raw": "3m"
+                },
+                "value": "3"
+              },
+              "computed": false,
+              "kind": "set",
+              "id": null,
+              "generator": false,
+              "async": false,
+              "params": [
+                {
+                  "type": "Identifier",
+                  "start":41,"end":42,"loc":{"start":{"line":1,"column":41},"end":{"line":1,"column":42},"identifierName":"_"},
+                  "name": "_"
+                }
+              ],
+              "body": {
+                "type": "BlockStatement",
+                "start":43,"end":45,"loc":{"start":{"line":1,"column":43},"end":{"line":1,"column":45}},
+                "body": [],
+                "directives": []
+              }
+            },
+            {
+              "type": "ObjectMethod",
+              "start":47,"end":60,"loc":{"start":{"line":1,"column":47},"end":{"line":1,"column":60}},
+              "method": true,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":53,"end":55,"loc":{"start":{"line":1,"column":53},"end":{"line":1,"column":55}},
+                "extra": {
+                  "rawValue": "4",
+                  "raw": "4m"
+                },
+                "value": "4"
+              },
+              "computed": false,
+              "kind": "method",
+              "id": null,
+              "generator": false,
+              "async": true,
+              "params": [],
+              "body": {
+                "type": "BlockStatement",
+                "start":58,"end":60,"loc":{"start":{"line":1,"column":58},"end":{"line":1,"column":60}},
+                "body": [],
+                "directives": []
+              }
+            },
+            {
+              "type": "ObjectMethod",
+              "start":62,"end":71,"loc":{"start":{"line":1,"column":62},"end":{"line":1,"column":71}},
+              "method": true,
+              "key": {
+                "type": "DecimalLiteral",
+                "start":63,"end":66,"loc":{"start":{"line":1,"column":63},"end":{"line":1,"column":66}},
+                "extra": {
+                  "rawValue": ".5",
+                  "raw": ".5m"
+                },
+                "value": ".5"
+              },
+              "computed": false,
+              "kind": "method",
+              "id": null,
+              "generator": true,
+              "async": false,
+              "params": [],
+              "body": {
+                "type": "BlockStatement",
+                "start":69,"end":71,"loc":{"start":{"line":1,"column":69},"end":{"line":1,"column":71}},
+                "body": [],
+                "directives": []
+              }
+            }
+          ],
+          "extra": {
+            "parenthesized": true,
+            "parenStart": 0
+          }
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-binary/input.js

@@ -0,0 +1 @@
+0b101011101m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-binary/options.json

@@ -0,0 +1,6 @@
+{
+  "plugins": [
+    "decimal"
+  ],
+  "throws": "Invalid decimal (1:0)"
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-e/input.js

@@ -0,0 +1 @@
+2e9m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-e/output.json

@@ -0,0 +1,29 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "errors": [
+    "SyntaxError: Invalid decimal (1:0)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "extra": {
+            "rawValue": "2e9",
+            "raw": "2e9m"
+          },
+          "value": "2e9"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-hexadecimal/input.js

@@ -0,0 +1 @@
+0x16432m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-hexadecimal/options.json

@@ -0,0 +1,6 @@
+{
+  "plugins": [
+    "decimal"
+  ],
+  "throws": "Invalid decimal (1:0)"
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-non-octal-decimal-int/input.js

@@ -0,0 +1 @@
+089m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-non-octal-decimal-int/output.json

@@ -0,0 +1,29 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "errors": [
+    "SyntaxError: Invalid decimal (1:0)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "extra": {
+            "rawValue": "089",
+            "raw": "089m"
+          },
+          "value": "089"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-octal-legacy/input.js

@@ -0,0 +1 @@
+016432m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-octal-legacy/output.json

@@ -0,0 +1,29 @@
+{
+  "type": "File",
+  "start":0,"end":7,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":7}},
+  "errors": [
+    "SyntaxError: Invalid decimal (1:0)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":7,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":7}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":7,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":7}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":7,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":7}},
+          "extra": {
+            "rawValue": "016432",
+            "raw": "016432m"
+          },
+          "value": "016432"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/invalid-octal-new/input.js

@@ -0,0 +1 @@
+0o16432m

packages/babel-parser/test/fixtures/experimental/decimal/invalid-octal-new/options.json

@@ -0,0 +1,6 @@
+{
+  "plugins": [
+    "decimal"
+  ],
+  "throws": "Invalid decimal (1:0)"
+}

packages/babel-parser/test/fixtures/experimental/decimal/options.json

@@ -0,0 +1,3 @@
+{
+  "plugins": ["decimal"]
+}

packages/babel-parser/test/fixtures/experimental/decimal/valid-decimal/input.js

@@ -0,0 +1 @@
+1.m

packages/babel-parser/test/fixtures/experimental/decimal/valid-decimal/output.json

@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+          "extra": {
+            "rawValue": "1.",
+            "raw": "1.m"
+          },
+          "value": "1."
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/valid-float-decimal/input.js

@@ -0,0 +1 @@
+100.0m

packages/babel-parser/test/fixtures/experimental/decimal/valid-float-decimal/output.json

@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+          "extra": {
+            "rawValue": "100.0",
+            "raw": "100.0m"
+          },
+          "value": "100.0"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/valid-float/input.js

@@ -0,0 +1 @@
+100m

packages/babel-parser/test/fixtures/experimental/decimal/valid-float/output.json

@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "extra": {
+            "rawValue": "100",
+            "raw": "100m"
+          },
+          "value": "100"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/valid-large/input.js

@@ -0,0 +1 @@
+9223372036854775807m

packages/babel-parser/test/fixtures/experimental/decimal/valid-large/output.json

@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":20,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":20}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":20,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":20}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":20,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":20}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":20,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":20}},
+          "extra": {
+            "rawValue": "9223372036854775807",
+            "raw": "9223372036854775807m"
+          },
+          "value": "9223372036854775807"
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/experimental/decimal/valid-small/input.js

@@ -0,0 +1 @@
+100.m

packages/babel-parser/test/fixtures/experimental/decimal/valid-small/output.json

@@ -0,0 +1,26 @@
+{
+  "type": "File",
+  "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}},
+        "expression": {
+          "type": "DecimalLiteral",
+          "start":0,"end":5,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":5}},
+          "extra": {
+            "rawValue": "100.",
+            "raw": "100.m"
+          },
+          "value": "100."
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/typings/babel-parser.d.ts

@@ -100,6 +100,7 @@ export type ParserPlugin =
     'classPrivateMethods' |
     'classPrivateProperties' |
     'classProperties' |
+    'decimal' |
     'decorators' |
     'decorators-legacy' |
     'doExpressions' |