Fix: major revision to valid and invalid numeric literal separator "sibling" characters (#745)

src/tokenizer/index.js

@@ -43,6 +43,48 @@ const forbiddenNumericSeparatorSiblings = {
   ],
 };
 
+const allowedNumericSeparatorSiblings = {};
+allowedNumericSeparatorSiblings.bin = [
+  // 0 - 1
+  48,
+  49,
+];
+allowedNumericSeparatorSiblings.oct = [
+  // 0 - 7
+  ...allowedNumericSeparatorSiblings.bin,
+  50,
+  51,
+  52,
+  53,
+  54,
+  55,
+];
+allowedNumericSeparatorSiblings.dec = [
+  // 0 - 9
+  ...allowedNumericSeparatorSiblings.oct,
+  56,
+  57,
+];
+
+allowedNumericSeparatorSiblings.hex = [
+  // 0 - 9, A - F, a - f,
+  ...allowedNumericSeparatorSiblings.dec,
+  // A - F
+  65,
+  66,
+  67,
+  68,
+  69,
+  70,
+  // a - f
+  97,
+  98,
+  99,
+  100,
+  101,
+  102,
+];
+
 // Object type used to represent tokens. Note that normally, tokens
 // simply exist as properties on the parser object. This is only
 // used for the onToken callback and the external tokenizer.
@@ -718,6 +760,15 @@ export default class Tokenizer extends LocationParser {
       radix === 16
         ? forbiddenNumericSeparatorSiblings.hex
         : forbiddenNumericSeparatorSiblings.decBinOct;
+    const allowedSiblings =
+      radix === 16
+        ? allowedNumericSeparatorSiblings.hex
+        : radix === 10
+          ? allowedNumericSeparatorSiblings.dec
+          : radix === 8
+            ? allowedNumericSeparatorSiblings.oct
+            : allowedNumericSeparatorSiblings.bin;
+
     let total = 0;
 
     for (let i = 0, e = len == null ? Infinity : len; i < e; ++i) {
@@ -728,6 +779,10 @@ export default class Tokenizer extends LocationParser {
         const prev = this.input.charCodeAt(this.state.pos - 1);
         const next = this.input.charCodeAt(this.state.pos + 1);
         if (code === 95) {
+          if (allowedSiblings.indexOf(next) === -1) {
+            this.raise(this.state.pos, "Invalid or unexpected token");
+          }
+
           if (
             forbiddenSiblings.indexOf(prev) > -1 ||
             forbiddenSiblings.indexOf(next) > -1 ||

test/fixtures/experimental/numeric-separator/invalid-0/actual.js

@@ -1 +1 @@
-1_
+1_

test/fixtures/experimental/numeric-separator/invalid-1/actual.js

@@ -1 +1 @@
-1_1_
+1_1_

test/fixtures/experimental/numeric-separator/invalid-10/actual.js

@@ -1 +1 @@
-0x1_1_
+0x1_1_

test/fixtures/experimental/numeric-separator/invalid-100/actual.js

@@ -0,0 +1 @@
+(1_)

test/fixtures/experimental/numeric-separator/invalid-100/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:2)" }

test/fixtures/experimental/numeric-separator/invalid-101/actual.js

@@ -0,0 +1 @@
+(1_1_)

test/fixtures/experimental/numeric-separator/invalid-101/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-102/actual.js

@@ -0,0 +1 @@
+(1_1__)

test/fixtures/experimental/numeric-separator/invalid-102/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-103/actual.js

@@ -0,0 +1 @@
+(1__1)

test/fixtures/experimental/numeric-separator/invalid-103/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:2)" }

test/fixtures/experimental/numeric-separator/invalid-104/actual.js

@@ -0,0 +1 @@
+(1_1_.1_1)

test/fixtures/experimental/numeric-separator/invalid-104/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-105/actual.js

@@ -0,0 +1 @@
+(1_1._1_1)

test/fixtures/experimental/numeric-separator/invalid-105/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:5)" }

test/fixtures/experimental/numeric-separator/invalid-106/actual.js

@@ -0,0 +1 @@
+(1_1.1_e1)

test/fixtures/experimental/numeric-separator/invalid-106/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-107/actual.js

@@ -0,0 +1 @@
+(1_1.1_E1)

test/fixtures/experimental/numeric-separator/invalid-107/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-108/actual.js

@@ -0,0 +1 @@
+(1_1.1e_1)

test/fixtures/experimental/numeric-separator/invalid-108/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }

test/fixtures/experimental/numeric-separator/invalid-109/actual.js

@@ -0,0 +1 @@
+(1_1.1E_1)

test/fixtures/experimental/numeric-separator/invalid-109/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }

test/fixtures/experimental/numeric-separator/invalid-11/actual.js

@@ -1 +1 @@
-0xa_1_
+0xa_1_

test/fixtures/experimental/numeric-separator/invalid-110/actual.js

@@ -0,0 +1 @@
+(0x1_1_)

test/fixtures/experimental/numeric-separator/invalid-110/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-111/actual.js

@@ -0,0 +1 @@
+(0xa_1_)

test/fixtures/experimental/numeric-separator/invalid-111/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-112/actual.js

@@ -0,0 +1 @@
+(0x_a_1)

test/fixtures/experimental/numeric-separator/invalid-112/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-113/actual.js

@@ -0,0 +1 @@
+(0x__1_1_)

test/fixtures/experimental/numeric-separator/invalid-113/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-114/actual.js

@@ -0,0 +1 @@
+(0x_1__1)

test/fixtures/experimental/numeric-separator/invalid-114/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-115/actual.js

@@ -0,0 +1 @@
+(0x_1_1_)

test/fixtures/experimental/numeric-separator/invalid-115/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-116/actual.js

@@ -0,0 +1 @@
+(0o_1_1)

test/fixtures/experimental/numeric-separator/invalid-116/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-117/actual.js

@@ -0,0 +1 @@
+(0o_11)

test/fixtures/experimental/numeric-separator/invalid-117/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-118/actual.js

@@ -0,0 +1 @@
+(0o_01_1_)

test/fixtures/experimental/numeric-separator/invalid-118/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-119/actual.js

@@ -0,0 +1 @@
+(0b_0_1_1)

test/fixtures/experimental/numeric-separator/invalid-119/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-12/actual.js

@@ -1 +1 @@
-0x_a_1
+0x_a_1

test/fixtures/experimental/numeric-separator/invalid-120/actual.js

@@ -0,0 +1 @@
+(0b_01_1_)

test/fixtures/experimental/numeric-separator/invalid-120/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-121/actual.js

@@ -0,0 +1 @@
+(0b01_1_)

test/fixtures/experimental/numeric-separator/invalid-121/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }

test/fixtures/experimental/numeric-separator/invalid-122/actual.js

@@ -0,0 +1 @@
+(0o1_1_)

test/fixtures/experimental/numeric-separator/invalid-122/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-123/actual.js

@@ -0,0 +1 @@
+(0o_1_1_)

test/fixtures/experimental/numeric-separator/invalid-123/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-124/actual.js

@@ -0,0 +1 @@
+{1_}

test/fixtures/experimental/numeric-separator/invalid-124/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:2)" }

test/fixtures/experimental/numeric-separator/invalid-125/actual.js

@@ -0,0 +1 @@
+{1_1_}

test/fixtures/experimental/numeric-separator/invalid-125/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-126/actual.js

@@ -0,0 +1 @@
+{1_1__}

test/fixtures/experimental/numeric-separator/invalid-126/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-127/actual.js

@@ -0,0 +1 @@
+{1__1}

test/fixtures/experimental/numeric-separator/invalid-127/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:2)" }

test/fixtures/experimental/numeric-separator/invalid-128/actual.js

@@ -0,0 +1 @@
+{1_1_.1_1}

test/fixtures/experimental/numeric-separator/invalid-128/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:4)" }

test/fixtures/experimental/numeric-separator/invalid-129/actual.js

@@ -0,0 +1 @@
+{1_1._1_1}

test/fixtures/experimental/numeric-separator/invalid-129/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:5)" }

test/fixtures/experimental/numeric-separator/invalid-13/actual.js

@@ -1 +1 @@
-0x__1_1_
+0x__1_1_

test/fixtures/experimental/numeric-separator/invalid-130/actual.js

@@ -0,0 +1 @@
+{1_1.1_e1}

test/fixtures/experimental/numeric-separator/invalid-130/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-131/actual.js

@@ -0,0 +1 @@
+{1_1.1_E1}

test/fixtures/experimental/numeric-separator/invalid-131/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-132/actual.js

@@ -0,0 +1 @@
+{1_1.1e_1}

test/fixtures/experimental/numeric-separator/invalid-132/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }

test/fixtures/experimental/numeric-separator/invalid-133/actual.js

@@ -0,0 +1 @@
+{1_1.1E_1}

test/fixtures/experimental/numeric-separator/invalid-133/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }

test/fixtures/experimental/numeric-separator/invalid-134/actual.js

@@ -0,0 +1 @@
+{0x1_1_}

test/fixtures/experimental/numeric-separator/invalid-134/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-135/actual.js

@@ -0,0 +1 @@
+{0xa_1_}

test/fixtures/experimental/numeric-separator/invalid-135/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:6)" }

test/fixtures/experimental/numeric-separator/invalid-136/actual.js

@@ -0,0 +1 @@
+{0x_a_1}

test/fixtures/experimental/numeric-separator/invalid-136/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-137/actual.js

@@ -0,0 +1 @@
+{0x__1_1_}

test/fixtures/experimental/numeric-separator/invalid-137/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-138/actual.js

@@ -0,0 +1 @@
+{0x_1__1}

test/fixtures/experimental/numeric-separator/invalid-138/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-139/actual.js

@@ -0,0 +1 @@
+{0x_1_1_}

test/fixtures/experimental/numeric-separator/invalid-139/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-14/actual.js

@@ -1 +1 @@
-0x_1__1
+0x_1__1

test/fixtures/experimental/numeric-separator/invalid-140/actual.js

@@ -0,0 +1 @@
+{0o_1_1}

test/fixtures/experimental/numeric-separator/invalid-140/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-141/actual.js

@@ -0,0 +1 @@
+{0o_11}

test/fixtures/experimental/numeric-separator/invalid-141/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-142/actual.js

@@ -0,0 +1 @@
+{0o_01_1_}

test/fixtures/experimental/numeric-separator/invalid-142/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-143/actual.js

@@ -0,0 +1 @@
+{0b_0_1_1}

test/fixtures/experimental/numeric-separator/invalid-143/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-144/actual.js

@@ -0,0 +1 @@
+{0b_01_1_}

test/fixtures/experimental/numeric-separator/invalid-144/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:3)" }

test/fixtures/experimental/numeric-separator/invalid-145/actual.js

@@ -0,0 +1 @@
+{0b01_1_}

test/fixtures/experimental/numeric-separator/invalid-145/options.json

@@ -0,0 +1 @@
+{ "throws": "Invalid or unexpected token (1:7)" }