cerxes/babel
3
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Subsume json (#7985)

Browse Source 
* Add Subsume JSON transform

https://github.com/tc39/proposal-json-superset is at stage 3.

This allows U+2028 LINE SEPARATOR and U+2029 PARAGRAPH SEPARATOR to appear unescaped inside strings and directives.

* Move to Stage 3

* Break diretive parsing

* Update README

* Handle multi-escape sequences

* Remove babylon files after rename
This commit is contained in:
Justin Ridgewell
2018-05-19 15:32:05 -04:00
committed by GitHub
parent 51eef099b3
commit 0879a6d608
34 changed files with 809 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
packages/babel-parser/src/tokenizer/index.js
View File

@@ -1069,6 +1069,7 @@ export default class Tokenizer extends LocationParser {
readString(quote: number): void {
let out = "",
chunkStart = ++this.state.pos;
const hasJsonStrings = this.hasPlugin("jsonStrings");
for (;;) {
if (this.state.pos >= this.input.length) {
this.raise(this.state.start, "Unterminated string constant");
@@ -1080,10 +1081,14 @@ export default class Tokenizer extends LocationParser {
// $FlowFixMe
out += this.readEscapedChar(false);
chunkStart = this.state.pos;
} else if (
hasJsonStrings &&
(ch === charCodes.lineSeparator || ch === charCodes.paragraphSeparator)
) {
++this.state.pos;
} else if (isNewLine(ch)) {
this.raise(this.state.start, "Unterminated string constant");
} else {
if (isNewLine(ch)) {
this.raise(this.state.start, "Unterminated string constant");
}
++this.state.pos;
}
}

2
packages/babel-parser/test/fixtures/experimental/json-strings/directive-line-separator/input.js vendored Normal file
View File

@@ -0,0 +1,2 @@
"before
after";
// ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')

105
packages/babel-parser/test/fixtures/experimental/json-strings/directive-line-separator/output.json vendored Normal file
View File

@@ -0,0 +1,105 @@
{
"type": "File",
"start": 0,
"end": 98,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 82
}
},
"program": {
"type": "Program",
"start": 0,
"end": 98,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 82
}
},
"sourceType": "script",
"body": [],
"directives": [
{
"type": "Directive",
"start": 0,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 15
}
},
"value": {
"type": "DirectiveLiteral",
"start": 0,
"end": 14,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 14
}
},
"value": "before
after",
"extra": {
"raw": "\"before
after\"",
"rawValue": "before
after"
}
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 16,
"end": 98,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 82
}
}
}
]
}
]
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 16,
"end": 98,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 82
}
}
}
]
}

2
packages/babel-parser/test/fixtures/experimental/json-strings/directive-paragraph-separator/input.js vendored Normal file
View File

@@ -0,0 +1,2 @@
"before
after";
// ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')

105
packages/babel-parser/test/fixtures/experimental/json-strings/directive-paragraph-separator/output.json vendored Normal file
View File

@@ -0,0 +1,105 @@
{
"type": "File",
"start": 0,
"end": 103,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 87
}
},
"program": {
"type": "Program",
"start": 0,
"end": 103,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 87
}
},
"sourceType": "script",
"body": [],
"directives": [
{
"type": "Directive",
"start": 0,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 15
}
},
"value": {
"type": "DirectiveLiteral",
"start": 0,
"end": 14,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 14
}
},
"value": "before
after",
"extra": {
"raw": "\"before
after\"",
"rawValue": "before
after"
}
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 16,
"end": 103,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 87
}
}
}
]
}
]
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 16,
"end": 103,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 87
}
}
}
]
}

3
packages/babel-parser/test/fixtures/experimental/json-strings/options.json vendored Normal file
View File

@@ -0,0 +1,3 @@
{
"plugins": ["jsonStrings"]
}

2
packages/babel-parser/test/fixtures/experimental/json-strings/string-line-separator/input.js vendored Normal file
View File

@@ -0,0 +1,2 @@
("before
after");
// ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')

107
packages/babel-parser/test/fixtures/experimental/json-strings/string-line-separator/output.json vendored Normal file
View File

@@ -0,0 +1,107 @@
{
"type": "File",
"start": 0,
"end": 101,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 83
}
},
"program": {
"type": "Program",
"start": 0,
"end": 101,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 83
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 17,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 17
}
},
"expression": {
"type": "StringLiteral",
"start": 1,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 1
},
"end": {
"line": 1,
"column": 15
}
},
"extra": {
"rawValue": "before
after",
"raw": "\"before
after\"",
"parenthesized": true,
"parenStart": 0
},
"value": "before
after"
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 101,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 83
}
}
}
]
}
],
"directives": []
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2028 LINE SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 101,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 83
}
}
}
]
}

2
packages/babel-parser/test/fixtures/experimental/json-strings/string-paragraph-separator/input.js vendored Normal file
View File

@@ -0,0 +1,2 @@
("before
after");
// ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')

107
packages/babel-parser/test/fixtures/experimental/json-strings/string-paragraph-separator/output.json vendored Normal file
View File

@@ -0,0 +1,107 @@
{
"type": "File",
"start": 0,
"end": 106,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 88
}
},
"program": {
"type": "Program",
"start": 0,
"end": 106,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 2,
"column": 88
}
},
"sourceType": "script",
"body": [
{
"type": "ExpressionStatement",
"start": 0,
"end": 17,
"loc": {
"start": {
"line": 1,
"column": 0
},
"end": {
"line": 1,
"column": 17
}
},
"expression": {
"type": "StringLiteral",
"start": 1,
"end": 15,
"loc": {
"start": {
"line": 1,
"column": 1
},
"end": {
"line": 1,
"column": 15
}
},
"extra": {
"rawValue": "before
after",
"raw": "\"before
after\"",
"parenthesized": true,
"parenStart": 0
},
"value": "before
after"
},
"trailingComments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 106,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 88
}
}
}
]
}
],
"directives": []
},
"comments": [
{
"type": "CommentLine",
"value": " ^ That's a U+2029 PARAGRAPH SEPARATOR UTF-16 char (between 'before' and 'after')",
"start": 18,
"end": 106,
"loc": {
"start": {
"line": 2,
"column": 0
},
"end": {
"line": 2,
"column": 88
}
}
}
]
}