Expose @babel/eslint-parser/experimental-worker (#13398)

* Expose `@babel/eslint-parser/experimental-worker`

* Fix `@babel/runtime` build on Windows
This commit is contained in:
Nicolò Ribaudo 2021-08-03 23:23:32 +02:00 committed by GitHub
parent 830b99dc83
commit 885e1e02f5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 305 additions and 197 deletions

View File

@ -23,6 +23,7 @@
"type": "commonjs", "type": "commonjs",
"exports": { "exports": {
".": "./lib/index.cjs", ".": "./lib/index.cjs",
"./experimental-worker": "./lib/experimental-worker.cjs",
"./package.json": "./package.json" "./package.json": "./package.json"
}, },
"peerDependencies": { "peerDependencies": {

View File

@ -4,13 +4,11 @@ const OriginalPatternVisitor = require("eslint-scope/lib/pattern-visitor");
const OriginalReferencer = require("eslint-scope/lib/referencer"); const OriginalReferencer = require("eslint-scope/lib/referencer");
const { getKeys: fallback } = require("eslint-visitor-keys"); const { getKeys: fallback } = require("eslint-visitor-keys");
const { getTypesInfo, getVisitorKeys } = require("./client.cjs");
let visitorKeysMap; let visitorKeysMap;
function getVisitorValues(nodeType) { function getVisitorValues(nodeType, client) {
if (visitorKeysMap) return visitorKeysMap[nodeType]; if (visitorKeysMap) return visitorKeysMap[nodeType];
const { FLOW_FLIPPED_ALIAS_KEYS, VISITOR_KEYS } = getTypesInfo(); const { FLOW_FLIPPED_ALIAS_KEYS, VISITOR_KEYS } = client.getTypesInfo();
const flowFlippedAliasKeys = FLOW_FLIPPED_ALIAS_KEYS.concat([ const flowFlippedAliasKeys = FLOW_FLIPPED_ALIAS_KEYS.concat([
"ArrayPattern", "ArrayPattern",
@ -63,6 +61,13 @@ class PatternVisitor extends OriginalPatternVisitor {
} }
class Referencer extends OriginalReferencer { class Referencer extends OriginalReferencer {
#client;
constructor(options, scopeManager, client) {
super(options, scopeManager);
this.#client = client;
}
// inherits. // inherits.
visitPattern(node, options, callback) { visitPattern(node, options, callback) {
if (!node) { if (!node) {
@ -264,7 +269,7 @@ class Referencer extends OriginalReferencer {
} }
// get property to check (params, id, etc...) // get property to check (params, id, etc...)
const visitorValues = getVisitorValues(node.type); const visitorValues = getVisitorValues(node.type, this.#client);
if (!visitorValues) { if (!visitorValues) {
return; return;
} }
@ -328,7 +333,7 @@ class Referencer extends OriginalReferencer {
} }
} }
module.exports = function analyzeScope(ast, parserOptions) { module.exports = function analyzeScope(ast, parserOptions, client) {
const options = { const options = {
ignoreEval: true, ignoreEval: true,
optimistic: false, optimistic: false,
@ -343,10 +348,10 @@ module.exports = function analyzeScope(ast, parserOptions) {
fallback, fallback,
}; };
options.childVisitorKeys = getVisitorKeys(); options.childVisitorKeys = client.getVisitorKeys();
const scopeManager = new escope.ScopeManager(options); const scopeManager = new escope.ScopeManager(options);
const referencer = new Referencer(options, scopeManager); const referencer = new Referencer(options, scopeManager, client);
referencer.visit(ast); referencer.visit(ast);

View File

@ -1,67 +1,105 @@
const path = require("path"); const path = require("path");
let send; const ACTIONS = {
GET_VERSION: "GET_VERSION",
GET_TYPES_INFO: "GET_TYPES_INFO",
GET_VISITOR_KEYS: "GET_VISITOR_KEYS",
GET_TOKEN_LABELS: "GET_TOKEN_LABELS",
MAYBE_PARSE: "MAYBE_PARSE",
MAYBE_PARSE_SYNC: "MAYBE_PARSE_SYNC",
};
exports.getVersion = sendCached("GET_VERSION"); class Client {
#send;
exports.getTypesInfo = sendCached("GET_TYPES_INFO"); constructor(send) {
this.#send = send;
}
exports.getVisitorKeys = sendCached("GET_VISITOR_KEYS"); #vCache;
getVersion() {
return (this.#vCache ??= this.#send(ACTIONS.GET_VERSION, undefined));
}
exports.getTokLabels = sendCached("GET_TOKEN_LABELS"); #tiCache;
getTypesInfo() {
return (this.#tiCache ??= this.#send(ACTIONS.GET_TYPES_INFO, undefined));
}
exports.maybeParse = (code, options) => send("MAYBE_PARSE", { code, options }); #vkCache;
getVisitorKeys() {
return (this.#vkCache ??= this.#send(ACTIONS.GET_VISITOR_KEYS, undefined));
}
function sendCached(action) { #tlCache;
let cache = null; getTokLabels() {
return (this.#tlCache ??= this.#send(ACTIONS.GET_TOKEN_LABELS, undefined));
}
return () => { maybeParse(code, options) {
if (!cache) cache = send(action, undefined); return this.#send(ACTIONS.MAYBE_PARSE, { code, options });
return cache; }
};
} }
if (process.env.BABEL_8_BREAKING) { // We need to run Babel in a worker for two reasons:
const { // 1. ESLint workers must be CJS files, and this is a problem
Worker, // since Babel 8+ uses native ESM
receiveMessageOnPort, // 2. ESLint parsers must run synchronously, but many steps
MessageChannel, // of Babel's config loading (which is done for each file)
SHARE_ENV, // can be asynchronous
} = require("worker_threads"); // If ESLint starts supporting async parsers, we can move
// everything back to the main thread.
exports.WorkerClient = class WorkerClient extends Client {
static #worker_threads_cache;
static get #worker_threads() {
return (WorkerClient.#worker_threads_cache ??= require("worker_threads"));
}
// We need to run Babel in a worker for two reasons: #worker = new WorkerClient.#worker_threads.Worker(
// 1. ESLint workers must be CJS files, and this is a problem
// since Babel 8+ uses native ESM
// 2. ESLint parsers must run synchronously, but many steps
// of Babel's config loading (which is done for each file)
// can be asynchronous
// If ESLint starts supporting async parsers, we can move
// everything back to the main thread.
const worker = new Worker(
path.resolve(__dirname, "../lib/worker/index.cjs"), path.resolve(__dirname, "../lib/worker/index.cjs"),
{ env: SHARE_ENV }, { env: WorkerClient.#worker_threads.SHARE_ENV },
); );
// The worker will never exit by itself. Prevent it from keeping #signal = new Int32Array(new SharedArrayBuffer(4));
// the main process alive.
worker.unref();
const signal = new Int32Array(new SharedArrayBuffer(4)); constructor() {
super((action, payload) => {
this.#signal[0] = 0;
const subChannel = new WorkerClient.#worker_threads.MessageChannel();
send = (action, payload) => { this.#worker.postMessage(
signal[0] = 0; { signal: this.#signal, port: subChannel.port1, action, payload },
const subChannel = new MessageChannel(); [subChannel.port1],
);
worker.postMessage({ signal, port: subChannel.port1, action, payload }, [ Atomics.wait(this.#signal, 0, 0);
subChannel.port1, const { message } = WorkerClient.#worker_threads.receiveMessageOnPort(
]); subChannel.port2,
);
Atomics.wait(signal, 0, 0); if (message.error) throw Object.assign(message.error, message.errorData);
const { message } = receiveMessageOnPort(subChannel.port2); else return message.result;
});
if (message.error) throw Object.assign(message.error, message.errorData); // The worker will never exit by itself. Prevent it from keeping
else return message.result; // the main process alive.
this.#worker.unref();
}
};
if (!process.env.BABEL_8_BREAKING) {
exports.LocalClient = class LocalClient extends Client {
static #handleMessage;
constructor() {
LocalClient.#handleMessage ??= require("./worker/handle-message.cjs");
super((action, payload) => {
return LocalClient.#handleMessage(
action === ACTIONS.MAYBE_PARSE ? ACTIONS.MAYBE_PARSE_SYNC : action,
payload,
);
});
}
}; };
} else {
send = require("./worker/index.cjs");
} }

View File

@ -0,0 +1,22 @@
const [major, minor] = process.versions.node.split(".").map(Number);
if (major < 12 || (major === 12 && minor < 3)) {
throw new Error(
"@babel/eslint-parser/experimental-worker requires Node.js >= 12.3.0",
);
}
const { normalizeESLintConfig } = require("./configuration.cjs");
const analyzeScope = require("./analyze-scope.cjs");
const baseParse = require("./parse.cjs");
const { WorkerClient } = require("./client.cjs");
const client = new WorkerClient();
exports.parseForESLint = function (code, options = {}) {
const normalizedOptions = normalizeESLintConfig(options);
const ast = baseParse(code, normalizedOptions, client);
const scopeManager = analyzeScope(ast, normalizedOptions, client);
return { ast, scopeManager, visitorKeys: client.getVisitorKeys() };
};

View File

@ -1,63 +1,20 @@
const semver = require("semver");
const { normalizeESLintConfig } = require("./configuration.cjs"); const { normalizeESLintConfig } = require("./configuration.cjs");
const analyzeScope = require("./analyze-scope.cjs"); const analyzeScope = require("./analyze-scope.cjs");
const { const baseParse = require("./parse.cjs");
getVersion,
getVisitorKeys,
getTokLabels,
maybeParse,
} = require("./client.cjs");
const convert = require("./convert/index.cjs");
const babelParser = require(require.resolve("@babel/parser", { const { LocalClient, WorkerClient } = require("./client.cjs");
paths: [require.resolve("@babel/core/package.json")], const client = new (
})); process.env.BABEL_8_BREAKING ? WorkerClient : LocalClient
)();
let isRunningMinSupportedCoreVersion = null;
function baseParse(code, options) {
// Ensure we're using a version of `@babel/core` that includes `parse()` and `tokTypes`.
const minSupportedCoreVersion = ">=7.2.0";
if (typeof isRunningMinSupportedCoreVersion !== "boolean") {
isRunningMinSupportedCoreVersion = semver.satisfies(
getVersion(),
minSupportedCoreVersion,
);
}
if (!isRunningMinSupportedCoreVersion) {
throw new Error(
`@babel/eslint-parser@${
PACKAGE_JSON.version
} does not support @babel/core@${getVersion()}. Please upgrade to @babel/core@${minSupportedCoreVersion}.`,
);
}
const { ast, parserOptions } = maybeParse(code, options);
if (ast) return ast;
try {
return convert.ast(
babelParser.parse(code, parserOptions),
code,
getTokLabels(),
getVisitorKeys(),
);
} catch (err) {
throw convert.error(err);
}
}
exports.parse = function (code, options = {}) { exports.parse = function (code, options = {}) {
return baseParse(code, normalizeESLintConfig(options)); return baseParse(code, normalizeESLintConfig(options), client);
}; };
exports.parseForESLint = function (code, options = {}) { exports.parseForESLint = function (code, options = {}) {
const normalizedOptions = normalizeESLintConfig(options); const normalizedOptions = normalizeESLintConfig(options);
const ast = baseParse(code, normalizedOptions); const ast = baseParse(code, normalizedOptions, client);
const scopeManager = analyzeScope(ast, normalizedOptions); const scopeManager = analyzeScope(ast, normalizedOptions, client);
return { ast, scopeManager, visitorKeys: getVisitorKeys() }; return { ast, scopeManager, visitorKeys: client.getVisitorKeys() };
}; };

View File

@ -0,0 +1,45 @@
"use strict";
const semver = require("semver");
const convert = require("./convert/index.cjs");
const babelParser = require(require.resolve("@babel/parser", {
paths: [require.resolve("@babel/core/package.json")],
}));
let isRunningMinSupportedCoreVersion = null;
module.exports = function parse(code, options, client) {
// Ensure we're using a version of `@babel/core` that includes `parse()` and `tokTypes`.
const minSupportedCoreVersion = ">=7.2.0";
if (typeof isRunningMinSupportedCoreVersion !== "boolean") {
isRunningMinSupportedCoreVersion = semver.satisfies(
client.getVersion(),
minSupportedCoreVersion,
);
}
if (!isRunningMinSupportedCoreVersion) {
throw new Error(
`@babel/eslint-parser@${
PACKAGE_JSON.version
} does not support @babel/core@${client.getVersion()}. Please upgrade to @babel/core@${minSupportedCoreVersion}.`,
);
}
const { ast, parserOptions } = client.maybeParse(code, options);
if (ast) return ast;
try {
return convert.ast(
babelParser.parse(code, parserOptions),
code,
client.getTokLabels(),
client.getVisitorKeys(),
);
} catch (err) {
throw convert.error(err);
}
};

View File

@ -41,7 +41,7 @@ function normalizeParserOptions(options) {
}; };
} }
function validateResolvedConfig(config, options) { function validateResolvedConfig(config, options, parseOptions) {
if (config !== null) { if (config !== null) {
if (options.requireConfigFile !== false) { if (options.requireConfigFile !== false) {
if (!config.hasFilesystemConfig()) { if (!config.hasFilesystemConfig()) {
@ -54,8 +54,10 @@ function validateResolvedConfig(config, options) {
throw new Error(error); throw new Error(error);
} }
} }
return config.options; if (config.options) return config.options;
} }
return getDefaultParserOptions(parseOptions);
} }
function getDefaultParserOptions(options) { function getDefaultParserOptions(options) {
@ -70,25 +72,14 @@ function getDefaultParserOptions(options) {
}; };
} }
module.exports = function normalizeBabelParseConfig(options) { exports.normalizeBabelParseConfig = async function (options) {
const parseOptions = normalizeParserOptions(options); const parseOptions = normalizeParserOptions(options);
const config = await babel.loadPartialConfigAsync(parseOptions);
if (process.env.BABEL_8_BREAKING) { return validateResolvedConfig(config, options, parseOptions);
return babel };
.loadPartialConfigAsync(parseOptions)
.then(config => validateConfigWithFallback(config)); exports.normalizeBabelParseConfigSync = function (options) {
} else { const parseOptions = normalizeParserOptions(options);
const config = babel.loadPartialConfigSync(parseOptions); const config = babel.loadPartialConfigSync(parseOptions);
return validateConfigWithFallback(config); return validateResolvedConfig(config, options, parseOptions);
}
function validateConfigWithFallback(inputConfig) {
const result = validateResolvedConfig(inputConfig, options);
if (result) {
return result;
} else {
// Fallback when `loadPartialConfig` returns `null` (e.g.: when the file is ignored)
return getDefaultParserOptions(parseOptions);
}
}
}; };

View File

@ -0,0 +1,36 @@
const babel = require("./babel-core.cjs");
const maybeParse = require("./maybeParse.cjs");
const { getVisitorKeys, getTokLabels } = require("./ast-info.cjs");
const {
normalizeBabelParseConfig,
normalizeBabelParseConfigSync,
} = require("./configuration.cjs");
module.exports = function handleMessage(action, payload) {
switch (action) {
case "GET_VERSION":
return babel.version;
case "GET_TYPES_INFO":
return {
FLOW_FLIPPED_ALIAS_KEYS: babel.types.FLIPPED_ALIAS_KEYS.Flow,
VISITOR_KEYS: babel.types.VISITOR_KEYS,
};
case "GET_TOKEN_LABELS":
return getTokLabels();
case "GET_VISITOR_KEYS":
return getVisitorKeys();
case "MAYBE_PARSE":
return normalizeBabelParseConfig(payload.options).then(options =>
maybeParse(payload.code, options),
);
case "MAYBE_PARSE_SYNC":
if (!process.env.BABEL_8_BREAKING) {
return maybeParse(
payload.code,
normalizeBabelParseConfigSync(payload.options),
);
}
}
throw new Error(`Unknown internal parser worker action: ${action}`);
};

View File

@ -1,66 +1,28 @@
const babel = require("./babel-core.cjs"); const babel = require("./babel-core.cjs");
const maybeParse = require("./maybeParse.cjs"); const handleMessage = require("./handle-message.cjs");
const { getVisitorKeys, getTokLabels } = require("./ast-info.cjs");
const normalizeBabelParseConfig = require("./configuration.cjs");
function handleMessage(action, payload) { const { parentPort } = require("worker_threads");
switch (action) {
case "GET_VERSION": parentPort.addListener("message", async ({ signal, port, action, payload }) => {
return babel.version; let response;
case "GET_TYPES_INFO":
return { try {
FLOW_FLIPPED_ALIAS_KEYS: babel.types.FLIPPED_ALIAS_KEYS.Flow, if (babel.init) await babel.init;
VISITOR_KEYS: babel.types.VISITOR_KEYS,
}; response = { result: await handleMessage(action, payload) };
case "GET_TOKEN_LABELS": } catch (error) {
return getTokLabels(); response = { error, errorData: { ...error } };
case "GET_VISITOR_KEYS":
return getVisitorKeys();
case "MAYBE_PARSE":
if (process.env.BABEL_8_BREAKING) {
return normalizeBabelParseConfig(payload.options).then(options =>
maybeParse(payload.code, options),
);
} else {
return maybeParse(
payload.code,
normalizeBabelParseConfig(payload.options),
);
}
} }
throw new Error(`Unknown internal parser worker action: ${action}`); try {
} port.postMessage(response);
} catch {
if (process.env.BABEL_8_BREAKING) { port.postMessage({
const { parentPort } = require("worker_threads"); error: new Error("Cannot serialize worker response"),
});
parentPort.addListener( } finally {
"message", port.close();
async ({ signal, port, action, payload }) => { Atomics.store(signal, 0, 1);
let response; Atomics.notify(signal, 0);
}
try { });
if (babel.init) await babel.init;
response = { result: await handleMessage(action, payload) };
} catch (error) {
response = { error, errorData: { ...error } };
}
try {
port.postMessage(response);
} catch {
port.postMessage({
error: new Error("Cannot serialize worker response"),
});
} finally {
port.close();
Atomics.store(signal, 0, 1);
Atomics.notify(signal, 0);
}
},
);
} else {
module.exports = handleMessage;
}

View File

@ -0,0 +1,13 @@
module.exports = {
root: true,
parser: "@babel/eslint-parser/experimental-worker",
parserOptions: {
babelOptions: {
configFile: __dirname + "/babel.config.mjs",
sourceType: "module",
},
},
rules: {
"template-curly-spacing": "error",
},
};

View File

@ -0,0 +1 @@
export default () => <div />;

View File

@ -0,0 +1,3 @@
export default {
presets: ["@babel/preset-react"],
};

View File

@ -1,6 +1,6 @@
module.exports = { module.exports = {
root: true, root: true,
parser: "@babel/eslint-parser", parser: "@babel/eslint-parser/experimental-worker",
parserOptions: { parserOptions: {
babelOptions: { babelOptions: {
configFile: __dirname + "/babel.config.mjs", configFile: __dirname + "/babel.config.mjs",

View File

@ -16,4 +16,21 @@ describe("Babel config files", () => {
]), ]),
).toMatchObject({ errorCount: 0 }); ).toMatchObject({ errorCount: 0 });
}); });
const babel7node12 =
process.env.BABEL_8_BREAKING || parseInt(process.versions.node) < 12
? it.skip
: it;
babel7node12("experimental worker works with babel.config.mjs", () => {
const engine = new eslint.CLIEngine({ ignore: false });
expect(
engine.executeOnFiles([
path.resolve(
path.dirname(fileURLToPath(import.meta.url)),
`../fixtures/mjs-config-file-babel-7/a.js`,
),
]),
).toMatchObject({ errorCount: 0 });
});
}); });

View File

@ -1,3 +1,5 @@
const supportsESM = parseInt(process.versions.node) >= 12;
module.exports = { module.exports = {
collectCoverageFrom: [ collectCoverageFrom: [
"packages/*/src/**/*.{js,mjs,ts}", "packages/*/src/**/*.{js,mjs,ts}",
@ -40,6 +42,7 @@ module.exports = {
"/test/__data__/", "/test/__data__/",
"<rootDir>/build/", "<rootDir>/build/",
], ],
resolver: supportsESM ? "./test/jestExportsMapResolver.cjs" : null,
// We don't need module name mappers here as depedencies of workspace // We don't need module name mappers here as depedencies of workspace
// package should be declared explicitly in the package.json // package should be declared explicitly in the package.json
// Yarn will generate correct file links so that Jest can resolve correctly // Yarn will generate correct file links so that Jest can resolve correctly

View File

@ -43,6 +43,7 @@
"babel-plugin-transform-charcodes": "^0.2.0", "babel-plugin-transform-charcodes": "^0.2.0",
"chalk": "^2.4.2", "chalk": "^2.4.2",
"charcodes": "^0.2.0", "charcodes": "^0.2.0",
"enhanced-resolve": "^5.8.2",
"eslint": "^7.27.0", "eslint": "^7.27.0",
"eslint-import-resolver-node": "^0.3.4", "eslint-import-resolver-node": "^0.3.4",
"eslint-plugin-flowtype": "^5.7.2", "eslint-plugin-flowtype": "^5.7.2",

View File

@ -127,7 +127,7 @@ function writeHelperFile(
buildHelper(runtimeName, pkgDirname, fullPath, helperName, { esm, corejs }) buildHelper(runtimeName, pkgDirname, fullPath, helperName, { esm, corejs })
); );
return `./${filePath}`; return esm ? `./helpers/esm/${fileName}` : `./helpers/${fileName}`;
} }
function writeHelpers(runtimeName, { corejs } = {}) { function writeHelpers(runtimeName, { corejs } = {}) {
@ -164,12 +164,13 @@ function writeHelpers(runtimeName, { corejs } = {}) {
// - Node.js <13.2.0 will fail resolving the first array entry, and will // - Node.js <13.2.0 will fail resolving the first array entry, and will
// fallback to the second entry (the CJS file) // fallback to the second entry (the CJS file)
// In Babel 8 we can simplify this. // In Babel 8 we can simplify this.
helperSubExports[`./${helperPath}`] = [ helperSubExports[`./${path.posix.join("helpers", helperName)}`] = [
{ node: cjs, import: esm, default: cjs }, { node: cjs, import: esm, default: cjs },
cjs, cjs,
]; ];
// For backward compatibility. We can remove this in Babel 8. // For backward compatibility. We can remove this in Babel 8.
helperSubExports[`./${path.join("helpers", "esm", helperName)}`] = esm; helperSubExports[`./${path.posix.join("helpers", "esm", helperName)}`] =
esm;
} }
writeHelperExports(runtimeName, helperSubExports); writeHelperExports(runtimeName, helperSubExports);

View File

@ -0,0 +1,11 @@
// Temporary workaround for https://github.com/facebook/jest/issues/9771
// Source: https://github.com/facebook/jest/issues/9771#issuecomment-841624042
const resolver = require("enhanced-resolve").create.sync({
conditionNames: ["node", "require", "default"],
extensions: [".js", ".json", ".node", ".ts"],
});
module.exports = function (request, options) {
return resolver(options.basedir, request);
};

View File

@ -5749,6 +5749,7 @@ __metadata:
babel-plugin-transform-charcodes: ^0.2.0 babel-plugin-transform-charcodes: ^0.2.0
chalk: ^2.4.2 chalk: ^2.4.2
charcodes: ^0.2.0 charcodes: ^0.2.0
enhanced-resolve: ^5.8.2
eslint: ^7.27.0 eslint: ^7.27.0
eslint-import-resolver-node: ^0.3.4 eslint-import-resolver-node: ^0.3.4
eslint-plugin-flowtype: ^5.7.2 eslint-plugin-flowtype: ^5.7.2
@ -7445,13 +7446,13 @@ __metadata:
languageName: node languageName: node
linkType: hard linkType: hard
"enhanced-resolve@npm:^5.7.0": "enhanced-resolve@npm:^5.7.0, enhanced-resolve@npm:^5.8.2":
version: 5.7.0 version: 5.8.2
resolution: "enhanced-resolve@npm:5.7.0" resolution: "enhanced-resolve@npm:5.8.2"
dependencies: dependencies:
graceful-fs: ^4.2.4 graceful-fs: ^4.2.4
tapable: ^2.2.0 tapable: ^2.2.0
checksum: 545cfa659e9cdccf1240bccbbd1791db7ec589979d71b35df5aeaf872dd8d13fab379ad73fa960f4cb32963b85492792c0fb0866f484043740014824ae6088b9 checksum: 1af3f6bcb92e849f6c18d44c427cbdaecac4be61023f2008d2ef0f8a48c909bf13afa9a5c04f9d030f27d9a7e27e40c367caa22cd63d2a7eb5fdbab7579d1538
languageName: node languageName: node
linkType: hard linkType: hard