feat: add tokenize option for parse command
MODIFY
internal/core/common/bridges/WorkerBridge.ts
internal/core/common/bridges/WorkerBridge.ts
+import {TokenBase} from "@internal/parser-core";
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
@@ -197,6 +198,14 @@ export default createBridge({
AnyRoot
>(),
+ tokenize: createBridgeEventDeclaration<
+ {
+ ref: FileReference;
+ options: WorkerParseOptions;
+ },
+ TokenBase[]
+ >(),
+
getFileBuffers: createBridgeEventDeclaration<
void,
[AbsoluteFilePath, WorkerBuffer][]

MODIFY
internal/core/common/file-handlers/css.ts
internal/core/common/file-handlers/css.ts
import {PartialExtensionHandler} from "./types";
-import {parseCSS} from "@internal/css-parser";
+import {parseCSS, tokenizeCSS} from "@internal/css-parser";
export const cssHandler: PartialExtensionHandler = {
language: "css",
@@ -23,4 +23,17 @@ export const cssHandler: PartialExtensionHandler = {
astModifiedFromSource: false,
};
},
+
+ async tokenize({integrity, path, file, worker}) {
+ const sourceText = await worker.readFileText(file);
+ const tokens = tokenizeCSS({
+ input: sourceText,
+ integrity,
+ path,
+ });
+ return {
+ sourceText,
+ tokens,
+ };
+ },
};

MODIFY
internal/core/common/file-handlers/html.ts
internal/core/common/file-handlers/html.ts
+import {parseHTML, tokenizeHTML} from "@internal/html-parser";
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
*/
import {PartialExtensionHandler} from "./types";
-import {parseHTML} from "@internal/html-parser";
// These are extensions that be implicitly tried when a file is referenced
// This is mostly for compatibility with Node.js projects. This list should not
@@ -38,4 +38,17 @@ export const htmlHandler: PartialExtensionHandler = {
astModifiedFromSource: false,
};
},
+
+ async tokenize({integrity, path, file, worker}) {
+ const sourceText = await worker.readFileText(file);
+ const tokens = tokenizeHTML({
+ input: sourceText,
+ integrity,
+ path,
+ });
+ return {
+ sourceText,
+ tokens,
+ };
+ },
};

MODIFY
internal/core/common/file-handlers/javascript.ts
internal/core/common/file-handlers/javascript.ts
import {ConstJSProgramSyntax, ConstJSSourceType} from "@internal/ast";
import {PartialExtensionHandler} from "./types";
-import {parseJS} from "@internal/js-parser";
+import {parseJS, tokenizeJS} from "@internal/js-parser";
// These are extensions that be implicitly tried when a file is referenced
// This is mostly for compatibility with Node.js projects. This list should not
@@ -61,6 +61,15 @@ function buildJSHandler(
astModifiedFromSource: false,
};
},
+
+ async tokenize({integrity, path, file, worker}) {
+ const sourceText = await worker.readFileText(file);
+ const tokens = tokenizeJS({input: sourceText, integrity, path});
+ return {
+ sourceText,
+ tokens,
+ };
+ },
};
}

MODIFY
internal/core/common/file-handlers/markdown.ts
internal/core/common/file-handlers/markdown.ts
import {PartialExtensionHandler} from "./types";
-import {parseMarkdown} from "@internal/markdown-parser";
+import {parseMarkdown, tokenizeMarkdown} from "@internal/markdown-parser";
export const markdownHandler: PartialExtensionHandler = {
language: "markdown",
@@ -23,4 +23,13 @@ export const markdownHandler: PartialExtensionHandler = {
astModifiedFromSource: false,
};
},
+
+ async tokenize({integrity, path, file, worker}) {
+ const sourceText = await worker.readFileText(file);
+ const tokens = tokenizeMarkdown({input: sourceText, integrity, path});
+ return {
+ sourceText,
+ tokens,
+ };
+ },
};

MODIFY
internal/core/common/file-handlers/types.ts
internal/core/common/file-handlers/types.ts
+import {TokenBase} from "@internal/parser-core";
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
@@ -50,6 +51,11 @@ export type ExtensionHandlerParseResult<ParseRoot extends AnyRoot = AnyRoot> = {
ast: ParseRoot;
};
+export interface ExtensionHandlerTokenizeResult {
+ sourceText: string;
+ tokens: TokenBase[];
+}
+
export type PartialExtensionHandler<ParseRoot extends AnyRoot = AnyRoot> = {
sourceTypeJS?: ConstJSSourceType;
isAsset?: boolean;
@@ -70,6 +76,10 @@ export type PartialExtensionHandler<ParseRoot extends AnyRoot = AnyRoot> = {
parse: (
opts: ExtensionParseInfo,
) => Promise<ExtensionHandlerParseResult<ParseRoot>>;
+
+ tokenize?: (
+ info: ExtensionParseInfo,
+ ) => Promise<ExtensionHandlerTokenizeResult>;
};
export type ExtensionHandler<ParseRoot extends AnyRoot = AnyRoot> = PartialExtensionHandler<ParseRoot> & {

MODIFY
internal/core/server/ServerRequest.ts
internal/core/server/ServerRequest.ts
+import {TokenBase} from "@internal/parser-core";
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
@@ -1004,6 +1005,19 @@ export default class ServerRequest {
);
}
+ public async requestWorkerTokenize(
+ path: AbsoluteFilePath,
+ opts: WorkerParseOptions,
+ ): Promise<TokenBase[]> {
+ this.checkCancelled();
+
+ return this.wrapRequestDiagnostic(
+ "tokenize",
+ path,
+ (bridge, ref) => bridge.events.tokenize.call({ref, options: opts}),
+ );
+ }
+
public async requestWorkerUpdateInlineSnapshots(
path: AbsoluteFilePath,
updates: InlineSnapshotUpdate[],

MODIFY
internal/core/server/commands/parse.ts
internal/core/server/commands/parse.ts
@@ -18,6 +18,7 @@ type Flags = {
allowDiagnostics: boolean;
compact: boolean;
sourceType: undefined | ConstJSSourceType;
+ tokenize: boolean;
};
export default createServerCommand({
@@ -31,12 +32,23 @@ export default createServerCommand({
allowDiagnostics: c.get("allowDiagnostics").asBoolean(false),
compact: c.get("compact").asBoolean(true),
sourceType: c.get("sourceType").asStringSetOrVoid(["module", "script"]),
+ tokenize: c.get("tokenize").asBoolean(false),
};
},
async callback(req: ServerRequest, flags: Flags): Promise<void> {
const {reporter} = req;
const filename = await req.resolveEntryAssertPathArg(0);
+ if (flags.tokenize) {
+ const tokens = await req.requestWorkerTokenize(
+ filename,
+ {
+ sourceTypeJS: flags.sourceType,
+ },
+ );
+
+ reporter.inspect(tokens);
+ } else {
let ast = await req.requestWorkerParse(
filename,
{
@@ -50,5 +62,6 @@ export default createServerCommand({
}
reporter.inspect(ast);
+ }
},
});

MODIFY
internal/core/worker/Worker.ts
internal/core/worker/Worker.ts
@@ -17,6 +17,7 @@ import {
WorkerPrefetchedModuleSignatures,
WorkerProject,
WorkerProjects,
+ WorkerTokenizeResult,
} from "./types";
import WorkerBridge from "../common/bridges/WorkerBridge";
import {ConstJSSourceType, JSRoot} from "@internal/ast";
@@ -39,6 +40,10 @@ import {
import {FSReadStream, FSStats, createFakeStats} from "@internal/fs";
import {FileReference} from "../common/types/files";
import {getFileHandlerFromPathAssert} from "../common/file-handlers/index";
+import {
+ ExtensionHandler,
+ ExtensionParseInfo,
+} from "./../common/file-handlers/types";
import WorkerAPI from "./WorkerAPI";
import {applyWorkerBufferPatch} from "./utils/applyWorkerBufferPatch";
import VirtualModules from "../common/VirtualModules";
@@ -281,6 +286,10 @@ export default class Worker {
return this.api.parse(payload.ref, payload.options) as RSERObject;
});
+ bridge.events.tokenize.subscribe((payload) => {
+ return this.api.tokenize(payload.ref, payload.options);
+ });
+
bridge.events.lint.subscribe((payload) => {
return this.api.lint(payload.ref, payload.options, payload.parseOptions);
});
@@ -558,7 +567,6 @@ export default class Worker {
options: WorkerParseOptions,
): Promise<WorkerParseResult> {
const path = ref.real;
- const {uid} = ref;
const project = this.getProject(ref);
// Fetch and validate extension handler
@@ -567,20 +575,7 @@ export default class Worker {
throw new Error(`We don't know how to parse ${path}`);
}
- // Get source type
- let sourceTypeJS: undefined | ConstJSSourceType;
- if (options.sourceTypeJS !== undefined) {
- sourceTypeJS = options.sourceTypeJS;
- } else if (handler.sourceTypeJS !== undefined) {
- sourceTypeJS = handler.sourceTypeJS;
- } else {
- sourceTypeJS = "script";
-
- const manifest = this.getPartialManifest(ref);
- if (manifest?.type === "module") {
- sourceTypeJS = "module";
- }
- }
+ const parseInfo = await this.getParseInfo(project, ref, handler, options);
const cacheEnabled = options.cache !== false;
@@ -595,7 +590,7 @@ export default class Worker {
if (
cachedResult.ast.type === "JSRoot" &&
- cachedResult.ast.sourceType !== sourceTypeJS
+ cachedResult.ast.sourceType !== parseInfo.sourceTypeJS
) {
useCached = false;
}
@@ -615,27 +610,9 @@ export default class Worker {
this.logger.info(markup`Parsing: ${path}`);
- const cacheFile = await this.cache.getFile(ref);
- const integrity = await this.cache.getIntegrity(ref);
- const {mtimeNs} = await cacheFile.getStats();
-
- const manifest = this.getPartialManifest(ref);
- let manifestPath: undefined | string;
- if (manifest !== undefined) {
- manifestPath = manifest.path.join();
- }
-
- const {sourceText, astModifiedFromSource, ast} = await handler.parse({
- sourceTypeJS,
- path: uid,
- manifestPath,
- integrity,
- mtimeNs,
- file: ref,
- worker: this,
- project,
- parseOptions: options,
- });
+ const {sourceText, astModifiedFromSource, ast} = await handler.parse(
+ parseInfo,
+ );
// If the AST is corrupt then we don't under any circumstance allow it
if (ast.corrupt && !options.allowCorrupt) {
@@ -660,8 +637,8 @@ export default class Worker {
project,
path,
astModifiedFromSource,
- integrity,
- mtimeNs,
+ integrity: parseInfo.integrity,
+ mtimeNs: parseInfo.mtimeNs,
};
if (cacheEnabled) {
@@ -671,6 +648,78 @@ export default class Worker {
return res;
}
+ public async tokenize(
+ ref: FileReference,
+ options: WorkerParseOptions,
+ ): Promise<WorkerTokenizeResult> {
+ const path = ref.real;
+ const project = this.getProject(ref);
+
+ // Fetch and validate extension handler
+ const {handler} = getFileHandlerFromPathAssert(ref.real, project.config);
+ if (handler.tokenize === undefined) {
+ throw new Error(`We don't know how to tokenize ${path}`);
+ }
+
+ const parseInfo = await this.getParseInfo(project, ref, handler, options);
+
+ this.logger.info(markup`Tokenizing: ${path}`);
+
+ const {sourceText, tokens} = await handler.tokenize(parseInfo);
+
+ const res: WorkerTokenizeResult = {
+ tokens,
+ sourceText,
+ project,
+ path,
+ integrity: parseInfo.integrity,
+ mtimeNs: parseInfo.mtimeNs,
+ };
+
+ return res;
+ }
+
+ private async getParseInfo(
+ project: WorkerProject,
+ ref: FileReference,
+ handler: ExtensionHandler,
+ options: WorkerParseOptions,
+ ): Promise<ExtensionParseInfo> {
+ const sourceTypeJS =
+ options.sourceTypeJS || this.getSourceTypeJs(ref, handler);
+
+ const cacheFile = await this.cache.getFile(ref);
+ const integrity = await this.cache.getIntegrity(ref);
+ const {mtimeNs} = await cacheFile.getStats();
+
+ const manifest = this.getPartialManifest(ref);
+ const manifestPath = manifest?.path.join();
+
+ return {
+ sourceTypeJS,
+ path: ref.uid,
+ manifestPath,
+ integrity,
+ mtimeNs,
+ file: ref,
+ worker: this,
+ project,
+ parseOptions: options,
+ };
+ }
+
+ private getSourceTypeJs(
+ fileRef: FileReference,
+ handler: ExtensionHandler,
+ ): ConstJSSourceType {
+ if (handler.sourceTypeJS !== undefined) {
+ return handler.sourceTypeJS;
+ } else {
+ const manifest = this.getPartialManifest(fileRef);
+ return manifest?.type === "module" ? "module" : "script";
+ }
+ }
+
private async evict(
{real, uid}: {
real: AbsoluteFilePath;

MODIFY
internal/core/worker/WorkerAPI.ts
internal/core/worker/WorkerAPI.ts
+import {TokenBase} from "@internal/parser-core";
/**
* Copyright (c) Facebook, Inc. and its affiliates.
*
@@ -430,6 +431,22 @@ export default class WorkerAPI {
return this.interceptDiagnostics(ast, {astModifiedFromSource});
}
+ public async tokenize(
+ ref: FileReference,
+ opts: WorkerParseOptions,
+ ): Promise<TokenBase[]> {
+ const {tokens} = await this.worker.tokenize(
+ ref,
+ {
+ ...opts,
+ sourceTypeJS: opts.sourceTypeJS,
+ cache: false,
+ },
+ );
+
+ return tokens;
+ }
+
public async format(
ref: FileReference,
options: WorkerFormatOptions,

MODIFY
internal/core/worker/types.ts
internal/core/worker/types.ts
+import {TokenBase} from "@internal/parser-core";
import {AnyRoot, ConstJSSourceType} from "@internal/ast";
import {Manifest} from "@internal/codec-js-manifest";
import {
@@ -41,6 +42,15 @@ export type WorkerParseResult = {
astModifiedFromSource: boolean;
};
+export type WorkerTokenizeResult = {
+ tokens: TokenBase[];
+ integrity: undefined | DiagnosticIntegrity;
+ mtimeNs: bigint;
+ project: WorkerProject;
+ path: AbsoluteFilePath;
+ sourceText: string;
+};
+
export type WorkerBuffer = {
content: string;
mtimeNs: bigint;

feat: add tokenize option for parse command (#1393)