feat(codegen): Respect label for tokens & support optional token unions
Changes the syntax codegen to respect the label for single tokens, for example, `optional: '?.'` and to create a single field for an optional token union, for example, `operator: ('a' | 'b')?`
MODIFY
xtask/src/codegen/ast.rs
xtask/src/codegen/ast.rs
@@ -144,7 +144,7 @@ fn handle_rule(
if name != "int_number" && name != "string" {
let field = Field::Token {
- name,
+ name: label.cloned().unwrap_or(name),
token_kinds: vec![],
optional,
};
@@ -244,6 +244,11 @@ fn handle_tokens_in_unions(
label: &str,
optional: bool,
) -> bool {
+ let (rule, optional) = match rule {
+ Rule::Opt(rule) => (&**rule, true),
+ _ => (rule, optional),
+ };
+
let rule = match rule {
Rule::Alt(rule) => rule,
_ => return false,

feat(codegen): Respect label for tokens & support optional token unions (#1755)

Changes the syntax codegen to respect the label for single tokens, for example, `optional: '?.'` and to create a single field for an optional token union, for example, `operator: ('a' | 'b')?`