import { FORMAL_NAMES } from "./formal_names" ; const cDigit = "0-9" ; const rLevel = new RegExp ( `^([${ cDigit }]*)` ) ; const cDelim = new RegExp ( "(\\s+)" ) ; const rDelim = new RegExp ( `^([${ cDelim }])` ) ; const cAt = "@" ; const cAlpha = "A-ZÀ-ÿa-z_" ; const cAlphanum = `${ cAlpha }${ cDigit }` ; const cHash = "#" ; const cNonAt = `${ cAlpha }${ cDigit }${ cDelim }${ cHash }` ; const cPointerChar = cNonAt ; const rPointer = new RegExp ( `^${ cAt }([${ cAlphanum }])([${ cPointerChar }])*${ cAt }` ) ; const rTag = new RegExp ( `^(_?[${ cAlphanum }]+)` ) ; const rLineItem = new RegExp ( /^(.*)/ ) ; type TagName = keyof typeof FORMAL_NAMES ; export type Line = { level : number ; tag : TagName ; xref_id ? : string ; pointer ? : string ; value ? : string ; } ; param buf returns export function tokenize ( buf ) { function expect ( re , message ) { const match = buf . match ( re ) ; if ( ! match ) throw new Error ( message ) ; buf = buf . substring ( match [ 0 ] . length ) ; return match [ 1 ] ; } buf = buf . trimStart ( ) ; let xref_id = undefined ; const levelStr = expect ( rLevel , "Expected level" ) ; if ( levelStr . length > 2 || ( levelStr . length === 2 && levelStr [ 0 ] === "0" ) ) { throw new Error ( `Invalid level: ${ levelStr }` ) ; } const level = parseInt ( levelStr ) ; expect ( rDelim , "Expected delimiter after level" ) ; const xref = buf . match ( rPointer ) ; if ( xref ) { xref_id = xref [ 0 ] ; buf = buf . substring ( xref [ 0 ] . length ) ; expect ( rDelim , "Expected delimiter after pointer" ) ; } const tag = expect ( rTag , "Expected tag" ) as TagName ; let line = { level , tag , } ; if ( xref_id ) line . xref_id = xref_id ; const delim = buf . match ( rDelim ) ; if ( delim ) { buf = buf . substring ( delim [ 0 ] . length ) ; const pointer_match = buf . match ( rPointer ) ; const value_match = buf . match ( rLineItem ) ; if ( pointer_match ) { line . pointer = pointer_match [ 0 ] ; } else if ( value_match ) { line . value = value_match [ 1 ] ; } } return line ; }