import * as Tokenizr from 'tokenizr' ; import { isNullOrUndefined } from 'util' ; /**
 * Created by martin on 14.05.2017.
 * A tokenizer for normalized messages.
 */ export const TEXT = 'TEXT' ; export const START_TAG = 'START_TAG' ; export const END_TAG = 'END_TAG' ; export const EMPTY_TAG = 'EMPTY_TAG' ; export const PLACEHOLDER = 'PLACEHOLDER' ; export const ICU_MESSAGE_REF = 'ICU_MESSAGE_REF' ; export const ICU_MESSAGE = 'ICU_MESSAGE' ; export interface Token { type : string ; value : any ; } export class ParsedMesageTokenizer { private getLexer ( ) { const lexer = new Tokenizr ( ) ; let plaintext = '' ; lexer . before ( ( ctx , match , rule ) => { if ( rule . name !== TEXT && plaintext !== '' ) { ctx . accept ( TEXT , { text : plaintext } ) ; plaintext = '' ; } } ) ; lexer . finish ( ( ctx ) => { if ( plaintext !== '' ) { ctx . accept ( TEXT , { text : plaintext } ) ; } } ) ; lexer . rule ( /<(br|hr|img|area|link|wbr)( id="([0-9])*")?\>/ , ( ctx , match ) => { const idcount = isNullOrUndefined ( match [ 3 ] ) ? 0 : parseInt ( match [ 3 ] , 10 ) ; ctx . accept ( EMPTY_TAG , { name : match [ 1 ] , idcounter : idcount } ) ; } , EMPTY_TAG ) ; lexer . rule ( /<([a-zA-Z][a-zA-Z-0-9]*)( id="([0-9]*)")?>/ , ( ctx , match ) => { const idcount = isNullOrUndefined ( match [ 3 ] ) ? 0 : parseInt ( match [ 3 ] , 10 ) ; ctx . accept ( START_TAG , { name : match [ 1 ] , idcounter : idcount } ) ; } , START_TAG ) ; lexer . rule ( /<\/([a-zA-Z][a-zA-Z-0-9]*)>/ , ( ctx , match ) => { ctx . accept ( END_TAG , { name : match [ 1 ] } ) ; } , END_TAG ) ; lexer . rule ( /{{([0-9]+)}}/ , ( ctx , match ) => { ctx . accept ( PLACEHOLDER , { idcounter : parseInt ( match [ 1 ] , 10 ) } ) ; } , PLACEHOLDER ) ; lexer . rule ( /<ICU-Message-Ref_([0-9]+)\/>/ , ( ctx , match ) => { ctx . accept ( ICU_MESSAGE_REF , { idcounter : parseInt ( match [ 1 ] , 10 ) } ) ; } , ICU_MESSAGE_REF ) ; lexer . rule ( /<ICU-Message\/>/ , ( ctx , match ) => { ctx . accept ( ICU_MESSAGE , { message : match [ 0 ] } ) ; } , ICU_MESSAGE ) ; lexer . rule ( /./ , ( ctx , match ) => { plaintext += match [ 0 ] ; ctx . ignore ( ) ; } , TEXT ) ; lexer . rule ( /[\t\r\n]+/ , ( ctx , match ) => { plaintext += match [ 0 ] ; ctx . ignore ( ) ; } , TEXT ) ; return lexer ; } tokenize ( normalizedMessage ) { const lexer = this . getLexer ( ) ; lexer . reset ( ) ; lexer . input ( normalizedMessage ) ; return lexer . tokens ( ) ; } }