#!/usr/bin/env python
# -*- encoding: utf-8 -*-

# Copyright: This module has been placed in the public domain.

import pp.preprocessor as pp
import easytorq

import sys

def getname():
    return "cobol"

def getversion():
    return (2, 0, 0, 0)

class CobolPreprocessor(pp.Base):
    def __init__(self):
        self.pat = None
        self.fmt = None
    
    def getoptiondescription(self):
        return "no options."
    
    def tonormalizedoptionstring(self, optionStr):
        if optionStr not in ( None, '', 'default' ):
            raise pp.InvalidOptionError, "invalid option: " + optionStr
        return 'default'
    
    def setoptions(self, optionStr):
        if optionStr not in ( None, '', 'default'):
            raise pp.InvalidOptionError, "invalid option: " + optionStr
        patternStr = """TEXT scan= +(xcep(eof) any) | insert(eol) eof; // ensure a line terminates by eol

TEXT scan= null <- (eol "-" +" "); // Continuation line

// remove line number and comment line
TEXT match=
    (*(" " | "&t;") "$" *(" " | "&t;") ("SET" | "set") *(" " | "&t;") ("SOURCEFORMAT" | "sourceformat") *(" " | "&t;") "&quot;" ("FREE" | "free") "&quot;" *(" " | "&t;") eol
        *(
            (comment <- ("*" | "/")  *(xcep(eol) any)) eol
            | *(xcep(eol) any) eol
        )
        eof
    )
    | (
        *(
            (null <- ("&(0-9);" | " ")("&(0-9);" | " ")("&(0-9);" | " ")("&(0-9);" | " ")("&(0-9);" | " ")("&(0-9);" | " ")) 
            (
                (comment <- ("*" | "/") *(xcep(eol) any)) eol
// if you need to recognize column 73 to end-of-line as comment
//                | raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw
//                    raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw
//                    raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw raw
//                    raw raw raw raw raw raw (comment <- *(xcep(eol) any)) eol
                | *(xcep(eol) any) eol
            )
            | (
                (comment <- ("*" | "/") *(xcep(eol) any)) eol
                | *(xcep(eol) any) eol
            )
        ) 
        eof
    );

TEXT scan= 
    (r_pic <- ("picture" | "PICTURE" | "pic" | "PIC"))
    *(
         +(null <- " " | "&t;")
        xcep(eof | "." | "value" | "VALUE")
        (
            preq("X") (
                (x100 <- "X(" *"0" "100)") | (x99 <- "X(" *"0" "99)") | (x98 <- "X(" *"0" "98)") | (x97 <- "X(" *"0" "97)") | (x96 <- "X(" *"0" "96)") | (x95 <- "X(" *"0" "95)") | (x94 <- "X(" *"0" "94)") | (x93 <- "X(" *"0" "93)") | (x92 <- "X(" *"0" "92)") | (x91 <- "X(" *"0" "91)")
                | (x90 <- "X(" *"0" "90)") | (x89 <- "X(" *"0" "89)") | (x88 <- "X(" *"0" "88)") | (x87 <- "X(" *"0" "87)") | (x86 <- "X(" *"0" "86)") | (x85 <- "X(" *"0" "85)") | (x84 <- "X(" *"0" "84)") | (x83 <- "X(" *"0" "83)") | (x82 <- "X(" *"0" "82)") | (x81 <- "X(" *"0" "81)")
                | (x80 <- "X(" *"0" "80)") | (x79 <- "X(" *"0" "79)") | (x78 <- "X(" *"0" "78)") | (x77 <- "X(" *"0" "77)") | (x76 <- "X(" *"0" "76)") | (x75 <- "X(" *"0" "75)") | (x74 <- "X(" *"0" "74)") | (x73 <- "X(" *"0" "73)") | (x72 <- "X(" *"0" "72)") | (x71 <- "X(" *"0" "71)")
                | (x70 <- "X(" *"0" "70)") | (x69 <- "X(" *"0" "69)") | (x68 <- "X(" *"0" "68)") | (x67 <- "X(" *"0" "67)") | (x66 <- "X(" *"0" "66)") | (x65 <- "X(" *"0" "65)") | (x64 <- "X(" *"0" "64)") | (x63 <- "X(" *"0" "63)") | (x62 <- "X(" *"0" "62)") | (x61 <- "X(" *"0" "61)")
                | (x60 <- "X(" *"0" "60)") | (x59 <- "X(" *"0" "59)") | (x58 <- "X(" *"0" "58)") | (x57 <- "X(" *"0" "57)") | (x56 <- "X(" *"0" "56)") | (x55 <- "X(" *"0" "55)") | (x54 <- "X(" *"0" "54)") | (x53 <- "X(" *"0" "53)") | (x52 <- "X(" *"0" "52)") | (x51 <- "X(" *"0" "51)")
                | (x50 <- "X(" *"0" "50)") | (x49 <- "X(" *"0" "49)") | (x48 <- "X(" *"0" "48)") | (x47 <- "X(" *"0" "47)") | (x46 <- "X(" *"0" "46)") | (x45 <- "X(" *"0" "45)") | (x44 <- "X(" *"0" "44)") | (x43 <- "X(" *"0" "43)") | (x42 <- "X(" *"0" "42)") | (x41 <- "X(" *"0" "41)")
                | (x40 <- "X(" *"0" "40)") | (x39 <- "X(" *"0" "39)") | (x38 <- "X(" *"0" "38)") | (x37 <- "X(" *"0" "37)") | (x36 <- "X(" *"0" "36)") | (x35 <- "X(" *"0" "35)") | (x34 <- "X(" *"0" "34)") | (x33 <- "X(" *"0" "33)") | (x32 <- "X(" *"0" "32)") | (x31 <- "X(" *"0" "31)")
                | (x30 <- "X(" *"0" "30)") | (x29 <- "X(" *"0" "29)") | (x28 <- "X(" *"0" "28)") | (x27 <- "X(" *"0" "27)") | (x26 <- "X(" *"0" "26)") | (x25 <- "X(" *"0" "25)") | (x24 <- "X(" *"0" "24)") | (x23 <- "X(" *"0" "23)") | (x22 <- "X(" *"0" "22)") | (x21 <- "X(" *"0" "21)")
                | (x20 <- "XXXXXXXXXXXXXXXXXXXX" | "X(" *"0" "20)")
                | (x19 <- "XXXXXXXXXXXXXXXXXXX" | "X(" *"0" "19)")
                | (x18 <- "XXXXXXXXXXXXXXXXXX" | "X(" *"0" "18)")
                | (x17 <- "XXXXXXXXXXXXXXXXX" | "X(" *"0" "17)")
                | (x16 <- "XXXXXXXXXXXXXXXX" | "X(" *"0" "16)")
                | (x15 <- "XXXXXXXXXXXXXXX" | "X(" *"0" "15)")
                | (x14 <- "XXXXXXXXXXXXXX" | "X(" *"0" "14)")
                | (x13 <- "XXXXXXXXXXXXX" | "X(" *"0" "13)")
                | (x12 <- "XXXXXXXXXXXX" | "X(" *"0" "12)")
                | (x11 <- "XXXXXXXXXXX" | "X(" *"0" "11)")
                | (x10 <- "XXXXXXXXXX" | "X(" *"0" "10)")
                | (x9 <- "XXXXXXXXX" | "X(" *"0" "9)")
                | (x8 <- "XXXXXXXX" | "X(" *"0" "8)")
                | (x7 <- "XXXXXXX" | "X(" *"0" "7)")
                | (x6 <- "XXXXXX" | "X(" *"0" "6)")
                | (x5 <- "XXXXX" | "X(" *"0" "5)")
                | (x4 <- "XXXX" | "X(" *"0" "4)")
                | (x3 <- "XXX" | "X(" *"0" "3)")
                | (x2 <- "XX" | "X(" *"0" "2)")
                | (x1 <- "X" ?("(" *"0" "1)"))
            )
            | preq("x") (
                (x100 <- "x(" *"0" "100)") | (x99 <- "x(" *"0" "99)") | (x98 <- "x(" *"0" "98)") | (x97 <- "x(" *"0" "97)") | (x96 <- "x(" *"0" "96)") | (x95 <- "x(" *"0" "95)") | (x94 <- "x(" *"0" "94)") | (x93 <- "x(" *"0" "93)") | (x92 <- "x(" *"0" "92)") | (x91 <- "x(" *"0" "91)")
                | (x90 <- "x(" *"0" "90)") | (x89 <- "x(" *"0" "89)") | (x88 <- "x(" *"0" "88)") | (x87 <- "x(" *"0" "87)") | (x86 <- "x(" *"0" "86)") | (x85 <- "x(" *"0" "85)") | (x84 <- "x(" *"0" "84)") | (x83 <- "x(" *"0" "83)") | (x82 <- "x(" *"0" "82)") | (x81 <- "x(" *"0" "81)")
                | (x80 <- "x(" *"0" "80)") | (x79 <- "x(" *"0" "79)") | (x78 <- "x(" *"0" "78)") | (x77 <- "x(" *"0" "77)") | (x76 <- "x(" *"0" "76)") | (x75 <- "x(" *"0" "75)") | (x74 <- "x(" *"0" "74)") | (x73 <- "x(" *"0" "73)") | (x72 <- "x(" *"0" "72)") | (x71 <- "x(" *"0" "71)")
                | (x70 <- "x(" *"0" "70)") | (x69 <- "x(" *"0" "69)") | (x68 <- "x(" *"0" "68)") | (x67 <- "x(" *"0" "67)") | (x66 <- "x(" *"0" "66)") | (x65 <- "x(" *"0" "65)") | (x64 <- "x(" *"0" "64)") | (x63 <- "x(" *"0" "63)") | (x62 <- "x(" *"0" "62)") | (x61 <- "x(" *"0" "61)")
                | (x60 <- "x(" *"0" "60)") | (x59 <- "x(" *"0" "59)") | (x58 <- "x(" *"0" "58)") | (x57 <- "x(" *"0" "57)") | (x56 <- "x(" *"0" "56)") | (x55 <- "x(" *"0" "55)") | (x54 <- "x(" *"0" "54)") | (x53 <- "x(" *"0" "53)") | (x52 <- "x(" *"0" "52)") | (x51 <- "x(" *"0" "51)")
                | (x50 <- "x(" *"0" "50)") | (x49 <- "x(" *"0" "49)") | (x48 <- "x(" *"0" "48)") | (x47 <- "x(" *"0" "47)") | (x46 <- "x(" *"0" "46)") | (x45 <- "x(" *"0" "45)") | (x44 <- "x(" *"0" "44)") | (x43 <- "x(" *"0" "43)") | (x42 <- "x(" *"0" "42)") | (x41 <- "x(" *"0" "41)")
                | (x40 <- "x(" *"0" "40)") | (x39 <- "x(" *"0" "39)") | (x38 <- "x(" *"0" "38)") | (x37 <- "x(" *"0" "37)") | (x36 <- "x(" *"0" "36)") | (x35 <- "x(" *"0" "35)") | (x34 <- "x(" *"0" "34)") | (x33 <- "x(" *"0" "33)") | (x32 <- "x(" *"0" "32)") | (x31 <- "x(" *"0" "31)")
                | (x30 <- "x(" *"0" "30)") | (x29 <- "x(" *"0" "29)") | (x28 <- "x(" *"0" "28)") | (x27 <- "x(" *"0" "27)") | (x26 <- "x(" *"0" "26)") | (x25 <- "x(" *"0" "25)") | (x24 <- "x(" *"0" "24)") | (x23 <- "x(" *"0" "23)") | (x22 <- "x(" *"0" "22)") | (x21 <- "x(" *"0" "21)")
                | (x20 <- "xxxxxxxxxxxxxxxxxxxx" | "x(" *"0" "20)")
                | (x19 <- "xxxxxxxxxxxxxxxxxxx" | "x(" *"0" "19)")
                | (x18 <- "xxxxxxxxxxxxxxxxxx" | "x(" *"0" "18)")
                | (x17 <- "xxxxxxxxxxxxxxxxx" | "x(" *"0" "17)")
                | (x16 <- "xxxxxxxxxxxxxxxx" | "x(" *"0" "16)")
                | (x15 <- "xxxxxxxxxxxxxxx" | "x(" *"0" "15)")
                | (x14 <- "xxxxxxxxxxxxxx" | "x(" *"0" "14)")
                | (x13 <- "xxxxxxxxxxxxx" | "x(" *"0" "13)")
                | (x12 <- "xxxxxxxxxxxx" | "x(" *"0" "12)")
                | (x11 <- "xxxxxxxxxxx" | "x(" *"0" "11)")
                | (x10 <- "xxxxxxxxxx" | "x(" *"0" "10)")
                | (x9 <- "xxxxxxxxx" | "x(" *"0" "9)")
                | (x8 <- "xxxxxxxx" | "x(" *"0" "8)")
                | (x7 <- "xxxxxxx" | "x(" *"0" "7)")
                | (x6 <- "xxxxxx" | "x(" *"0" "6)")
                | (x5 <- "xxxxx" | "x(" *"0" "5)")
                | (x4 <- "xxxx" | "x(" *"0" "4)")
                | (x3 <- "xxx" | "x(" *"0" "3)")
                | (x2 <- "xx" | "x(" *"0" "2)")
                | (x1 <- "x" ?("(" *"0" "1)"))
            )
            | +(
                (s <- "s" | "S")
                | (v <- "v" | "V")
                | (p <- "p" | "P")
                preq("Z") (
                    (z20 <- "ZZZZZZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZZZZZZ9" | "Z(" *"0" "20)" | "Z(" *"0" "19)9" ?("(" *"0" "1)"))
                    | (z19 <- "ZZZZZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZZZZZ9" | "Z(" *"0" "19)" | "Z(" *"0" "18)9" ?("(" *"0" "1)"))
                    | (z18 <- "ZZZZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZZZZ9" | "Z(" *"0" "18)" | "Z(" *"0" "17)9" ?("(" *"0" "1)"))
                    | (z17 <- "ZZZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZZZ9" | "Z(" *"0" "17)" | "Z(" *"0" "16)9" ?("(" *"0" "1)"))
                    | (z16 <- "ZZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZZ9" | "Z(" *"0" "16)" | "Z(" *"0" "15)9" ?("(" *"0" "1)"))
                    | (z15 <- "ZZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZZ9" | "Z(" *"0" "15)" | "Z(" *"0" "14)9" ?("(" *"0" "1)"))
                    | (z14 <- "ZZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZZ9" | "Z(" *"0" "14)" | "Z(" *"0" "13)9" ?("(" *"0" "1)"))
                    | (z13 <- "ZZZZZZZZZZZZZ" | "ZZZZZZZZZZZZ9" | "Z(" *"0" "13)" | "Z(" *"0" "12)9" ?("(" *"0" "1)"))
                    | (z12 <- "ZZZZZZZZZZZZ" | "ZZZZZZZZZZZ9" | "Z(" *"0" "12)" | "Z(" *"0" "11)9" ?("(" *"0" "1)"))
                    | (z11 <- "ZZZZZZZZZZZ" | "ZZZZZZZZZZ9" | "Z(" *"0" "11)" | "Z(" *"0" "10)9" ?("(" *"0" "1)"))
                    | (z10 <- "ZZZZZZZZZZ" | "ZZZZZZZZZ9" | "Z(" *"0" "10)" | "Z(" *"0" "9)9" ?("(" *"0" "1)"))
                    | (z9 <- "ZZZZZZZZZ" | "ZZZZZZZZ9" | "Z(" *"0" "9)" | "Z(" *"0" "8)9" ?("(" *"0" "1)"))
                    | (z8 <- "ZZZZZZZZ" | "ZZZZZZZ9" | "Z(" *"0" "8)" | "Z(" *"0" "7)9" ?("(" *"0" "1)"))
                    | (z7 <- "ZZZZZZZ" | "ZZZZZZ9" | "Z(" *"0" "7)" | "Z(" *"0" "6)9" ?("(" *"0" "1)"))
                    | (z6 <- "ZZZZZZ" | "ZZZZZ9" | "Z(" *"0" "6)" | "Z(" *"0" "5)9" ?("(" *"0" "1)"))
                    | (z5 <- "ZZZZZ" | "ZZZZ9" | "Z(" *"0" "5)" | "Z(" *"0" "4)9" ?("(" *"0" "1)"))
                    | (z4 <- "ZZZZ" | "ZZZ9" | "Z(" *"0" "4)" | "Z(" *"0" "3)9" ?("(" *"0" "1)"))
                    | (z3 <- "ZZZ" | "ZZ9" | "Z(" *"0" "3)" | "Z(" *"0" "2)9" ?("(" *"0" "1)"))
                    | (z2 <- "ZZ" | "Z9" | "Z(" *"0" "2)" | "Z(" *"0" "1)9" ?("(" *"0" "1)"))
                    | (z1 <- "Z" ?("(" *"0" "1)"))
                )
                | preq("z") (
                    (z20 <- "zzzzzzzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzzzzzzz9" | "z(" *"0" "20)" | "z(" *"0" "19)9" ?("(" *"0" "1)"))
                    | (z19 <- "zzzzzzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzzzzzz9" | "z(" *"0" "19)" | "z(" *"0" "18)9" ?("(" *"0" "1)"))
                    | (z18 <- "zzzzzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzzzzz9" | "z(" *"0" "18)" | "z(" *"0" "17)9" ?("(" *"0" "1)"))
                    | (z17 <- "zzzzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzzzz9" | "z(" *"0" "17)" | "z(" *"0" "16)9" ?("(" *"0" "1)"))
                    | (z16 <- "zzzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzzz9" | "z(" *"0" "16)" | "z(" *"0" "15)9" ?("(" *"0" "1)"))
                    | (z15 <- "zzzzzzzzzzzzzzz" | "zzzzzzzzzzzzzz9" | "z(" *"0" "15)" | "z(" *"0" "14)9" ?("(" *"0" "1)"))
                    | (z14 <- "zzzzzzzzzzzzzz" | "zzzzzzzzzzzzz9" | "z(" *"0" "14)" | "z(" *"0" "13)9" ?("(" *"0" "1)"))
                    | (z13 <- "zzzzzzzzzzzzz" | "zzzzzzzzzzzz9" | "z(" *"0" "13)" | "z(" *"0" "12)9" ?("(" *"0" "1)"))
                    | (z12 <- "zzzzzzzzzzzz" | "zzzzzzzzzzz9" | "z(" *"0" "12)" | "z(" *"0" "11)9" ?("(" *"0" "1)"))
                    | (z11 <- "zzzzzzzzzzz" | "zzzzzzzzzz9" | "z(" *"0" "11)" | "z(" *"0" "10)9" ?("(" *"0" "1)"))
                    | (z10 <- "zzzzzzzzzz" | "zzzzzzzzz9" | "z(" *"0" "10)" | "z(" *"0" "9)9" ?("(" *"0" "1)"))
                    | (z9 <- "zzzzzzzzz" | "zzzzzzzz9" | "z(" *"0" "9)" | "z(" *"0" "8)9" ?("(" *"0" "1)"))
                    | (z8 <- "zzzzzzzz" | "zzzzzzz9" | "z(" *"0" "8)" | "z(" *"0" "7)9" ?("(" *"0" "1)"))
                    | (z7 <- "zzzzzzz" | "zzzzzz9" | "z(" *"0" "7)" | "z(" *"0" "6)9" ?("(" *"0" "1)"))
                    | (z6 <- "zzzzzz" | "zzzzz9" | "z(" *"0" "6)" | "z(" *"0" "5)9" ?("(" *"0" "1)"))
                    | (z5 <- "zzzzz" | "zzzz9" | "z(" *"0" "5)" | "z(" *"0" "4)9" ?("(" *"0" "1)"))
                    | (z4 <- "zzzz" | "zzz9" | "z(" *"0" "4)" | "z(" *"0" "3)9" ?("(" *"0" "1)"))
                    | (z3 <- "zzz" | "zz9" | "z(" *"0" "3)" | "z(" *"0" "2)9" ?("(" *"0" "1)"))
                    | (z2 <- "zz" | "z9" | "z(" *"0" "2)" | "z(" *"0" "1)9" ?("(" *"0" "1)"))
                    | (z1 <- "z" ?("(" *"0" "1)"))
                )
                | preq("9") (
                    (z20 <- "99999999999999999999" | "9(" *"0" "20)")
                    | (z19 <- "9999999999999999999" | "9(" *"0" "19)")
                    | (z18 <- "999999999999999999" | "9(" *"0" "18)")
                    | (z17 <- "99999999999999999" | "9(" *"0" "17)")
                    | (z16 <- "9999999999999999" | "9(" *"0" "16)")
                    | (z15 <- "999999999999999" | "9(" *"0" "15)")
                    | (z14 <- "99999999999999" | "9(" *"0" "14)")
                    | (z13 <- "9999999999999" | "9(" *"0" "13)")
                    | (z12 <- "999999999999" | "9(" *"0" "12)")
                    | (z11 <- "99999999999" | "9(" *"0" "11)")
                    | (z10 <- "9999999999" | "9(" *"0" "10)")
                    | (z9 <- "999999999" | "9(" *"0" "9)")
                    | (z8 <- "99999999" | "9(" *"0" "8)")
                    | (z7 <- "9999999" | "9(" *"0" "7)")
                    | (z6 <- "999999" | "9(" *"0" "6)")
                    | (z5 <- "99999" | "9(" *"0" "5)")
                    | (z4 <- "9999" | "9(" *"0" "4)")
                    | (z3 <- "999" | "9(" *"0" "3)")
                    | (z2 <- "99" | "9(" *"0" "2)")
                    | (z1 <- "9" ?("(" *"0" "1)"))
                )
            )
        )
    )
    | (l_string <- ?("X" | "NC" | "NX" | "N") ("&quot;" *(xcep("&quot;" | eol) any) "&quot;" | "&squot;" *(xcep("&squot;" | eol) any) "&squot;") 
        *("-" eol ("&quot;" *(xcep("&quot;" | eol) any) "&quot;" | "&squot;" *(xcep("&squot;" | eol) any) "&squot;")))
    | (l_float <- +"&(0-9);" "." *"&(0-9);" ?(("e" | "E") ?("-" | "+") +"&(0-9);"))
    | (l_float <- +"&(0-9);" ("e" | "E") ?("-" | "+") +"&(0-9);")
    | (l_int <- +"&(0-9);" xcep(
            "-" | "&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@"
    ))
    | (l_int <- "B" "&quot;" +("0" | "1") "&quot;" | "&squot;" +("0" | "1") "&squot;") 
    | (l_int <- "BX" "&quot;" +("&(0-9);" | "&(A-F);" | "&(a-f);") "&quot;" | "&squot;" +("&(0-9);" | "&(A-F);" | "&(a-f);") "&squot;") 
    | preq("&(a-z);" | "&(A-Z);") (
        (r_zero <- "zeroes" | "ZEROES" | "zeros" | "ZEROS" | "zero" | "ZERO")
        | (r_zero_fill <- "zero-fill" | "ZERO-FILL")
        | (r_write_only <- "write-only" | "WRITE-ONLY")
        | (r_write <- "write" | "WRITE")
        | (r_working_storage <- "working-storage" | "WORKING-STORAGE")
        | (r_words <- "words" | "WORDS")
        | (r_within <- "within" | "WITHIN")
        | (r_with <- "with" | "WITH")
        | (r_when_compiled <- "when-compiled" | "WHEN-COMPILED")
        | (r_when <- "when" | "WHEN")
        | (r_wait <- "wait" | "WAIT")
        | (r_varying <- "varying" | "VARYING")
        | (r_value <- "values" | "VALUES" 
            | "value" | "VALUE")
        | (r_validate <- "validate" | "VALIDATE")
        | (r_valid <- "valid" | "VALID")
        | (r_using <- "using" | "USING")
        | (r_use <- "use" | "USE")
        | (r_usage_mode <- "usage-mode" | "USAGE-MODE")
        | (r_usage <- "usage" | "USAGE")
        | (r_upon <- "upon" | "UPON")
        | (r_update <- "update" | "UPDATE")
        | (r_up <- "up" | "UP")
        | (r_until <- "until" | "UNTIL")
        | (r_unstring <- "unstring" | "UNSTRING")
        | (r_unlock <- "unlock" | "UNLOCK")
        | (r_universal <- "universal" | "UNIVERSAL")
        | (r_unit <- "unit" | "UNIT")
        | (r_unequal <- "unequal" | "UNEQUAL")
        | (r_underline <- "underline" | "UNDERLINE")
        | (r_typedef <- "typedef" | "TYPEDEF")
        | (r_type <- "type" | "TYPE")
        | (r_true <- "true" | "TRUE")
        | (r_transceive <- "transceive" | "TRANSCEIVE")
        | (r_transaction <- "transaction" | "TRANSACTION")
        | (r_trailing_sign <- "trailing-sign" | "TRAILING-SIGN")
        | (r_trailing <- "trailing" | "TRAILING")
        | (r_track_overflow <- "track-overflow" | "TRACK-OVERFLOW")
        | (r_track_limit <- "track-limit" | "TRACK-LIMIT")
        | (r_track_area <- "track-area" | "TRACK-AREA")
        | (r_track <- "tracks" | "TRACKS" 
            | "track" | "TRACK")
        | (r_trace <- "trace" | "TRACE")
        | (r_top <- "top" | "TOP")
        | (r_to <- "to" | "TO")
        | (r_title <- "title" | "TITLE")
        | (r_timeout <- "timeout" | "TIMEOUT")
        | (r_time <- "times" | "TIMES" 
            | "time" | "TIME")
        | (r_through <- "through" | "THROUGH" 
            | "thru" | "THRU")
        | (r_then <- "then" | "THEN")
        | (r_than <- "than" | "THAN")
        | (r_text <- "text" | "TEXT")
        | (r_test <- "test" | "TEST")
        | (r_terminate <- "terminate" | "TERMINATE")
        | (r_terminal <- "terminal" | "TERMINAL")
        | (r_tennant <- "tennant" | "TENNANT")
        | (r_tenant <- "tenant" | "TENANT")
        | (r_tape <- "tape" | "TAPE")
        | (r_tallying <- "tallying" | "TALLYING")
        | (r_tally <- "tally" | "TALLY")
        | (r_table <- "table" | "TABLE")
        | (r_synchronized <- "synchronized" | "SYNCHRONIZED")
        | (r_sync <- "sync" | "SYNC")
        | (r_symbolic <- "symbolic" | "SYMBOLIC")
        | (r_suppress <- "suppress" | "SUPPRESS")
        | (r_super <- "super" | "SUPER")
        | (r_sum <- "sum" | "SUM")
        | (r_suffix <- "suffix" | "SUFFIX")
        | (r_successive <- "successive" | "SUCCESSIVE")
        | (r_subtract <- "subtract" | "SUBTRACT")
        | (r_subschema_name <- "subschema-name" | "SUBSCHEMA-NAME")
        | (r_subrange <- "subrange" | "SUBRANGE")
        | (r_sub_schema <- "sub-schema" | "SUB-SCHEMA")
        | (r_sub_queue_3 <- "sub-queue-3" | "SUB-QUEUE-3")
        | (r_sub_queue_2 <- "sub-queue-2" | "SUB-QUEUE-2")
        | (r_sub_queue_1 <- "sub-queue-1" | "SUB-QUEUE-1")
        | (r_string <- "string" | "STRING")
        | (r_store <- "store" | "STORE")
        | (r_stop <- "stop" | "STOP")
        | (r_status <- "status" | "STATUS")
        | (r_stations <- "stations" | "STATIONS")
        | (r_station <- "station" | "STATION")
        | (r_start <- "start" | "START")
        | (r_standard_2 <- "standard-2" | "STANDARD-2")
        | (r_standard_1 <- "standard-1" | "STANDARD-1")
        | (r_standard <- "standard" | "STANDARD")
        | (r_special_names <- "special-names" | "SPECIAL-NAMES")
        | (l_string <- "spaces" | "SPACES" 
            | "space" | "SPACE")
        | (r_space_fill <- "space-fill" | "SPACE-FILL")
        | (r_source_computer <- "source-computer" | "SOURCE-COMPUTER")
        | (r_source <- "source" | "SOURCE")
        | (r_sort_status <- "sort-status" | "SORT-STATUS")
        | (r_sort_return <- "sort-return" | "SORT-RETURN")
        | (r_sort_mode_size <- "sort-mode-size" | "SORT-MODE-SIZE")
        | (r_sort_message <- "sort-message" | "SORT-MESSAGE")
        | (r_sort_merge <- "sort-merge" | "SORT-MERGE")
        | (r_sort_file_size <- "sort-file-size" | "SORT-FILE-SIZE")
        | (r_sort_core_size <- "sort-core-size" | "SORT-CORE-SIZE")
        | (r_sort_control <- "sort-control" | "SORT-CONTROL")
        | (r_sort <- "sort" | "SORT")
        | (r_skip3 <- "skip3" | "SKIP3")
        | (r_skip2 <- "skip2" | "SKIP2")
        | (r_skip1 <- "skip1" | "SKIP1")
        | (r_size <- "size" | "SIZE")
        | (r_single <- "single" | "SINGLE")
        | (r_simple <- "simple" | "SIMPLE")
        | (r_sign <- "sign" | "SIGN")
        | (r_shift_out <- "shift-out" | "SHIFT-OUT")
        | (r_shift_in <- "shift-in" | "SHIFT-IN")
        | (r_shared <- "shared" | "SHARED")
        | (r_set <- "set" | "SET")
        | (r_session_id <- "session-id" | "SESSION-ID")
        | (r_session <- "session" | "SESSION")
        | (r_service <- "service" | "SERVICE")
        | (r_sequential <- "sequential" | "SEQUENTIAL")
        | (r_sequence <- "sequence" | "SEQUENCE")
        | (r_separate <- "separate" | "SEPARATE")
        | (r_sentence <- "sentence" | "SENTENCE")
        | (r_send <- "send" | "SEND")
        | (r_self <- "self" | "SELF")
        | (r_selective <- "selective" | "SELECTIVE")
        | (r_selected <- "selected" | "SELECTED")
        | (r_select <- "select" | "SELECT")
        | (r_segment_limit <- "segment-limit" | "SEGMENT-LIMIT")
        | (r_segment <- "segment" | "SEGMENT")
        | (r_seek <- "seek" | "SEEK")
        | (r_security <- "security" | "SECURITY")
        | (r_secure <- "secure" | "SECURE")
        | (r_section <- "section" | "SECTION")
        | (r_search <- "search" | "SEARCH")
        | (r_sd <- "sd" | "SD")
        | (r_screen <- "screen" | "SCREEN")
        | (r_saved_area <- "saved-area" | "SAVED-AREA")
        | (r_same <- "same" | "SAME")
        | (r_sa <- "sa" | "SA")
        | (r_run <- "run" | "RUN")
        | (r_rounded <- "rounded" | "ROUNDED")
        | (r_rollback <- "rollback" | "ROLLBACK")
        | (r_roll_out <- "roll-out" | "ROLL-OUT")
        | (r_right_justify <- "right-justify" | "RIGHT-JUSTIFY")
        | (r_right <- "right" | "RIGHT")
        | (r_rh <- "rh" | "RH")
        | (r_rf <- "rf" | "RF")
        | (r_rewrite <- "rewrite" | "REWRITE")
        | (r_rewind <- "rewind" | "REWIND")
        | (r_reversed <- "reversed" | "REVERSED")
        | (r_reverse_video <- "reverse-video" | "REVERSE-VIDEO")
        | (r_returning <- "returning" | "RETURNING")
        | (r_return_code <- "return-code" | "RETURN-CODE")
        | (r_return <- "return" | "RETURN")
        | (r_retrieval <- "retrieval" | "RETRIEVAL")
        | (r_retaining <- "retaining" | "RETAINING")
        | (r_reset <- "reset" | "RESET")
        | (r_reserve <- "reserve" | "RESERVE")
        | (r_rerun <- "rerun" | "RERUN")
        | (r_required <- "required" | "REQUIRED")
        | (r_repository <- "repository" | "REPOSITORY")
        | (r_reporting <- "reporting" | "REPORTING")
        | (r_report <- "reports" | "REPORTS" 
            | "report" | "REPORT")
        | (r_replacing <- "replacing" | "REPLACING")
        | (r_replace <- "replace" | "REPLACE")
        | (r_repeated <- "repeated" | "REPEATED")
        | (r_reorg_criteria <- "reorg-criteria" | "REORG-CRITERIA")
        | (r_renames <- "renames" | "RENAMES")
        | (r_removal <- "removal" | "REMOVAL")
        | (r_remarks <- "remarks" | "REMARKS")
        | (r_remainder <- "remainder" | "REMAINDER")
        | (r_reload <- "reload" | "RELOAD")
        | (r_release <- "release" | "RELEASE")
        | (r_relative <- "relative" | "RELATIVE")
        | (r_relation <- "relation" | "RELATION")
        | (r_reference <- "references" | "REFERENCES" 
            | "reference" | "REFERENCE")
        | (r_reel <- "reel" | "REEL")
        | (r_redefines <- "redefines" | "REDEFINES")
        | (r_records <- "records" | "RECORDS")
        | (r_recording <- "recording" | "RECORDING")
        | (r_record_overflow <- "record-overflow" | "RECORD-OVERFLOW")
        | (r_record_name <- "record-name" | "RECORD-NAME")
        | (r_record <- "record" | "RECORD")
        | (r_reconnect <- "reconnect" | "RECONNECT")
        | (r_receive <- "receive" | "RECEIVE")
        | (r_realm <- "realm" | "REALM")
        | (r_ready <- "ready" | "READY")
        | (r_read <- "read" | "READ")
        | (r_rd <- "rd" | "RD")
        | (r_range <- "range" | "RANGE")
        | (r_random <- "random" | "RANDOM")
        | (r_raising <- "raising" | "RAISING")
        | (r_raise <- "raise" | "RAISE")
        | (r_quote <- "quotes" | "QUOTES" 
            | "quote" | "QUOTES")
        | (r_queue <- "queue" | "QUEUE")
        | (r_purge <- "purge" | "PURGE")
        | (r_prototype <- "prototype" | "PROTOTYPE")
        | (r_protected <- "protected" | "PROTECTED")
        | (r_property <- "property" | "PROPERTY")
        | (r_prompt <- "prompt" | "PROMPT")
        | (r_program_status <- "program-status" | "PROGRAM-STATUS")
        | (r_program_id <- "program-id" | "PROGRAM-ID")
        | (r_program <- "program" | "PROGRAM")
        | (r_processing <- "processing" | "PROCESSING")
        | (r_proceed <- "proceed" | "PROCEED")
        | (r_procedure <- "procedures" | "PROCEDURES" 
            | "procedure" | "PROCEDURE")
        | (r_prior <- "prior" | "PRIOR")
        | (r_printing <- "printing" | "PRINTING")
        | (r_previous <- "previous" | "PREVIOUS")
        | (r_present <- "present" | "PRESENT")
        | (r_prefix <- "prefix" | "PREFIX")
        | (r_positive <- "positive" | "POSITIVE")
        | (r_positioning <- "positioning" | "POSITIONING")
        | (r_position <- "position" | "POSITION")
        | (r_pointer <- "pointer" | "POINTER")
        | (r_plus <- "plus" | "PLUS")
        | (r_ph <- "ph" | "PH")
        | (r_pf <- "pf" | "PF")
        | (r_perform <- "perform" | "PERFORM")
        | (r_password <- "password" | "PASSWORD")
        | (r_paragraph <- "paragraph" | "PARAGRAPH")
        | (r_page_counter <- "page-counter" | "PAGE-COUNTER")
        | (r_page <- "page" | "PAGE")
        | (r_padding <- "padding" | "PADDING")
        | (r_packed_decimal <- "packed-decimal" | "PACKED-DECIMAL")
        | (r_owner <- "owner" | "OWNER")
        | (r_override <- "override" | "OVERRIDE")
        | (r_overline <- "overline" | "OVERLINE")
        | (r_overflow <- "overflow" | "OVERFLOW")
        | (r_output <- "output" | "OUTPUT")
        | (r_otherwise <- "otherwise" | "OTHERWISE")
        | (r_other <- "other" | "OTHER")
        | (r_organization <- "organization" | "ORGANIZATION")
        | (r_order <- "order" | "ORDER")
        | (r_or <- "or" | "OR")
        | (r_optional <- "optional" | "OPTIONAL")
        | (r_open <- "open" | "OPEN")
        | (r_only <- "only" | "ONLY")
        | (r_on <- "on" | "ON")
        | (r_omitted <- "omitted" | "OMITTED")
        | (r_off <- "off" | "OFF")
        | (r_in_of <- "of" | "OF") // "in", "of"
        | (r_occurs <- "occurs" | "OCCURS")
        | (r_object_computer <- "object-computer" | "OBJECT-COMPUTER")
        | (r_object <- "object" | "OBJECT")
        | (r_numeric_edited <- "numeric-edited" | "NUMERIC-EDITED")
        | (r_numeric <- "numeric" | "NUMERIC")
        | (r_number <- "number" | "NUMBER")
        | (r_null <- "nulls" | "NULLS" 
            | "null" | "NULL")
        | (r_note <- "note" | "NOTE")
        | (r_not <- "not" | "NOT")
        | (r_none <- "none" | "NONE")
        | (r_nominal <- "nominal" | "NOMINAL")
        | (r_no <- "no" | "NO")
        | (r_next <- "next" | "NEXT")
        | (r_negative <- "negative" | "NEGATIVE")
        | (r_native <- "native" | "NATIVE")
        | (r_national_edited <- "national-edited" | "NATIONAL-EDITED")
        | (r_national <- "national" | "NATIONAL")
        | (r_named <- "named" | "NAMED")
        | (r_multiply <- "multiply" | "MULTIPLY")
        | (r_multiple <- "multiple" | "MULTIPLE")
        | (r_multiconversation_mode <- "multiconversation-mode" | "MULTICONVERSATION-MODE")
        | (r_multicon <- "multicon" | "MULTICON")
        | (r_move <- "move" | "MOVE")
        | (r_more_labels <- "more-labels" | "MORE-LABELS")
        | (r_modules <- "modules" | "MODULES")
        | (r_modify <- "modify" | "MODIFY")
        | (r_mode_3 <- "mode-3" | "MODE-3")
        | (r_mode_2 <- "mode-2" | "MODE-2")
        | (r_mode_1 <- "mode-1" | "MODE-1")
        | (r_mode <- "mode" | "MODE")
        | (r_method_id <- "method-id" | "METHOD-ID") 
            ?(
                (null <- *(" " | "&t;")) (period <- ".") (null <- *(" " | "&t;"))
                ?(
                    ((r_get <- "get" | "GET") | (r_set <- "set" | "SET")) (null <- +(" " | "&t;")) (r_property <- "property" | "PROPERTY") (null <- *(" " | "&t;"))
                )
                (null <- "&quot;" | "&squot;")
                (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
                *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
                (null <- "&quot;" | "&squot;")
            )
        | (r_method <- "method" | "METHOD")
        | (r_message <- "message" | "MESSAGE")
        | (r_merge <- "merge" | "MERGE")
        | (r_memory <- "memory" | "MEMORY")
        | (r_member <- "member" | "MEMBER")
        | (r_manual <- "manual" | "MANUAL")
        | (r_lowlight <- "lowlight" | "LOWLIGHT")
        | (r_low_value <- "low-values" | "LOW-VALUES" 
            | "low-value" | "LOW-VALUE")
        | (r_lock <- "lock" | "LOCK")
        | (r_locally <- "locally" | "LOCALLY")
        | (r_linkage <- "linkage" | "LINKAGE")
        | (r_line_counter <- "line-counter" | "LINE-COUNTER")
        | (r_line <- "lines" | "LINES" 
            | "line" | "LINE")
        | (r_linage_counter <- "linage-counter" | "LINAGE-COUNTER")
        | (r_linage <- "linage" | "LINAGE")
        | (r_limited <- "limited" | "LIMITED")
        | (r_limit <- "limits" | "LIMITS" 
            | "limit" | "LIMIT")
        | (r_less <- "less" | "LESS")
        | (r_length <- "length" | "LENGTH")
        | (r_leftline <- "leftline" | "LEFTLINE")
        | (r_left_justify <- "left-justify" | "LEFT-JUSTIFY")
        | (r_left <- "left" | "LEFT")
        | (r_leading <- "leading" | "LEADING")
        | (r_ld <- "ld" | "LD")
        | (r_last <- "last" | "LAST")
        | (r_label <- "label" | "LABEL")
        | (r_key <- "key" | "KEY")
        | (r_keep <- "keep" | "KEEP")
        | (r_kanji <- "kanji" | "KANJI")
        | (r_justified <- "justified" | "JUSTIFIED")
        | (r_just <- "just" | "JUST")
        | (r_joining <- "joining" | "JOINING")
        | (r_job <- "job" | "JOB")
        | (r_japanese <- "japanese" | "JAPANESE")
        | (r_is <- "is" | "IS")
        | (r_invoke <- "invoke" | "INVOKE")
            ?(
                (null <- +(" " | "&t;"))
                (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
                *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
                (null <- *(" " | "&t;") ("&quot;" | "&squot;"))
                (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
                *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
                (null <- "&quot;" | "&squot;")
            )
        | (r_invariant <- "invariant" | "INVARIANT")
        | (r_invalid <- "invalid" | "INVALID")
        | (r_into <- "into" | "INTO")
        | (r_interface <- "interface" | "INTERFACE")
        | (r_interface_id <- "interface-id" | "INTERFACE-ID")
        | (r_installation <- "installation" | "INSTALLATION")
        | (r_inspect <- "inspect" | "INSPECT")
        | (r_input_output <- "input-output" | "INPUT-OUTPUT")
        | (r_input <- "input" | "INPUT")
        | (r_initiate <- "initiate" | "INITIATE")
        | (r_initialize <- "initialize" | "INITIALIZE")
        | (r_initial <- "initial" | "INITIAL")
        | (r_inherits <- "inherits" | "INHERITS")
        | (r_indicate <- "indicate" | "INDICATE")
        | (r_indexed <- "indexed" | "INDEXED")
        | (r_index_n <- "index-n" | "INDEX-N")
        | (r_index <- "index" | "INDEX")
        | (r_include <- "include" | "INCLUDE")
        | (r_in_of <- "in" | "IN") // "in", "of"
        | (r_if <- "if" | "IF")
        | (r_identification <- "identification" | "IDENTIFICATION")
        | (r_id <- "id" | "ID")
        | (r_i_o_control <- "i-o-control" | "I-O-CONTROL")
        | (r_i_o <- "i-o" | "I-O")
        | (r_highlight <- "highlight" | "HIGHLIGHT")
        | (r_high_value <- "high-values" | "HIGH-VALUES" 
            | "high-value" | "HIGH-VALUE")
        | (r_heading <- "heading" | "HEADING")
        | (r_group <- "group" | "GROUP")
        | (r_grid <- "grid" | "GRID")
        | (r_greater <- "greater" | "GREATER")
        | (r_goback <- "goback" | "GOBACK")
        | (r_go <- "go" | "GO")
        | (r_global <- "global" | "GLOBAL")
        | (r_giving <- "giving" | "GIVING")
        | (r_get <- "get" | "GET")
        | (r_generate <- "generate" | "GENERATE")
        | (r_function <- "function" | "FUNCTION")
        | (r_full <- "full" | "FULL")
        | (r_from <- "from" | "FROM")
        | (r_free <- "free" | "FREE")
        | (r_formatted <- "formatted" | "FORMATTED")
        | (r_format <- "format" | "FORMAT")
        | (r_form <- "form" | "FORM")
        | (r_foreground_color <- "foreground-color" | "FOREGROUND-COLOR")
        | (r_for <- "for" | "FOR")
        | (r_footing <- "footing" | "FOOTING")
        | (r_fladd <- "fladd" | "FLADD")
        | (r_first <- "first" | "FIRST")
        | (r_finish <- "finish" | "FINISH")
        | (r_find <- "find" | "FIND")
        | (r_final <- "final" | "FINAL")
        | (r_filler <- "filler" | "FILLER")
        | (r_file_limit <- "file-limits" | "FILE-LIMITS" 
            | "file-limit" | "FILE-LIMIT")
        | (r_file_control <- "file-control" | "FILE-CONTROL")
        | (r_file <- "files" | "FILES" 
            | "file" | "FILE")
        | (r_fetch <- "fetch" | "FETCH")
        | (r_fd <- "fd" | "FD")
        | (r_false <- "false" | "FALSE")
        | (r_factory <- "factory" | "FACTORY")
        | (r_external <- "external" | "EXTERNAL")
        | (r_extend <- "extend" | "EXTEND")
        | (r_exor <- "exor" | "EXOR")
        | (r_exit <- "exit" | "EXIT")
        | (r_exec <- "exec" | "EXEC")
        | (r_exclusive <- "exclusive" | "EXCLUSIVE")
        | (r_exception_object <- "exception-object" | "EXCEPTION-OBJECT")
        | (r_exception <- "exception" | "EXCEPTION")
        | (r_exceeds <- "exceeds" | "EXCEEDS")
        | (r_examine <- "examine" | "EXAMINE")
        | (r_exact <- "exact" | "EXACT")
        | (r_every <- "every" | "EVERY")
        | (r_evaluate <- "evaluate" | "EVALUATE")
        | (r_esi <- "esi" | "ESI")
        | (r_error <- "error" | "ERROR")
        | (r_erase <- "erase" | "ERASE")
        | (r_equals <- "equals" | "EQUALS")
        | (r_equal <- "equal" | "EQUAL")
        | (r_eos <- "eos" | "EOS")
        | (r_eop <- "eop" | "EOP")
        | (r_eol <- "eol" | "EOL")
        | (r_environment <- "environment" | "ENVIRONMENT")
        | (r_entry <- "entry" | "ENTRY")
        | (r_enter <- "enter" | "ENTER")
        | (r_ending <- "ending" | "ENDING")
        | (r_endcobol <- "endcobol" | "ENDCOBOL")
        | (r_end_write <- "end-write" | "END-WRITE")
        | (r_end_unstring <- "end-unstring" | "END-UNSTRING")
        | (r_end_transceive <- "end-transceive" | "END-TRANSCEIVE")
        | (r_end_subtract <- "end-subtract" | "END-SUBTRACT")
        | (r_end_string <- "end-string" | "END-STRING")
        | (r_end_start <- "end-start" | "END-START")
        | (r_end_send <- "end-send" | "END-SEND")
        | (r_end_search <- "end-search" | "END-SEARCH")
        | (r_end_rewrite <- "end-rewrite" | "END-REWRITE")
        | (r_end_return <- "end-return" | "END-RETURN")
        | (r_end_receive <- "end-receive" | "END-RECEIVE")
        | (r_end_read <- "end-read" | "END-READ")
        | (r_end_perform <- "end-perform" | "END-PERFORM")
        | (r_end_of_page <- "end-of-page" | "END-OF-PAGE")
        | (r_end_multiply <- "end-multiply" | "END-MULTIPLY")
        | (r_end_invoke <- "end-invoke" | "END-INVOKE")
        | (r_end_if <- "end-if" | "END-IF")
        | (r_end_exec <- "end-exec" | "END-EXEC")
        | (r_end_evaluate <- "end-evaluate" | "END-EVALUATE")
        | (r_end_enable <- "end-enable" | "END-ENABLE")
        | (r_end_divide <- "end-divide" | "END-DIVIDE")
        | (r_end_display <- "end-display" | "END-DISPLAY")
        | (r_end_disable <- "end-disable" | "END-DISABLE")
        | (r_end_delete <- "end-delete" | "END-DELETE")
        | (r_end_compute <- "end-compute" | "END-COMPUTE")
        | (r_end_call <- "end-call" | "END-CALL")
        | (r_end_add <- "end-add" | "END-ADD")
        | (r_end_accept <- "end-accept" | "END-ACCEPT")
        | (r_end_method <- ("end" | "END") (null <- +(" " | "&t;")) ("method" | "METHOD") 
            (null <- *(" " | "&t;") ("&quot;" | "&squot;"))
            (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
            *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
            (null <- "&quot;" | "&squot;"))
        | (r_end_class <- ("end" | "END") (null <- +(" " | "&t;")) ("class" | "CLASS") 
            (null <- *(" " | "&t;") ("&quot;" | "&squot;"))
            (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
            *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
            (null <- "&quot;" | "&squot;"))
        | (r_end <- "end" | "END")
        | (r_enable <- "enable" | "ENABLE")
        | (r_empty <- "empty" | "EMPTY")
        | (r_emi <- "emi" | "EMI")
        | (r_else <- "else" | "ELSE")
        | (r_eject <- "eject" | "EJECT")
        | (r_egi <- "egi" | "EGI")
        | (r_egcs <- "egcs" | "EGCS")
        | (r_edit_status <- "edit-status" | "EDIT-STATUS")
        | (r_edit_option <- "edit-option" | "EDIT-OPTION")
        | (r_edit_mode <- "edit-mode" | "EDIT-MODE")
        | (r_edit_cursor <- "edit-cursor" | "EDIT-CURSOR")
        | (r_edit_color <- "edit-color" | "EDIT-COLOR")
        | (r_dynamic <- "dynamic" | "DYNAMIC")
        | (r_duplicate <- "duplicates" | "DUPLICATES" 
            | "duplicate" | "DUPLICATE")
        | (r_down <- "down" | "DOWN")
        | (r_division <- "division" | "DIVISION")
        | (r_divide <- "divide" | "DIVIDE")
        | (r_display_n <- "display-n" | "DISPLAY-N")
        | (r_display_exit <- "display-exit" | "DISPLAY-EXIT")
        | (r_display_1 <- "display-1" | "DISPLAY-1")
        | (r_display <- "display" | "DISPLAY")
        | (r_disjoining <- "disjoining" | "DISJOINING")
        | (r_disconnect <- "disconnect" | "DISCONNECT")
        | (r_disable <- "disable" | "DISABLE")
        | (r_direct <- "direct" | "DIRECT")
        | (r_device <- "device" | "DEVICE")
        | (r_detail <- "detail" | "DETAIL")
        | (r_destination_3 <- "destination-3" | "DESTINATION-3")
        | (r_destination_2 <- "destination-2" | "DESTINATION-2")
        | (r_destination_1 <- "destination-1" | "DESTINATION-1")
        | (r_destination <- "destination" | "DESTINATION")
        | (r_descending <- "descending" | "DESCENDING")
        | (r_depending <- "depending" | "DEPENDING")
        | (r_delimiter <- "delimiter" | "DELIMITER")
        | (r_delimited <- "delimited" | "DELIMITED")
        | (r_delete <- "delete" | "DELETE")
        | (r_default <- "default" | "DEFAULT")
        | (r_declaratives <- "declaratives" | "DECLARATIVES")
        | (r_decimal_point <- "decimal-point" | "DECIMAL-POINT")
        | (r_debugging <- "debugging" | "DEBUGGING")
        | (r_debug_sub_3 <- "debug-sub-3" | "DEBUG-SUB-3")
        | (r_debug_sub_2 <- "debug-sub-2" | "DEBUG-SUB-2")
        | (r_debug_sub_1 <- "debug-sub-1" | "DEBUG-SUB-1")
        | (r_debug_name <- "debug-name" | "DEBUG-NAME")
        | (r_debug_line <- "debug-line" | "DEBUG-LINE")
        | (r_debug_item <- "debug-item" | "DEBUG-ITEM")
        | (r_debug_contents <- "debug-contents" | "DEBUG-CONTENTS")
        | (r_dead_lock <- "dead-lock" | "DEAD-LOCK")
        | (r_de <- "de" | "DE")
        | (r_db_status <- "db-status" | "DB-STATUS")
        | (r_db_set_name <- "db-set-name" | "DB-SET-NAME")
        | (r_db_record_name <- "db-record-name" | "DB-RECORD-NAME")
        | (r_db_exception <- "db-exception" | "DB-EXCEPTION")
        | (r_db_data_name <- "db-data-name" | "DB-DATA-NAME")
        | (r_db_access_control_key <- "db-access-control-key" | "DB-ACCESS-CONTROL-KEY")
        | (r_db <- "db" | "DB")
        | (r_day_of_week <- "day-of-week" | "DAY-OF-WEEK")
        | (r_day <- "day" | "DAY")
        | (r_date_written <- "date-written" | "DATE-WRITTEN")
        | (r_date_compiled <- "date-compiled" | "DATE-COMPILED")
        | (r_date <- "date" | "DATE")
        | (r_data <- "data" | "DATA")
        | (r_cursor <- "cursor" | "CURSOR")
        | (r_current <- "current" | "CURRENT")
        | (r_currency <- "currency" | "CURRENCY")
        | (r_crt_under <- "crt-under" | "CRT-UNDER")
        | (r_crt <- "crt" | "CRT")
        | (r_crp <- "crp" | "CRP")
        | (r_count <- "count" | "COUNT")
        | (r_corresponding <- "corresponding" | "CORRESPONDING")
        | (r_corr <- "corr" | "CORR")
        | (r_core_index <- "core-index" | "CORE-INDEX")
        | (r_copy <- "copy" | "COPY")
        | (r_converting <- "converting" | "CONVERTING")
        | (r_control_character <- "control-character" | "CONTROL-CHARACTER")
        | (r_control <- "controls" | "CONTROLS" 
            | "control" | "CONTROL")
        | (r_continue <- "continue" | "CONTINUE")
        | (r_content <- "content" | "CONTENT")
        | (r_contains <- "contains" | "CONTAINS")
        | (r_contained <- "contained" | "CONTAINED")
        | (r_constant <- "constant" | "CONSTANT")
        | (r_connect <- "connect" | "CONNECT")
        | (r_configuration <- "configuration" | "CONFIGURATION")
        | (r_compute <- "compute" | "COMPUTE")
        | (r_computational_n <- "computational-n" | "COMPUTATIONAL-N")
        | (r_computational_5 <- "computational-5" | "COMPUTATIONAL-5")
        | (r_computational_4 <- "computational-4" | "COMPUTATIONAL-4")
        | (r_computational_3 <- "computational-3" | "COMPUTATIONAL-3")
        | (r_computational_2 <- "computational-2" | "COMPUTATIONAL-2")
        | (r_computational_1 <- "computational-1" | "COMPUTATIONAL-1")
        | (r_computational <- "computational" | "COMPUTATIONAL")
        | (r_complex <- "complex" | "COMPLEX")
        | (r_comp_n <- "comp-n" | "COMP-N")
        | (r_comp_5 <- "comp-5" | "COMP-5")
        | (r_comp_4 <- "comp-4" | "COMP-4")
        | (r_comp_3 <- "comp-3" | "COMP-3")
        | (r_comp_2 <- "comp-2" | "COMP-2")
        | (r_comp_1 <- "comp-1" | "COMP-1")
        | (r_comp <- "comp" | "COMP")
        | (r_communication <- "communication" | "COMMUNICATION")
        | (r_common <- "common" | "COMMON")
        | (r_commit <- "commit" | "COMMIT")
        | (r_command <- "command" | "COMMAND")
        | (r_comma <- "comma" | "COMMA")
        | (r_com_reg <- "com-reg" | "COM-REG")
        | (r_column <- "column" | "COLUMN")
        | (r_collating <- "collating" | "COLLATING")
        | (r_code_set <- "code-set" | "CODE-SET")
        | (r_code <- "code" | "CODE")
        | (r_cobol <- "cobol" | "COBOL")
        | (r_close <- "close" | "CLOSE")
        | (r_clock_units <- "clock-units" | "CLOCK-UNITS")
        | (r_class_id <- "class-id" | "CLASS-ID")
        | (r_class <- "class" | "CLASS")
        | (r_characters <- "characters" | "CHARACTERS")
        | (r_character <- "character" | "CHARACTER")
        | (r_changed <- "changed" | "CHANGED")
        | (r_ch <- "ch" | "CH")
        | (r_cf <- "cf" | "CF")
        | (r_cd <- "cd" | "CD")
        | (r_cbl <- "cbl" | "CBL")
        | (r_cancel <- "cancel" | "CANCEL")
        | (r_call <- "call" | "CALL")
            ?(
                (null <- *(" " | "&t;") ("&quot;" | "&squot;"))
                (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
                *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"))
                (null <- "&quot;" | "&squot;")
            )
        | (r_by <- "by" | "BY")
        | (r_bottom <- "bottom" | "BOTTOM")
        | (r_boolean <- "boolean" | "BOOLEAN")
        | (r_block <- "block" | "BLOCK")
        | (r_blink <- "blink" | "BLINK")
        | (r_blank <- "blank" | "BLANK")
        | (r_bit <- "bits" | "BITS" 
            | "bit" | "BIT")
        | (r_binary <- "binary" | "BINARY")
        | (r_bell <- "bell" | "BELL")
        | (r_beginning <- "beginning" | "BEGINNING")
        | (r_before <- "before" | "BEFORE")
        | (r_based_storage <- "based-storage" | "BASED-STORAGE")
        | (r_based <- "based" | "BASED")
        | (r_background_color <- "background-color" | "BACKGROUND-COLOR")
        | (r_b_or <- "b-or" | "B-OR")
        | (r_b_not <- "b-not" | "B-NOT")
        | (r_b_less <- "b-less" | "B-LESS")
        | (r_b_exor <- "b-exor" | "B-EXOR")
        | (r_b_and <- "b-and" | "B-AND")
        | (r_automatic <- "automatic" | "AUTOMATIC")
        | (r_auto <- "auto" | "AUTO")
        | (r_author <- "author" | "AUTHOR")
        | (r_at <- "at" | "AT")
        | (r_assign <- "assign" | "ASSIGN")
        | (r_ascending <- "ascending" | "ASCENDING")
        | (r_as <- "as" | "AS")
        | (r_arithmetic <- "arithmetic" | "ARITHMETIC")
        | (r_area <- "areas" | "AREAS" 
            | "area" | "AREA")
        | (r_is <- "are" | "ARE") // | (r_are <- "are" | "ARE")
        | (r_apply <- "apply" | "APPLY")
        | (r_any <- "any" | "ANY")
        | (r_and <- "and" | "AND")
        | (r_alternate <- "alternate" | "ALTERNATE")
        | (r_alter <- "alter" | "ALTER")
        | (r_also <- "also" | "ALSO")
        | (r_alphanumeric_edited <- "alphanumeric-edited" | "ALPHANUMERIC-EDITED")
        | (r_alphanumeric <- "alphanumeric" | "ALPHANUMERIC")
        | (r_alphabetic_upper <- "alphabetic-upper" | "ALPHABETIC-UPPER")
        | (r_alphabetic_lower <- "alphabetic-lower" | "ALPHABETIC-LOWER")
        | (r_alphabetic <- "alphabetic" | "ALPHABETIC")
        | (r_alphabet <- "alphabet" | "ALPHABET")
        | (r_all <- "all" | "ALL")
        | (r_after <- "after" | "AFTER")
        | (r_advancing <- "advancing" | "ADVANCING")
        | (r_address <- "address" | "ADDRESS")
        | (r_add <- "add" | "ADD")
        | (r_actual <- "actual" | "ACTUAL")
        | (r_access <- "access" | "ACCESS")
        | (r_accept <- "accept" | "ACCEPT")
    ) xcep("&(a-z);" | "&(A-Z);" | "-" | "&(0-9);")
    | (op_le <- "<=")
    | (op_ge <- ">=")
    | (op_lt <- "<")
    | (op_gt <- ">")
    | (op_eq <- "=")
    | (op_power <- "**")
    | (comment <- "*>" *(xcep(eol) any))
    | (op_star <- "*")
    | (op_plus <- "+")
    | (op_minus <- "-")
    | (op_div <- "/")
    | (semicolon <- ";")
    | (colon <- ":")
    | (comma <- ",") 
    | (LP <- "(") | (RP <- ")") 
    | (period <- ".")
    | (word <- *(+"&(0-9);" ?"-") ("&(a-z);" | "&(A-Z);" | "_" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "@") 
      *("&(a-z);" | "&(A-Z);" | "_" | "-" | "&#x(2e80-2fdf);" | "&#x(3040-30ff);" | "&#x(3400-9faf);" | "&#x(ff66-ff9f);" | "&#x(ff21-ff3a);" | "&#x(ff41-ff5a);" | "&(0-9);" | "&#x(ff10-ff19);" | "&#x(ff0d-ff0e);"));

TEXT scan= (null <- " " | "&t;" | comment | eol);

TEXT scan= (l_string <- l_string +("&" l_string))
    | (l_bool <- r_true | r_false);

TEXT scan= (r_at_end <- r_at r_end)
    | (r_go_to <- r_go r_to)
    | (r_next_sentence <- r_next r_sentence)
    | (r_depending_on <- r_depending r_on)
    | (r_delimited_by <- r_delimited r_by)
    | (r_type_is <- r_type r_is)
    | (r_printing_mode <- r_printing r_mode)
    | (r_by_value <- r_by r_value)
    | (r_by_reference <- r_by r_reference)
    | (r_exit_program <- r_exit r_program)
    | (r_exit_perform <- r_exit r_perform)
    | (r_is_global <- r_is r_global)
    | (r_is_external <- r_is r_external)
    | (r_indexed_by <- r_indexed r_by)
    | (op_le <- r_less r_than r_or r_equal r_to)
    | (op_ge <- r_greater r_than r_or r_equal r_to)
    | (op_lt <- r_less r_than)
    | (op_gt <- r_greater)
    | (op_eq <- r_equal r_to)
    | (r_end_factory <- r_end r_factory) 
    | (r_end_object <- r_end r_object)
    | (r_end_method <- r_end r_method)
    | (r_end_class <- r_end r_class)
    | (r_end_program <- r_end r_program)
    | (r_end_interface <- r_end r_interface)
    | (null <- r_identification r_division period) (r_class_id | r_factory | r_object | r_method_id | r_interface_id)
    | (division_start <- any r_division)
    | (section_start <- any r_section);

TEXT scan= (def_block <- 
        r_class_id period word ?(r_inherits word period) *(xcep(r_factory | r_object | r_end_class | eof) any)
        ?(def_block <- r_factory period 
            *(
                (def_block <- r_method_id period ?((r_get | r_set) r_property) word ?(r_override period) *(xcep(r_end_method | eof) any) r_end_method ?word period)
                | xcep(r_end_factory | eof) any
            ) 
            r_end_factory ?word period)
        ?(def_block <- r_object period
            *(
                (def_block <- r_method_id period ?((r_get | r_set) r_property) word ?(r_override period) *(xcep(r_end_method | eof) any) r_end_method ?word period)
                | xcep(r_end_object | eof) any
            ) 
            r_end_object ?word period
        )
        r_end_class ?word period
    )
    | (null <- r_interface_id period word ?(r_inherits word period) *(xcep(r_end_interface | eof) any) r_end_interface ?word period);

TEXT scan= 
    (division_block <-
        (division_start match (r_data | r_environment) r_division) period
        *(
            (section_block <- section_start *(xcep(section_start | division_start | def_block | eof) any))
            | xcep(division_start | def_block | eof) any
        )
    )
    | (division_block <-
        (division_start match r_procedure r_division)
        ?(r_using *(r_by_value | r_by_reference | word)) ?(r_returning *word)
        period
        *(
            (section_block <- section_start *(xcep(section_start | division_start | def_block | eof) any))
            | xcep(division_start | def_block | eof) any
        )
    )
    | (
        null <- (division_start match r_identification r_division) period *(xcep(division_start | def_block | eof) any)
    )
    | (division_block <- division_start period *(xcep(division_start | def_block | eof) any))
    | (def_block scan ^);

TEXT scan= xcep(r_end_string | r_end_unstring | r_end_evaluate | r_else | r_end_if | r_end_perform | r_end_search | period | eof) (
        (block <- r_string *^ (preq(period) | r_end_string))
        | (block <- r_unstring *^ ?(preq(period) | r_end_unstring))
        | (block <- r_evaluate *^ ?(preq(period) | r_end_evaluate))
        | (block <- r_if *^ ?(r_else *^) ?(preq(period) | preq(r_else) | r_end_if))
        | (block <- r_perform *^ ?(preq(period) | r_end_perform))
        | (block <- r_search *^ ?(preq(period) | r_end_search))
        | (division_block scan ^)
        | (section_block scan ^)
        | (def_block scan ^)
        | any
    );

// insert tokens for control-flow complexity counter
TEXT scan= (r_if | r_evaluate) insert(c_cond) 
    | r_search insert(c_loop)
    | r_perform insert(c_loop) (word r_through | r_varying | (word | l_int) r_time | r_until)
    | (division_block scan ^)
    | (section_block scan ^)
    | (def_block scan ^)
    | (block scan ^);

TEXT scan= program_block <- xcep(r_end_program | class_block | eof) any r_end_program ?word period;
"""
        self.pat = easytorq.Pattern(patternStr)
        
        fmt = easytorq.CngFormatter()
        
        # parameter by default
        fmt.addreplace('word', 'word|%s')
        
        # non parameter by default
        fmt.addreplace('l_int', 'l_int|%s')
        fmt.addreplace('l_float', 'l_float|%s')
        fmt.addreplace('l_string', 'l_string|%s')
        fmt.addreplace('l_bool', 'l_bool|%s')
        
        fmt.addreplace('period', 'suffix:period')
        fmt.addformat('def_block', '(def_block', ')def_block')
        fmt.addflatten('division_start')
        fmt.addformat('division_block', '(division_block', ')division_block')
        fmt.addflatten('section_start')
        fmt.addformat('section_block', '(section_block', ')section_block')
        fmt.addformat('program_block', '(program_block', ')program_block')
        fmt.addformat('block', '(block', ')block')
        self.fmt = fmt
    
    def getnormalizedoptionstring(self):
        return 'default'
    
    def getdefaultparameterizing(self):
        return { "word" : pp.Param.P_MATCH }
    
    def parse(self, sourceCodeStrInUtf8):
        if self.pat == None:
            self.setoptions(None)
        
        t = easytorq.Tree(sourceCodeStrInUtf8)
        self.pat.apply(t)
        s = self.fmt.format(t)
        return s
    
def getpreprocessor():
    return CobolPreprocessor()

if __name__ == '__main__':
    cnv = easytorq.ICUConverter()
    cnv.setencoding("char")
    
    f = file(sys.argv[1], "rb")
    str = f.read()
    f.close()
    
    strUtf8 = cnv.decode(str)
    
    prep = preprocessor()
    prep.setoptions("")
    s = prep.parse(strUtf8)
    sys.stdout.write(s)
    
