pylatexenc/latexwalker.py
def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
        """

        s = self.s # shorthand

        with _PushPropOverride(self, 'keep_inline_math', keep_inline_math):

            space = ''
            while (pos < len(s) and s[pos].isspace()):
                space += s[pos]
                pos += 1
                if (space.endswith('\n\n')):  # two \n's indicate new paragraph.
                    # Adding pre-space is overkill here I think.
                    return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='')


            if (pos >= len(s)):
                raise LatexWalkerEndOfStream()

            if (s[pos] == '\\'):
                # escape sequence
                i = 2
                macro = s[pos+1] # next char is necessarily part of macro
                # following chars part of macro only if all are alphabetical
                isalphamacro = False
                if (s[pos+1].isalpha()):
                    isalphamacro = True
                    while pos+i<len(s) and s[pos+i].isalpha():
                        macro += s[pos+i]
                        i += 1
                # possibly followed by a star
                if (pos+i<len(s) and s[pos+i] == '*'):
                    macro += '*'
                    i += 1

                # see if we have a begin/end environment
                if (environments and (macro == 'begin' or macro == 'end')):
                    # \begin{environment} or \end{environment}
                    envmatch = re.match(r'^\s*\{([\w*]+)\}', s[pos+i:])
                    if (envmatch is None):
                        raise LatexWalkerParseError(
                            s=s,
                            pos=pos,
                            msg="Bad \\%s macro: expected {environment}" %(macro)
                        )

                    return LatexToken(
                        tok=('begin_environment' if macro == 'begin' else  'end_environment'),
                        arg=envmatch.group(1),
                        pos=pos,
                        len=i+envmatch.end(), # !!envmatch.end() counts from pos+i
                        pre_space=space
                        )

                # get the following whitespace, and store it in the macro's post_space
                post_space = ''
                if isalphamacro:
                    # important, LaTeX does not consume space after non-alpha macros, like \&
                    while pos+i<len(s) and s[pos+i].isspace():
                        post_space += s[pos+i]
                        i += 1

                return LatexToken(tok='macro', arg=macro, pos=pos, len=i,
                                  pre_space=space, post_space=post_space)

            if (s[pos] == '%'):
                # latex comment
                m = re.search(r'(\n|\r|\n\r)\s*', s[pos:])
                mlen = None
                if m is not None:
                    arglen = m.start() # relative to pos already
                    mlen = m.end() # relative to pos already
                    mspace = m.group()
                else:
                    arglen = len(s)-pos# [  ==len(s[pos:])  ]
                    mlen = arglen
                    mspace = ''
                return LatexToken(tok='comment', arg=s[pos+1:pos+arglen], pos=pos, len=mlen,
                                  pre_space=space, post_space=mspace)

            openbracechars = '{'
            closebracechars = '}'
            if not brackets_are_chars:
                openbracechars += '['
                closebracechars += ']'

            if s[pos] in openbracechars:
                return LatexToken(tok='brace_open', arg=s[pos], pos=pos, len=1, pre_space=space)

            if s[pos] in closebracechars:
                return LatexToken(tok='brace_close', arg=s[pos], pos=pos, len=1, pre_space=space)

            # check if it is an inline math char, if we care about inline math.
            if (s[pos] == '$' and self.keep_inline_math):
                # check that we don't have double-$$, which would be a display environment.
                if not (pos+1 < len(s) and s[pos+1] == '$'):
                    return LatexToken(tok='mathmode_inline', arg=s[pos], pos=pos, len=1, pre_space=space)
                # otherwise, proceed to 'char' type.

            return LatexToken(tok='char', arg=s[pos], pos=pos, len=1, pre_space=space)
def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
        """

        s = self.s # shorthand

        with _PushPropOverride(self, 'keep_inline_math', keep_inline_math):

            space = ''
            while (pos < len(s) and s[pos].isspace()):
                space += s[pos]
                pos += 1
                if (space.endswith('\n\n')):  # two \n's indicate new paragraph.
                    # Adding pre-space is overkill here I think.
                    return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='')


            if (pos >= len(s)):
                raise LatexWalkerEndOfStream()

            if (s[pos] == '\\'):
                # escape sequence
                i = 2
                macro = s[pos+1] # next char is necessarily part of macro
                # following chars part of macro only if all are alphabetical
                isalphamacro = False
                if (s[pos+1].isalpha()):
                    isalphamacro = True
                    while pos+i<len(s) and s[pos+i].isalpha():
                        macro += s[pos+i]
                        i += 1
                # possibly followed by a star
                if (pos+i<len(s) and s[pos+i] == '*'):
                    macro += '*'
                    i += 1

                # see if we have a begin/end environment
                if (environments and (macro == 'begin' or macro == 'end')):
                    # \begin{environment} or \end{environment}
                    envmatch = re.match(r'^\s*\{([\w*]+)\}', s[pos+i:])
                    if (envmatch is None):
                        raise LatexWalkerParseError(
                            s=s,
                            pos=pos,
                            msg="Bad \\%s macro: expected {environment}" %(macro)
                        )

                    return LatexToken(
                        tok=('begin_environment' if macro == 'begin' else  'end_environment'),
                        arg=envmatch.group(1),
                        pos=pos,
                        len=i+envmatch.end(), # !!envmatch.end() counts from pos+i
                        pre_space=space
                        )

                # get the following whitespace, and store it in the macro's post_space
                post_space = ''
                if isalphamacro:
                    # important, LaTeX does not consume space after non-alpha macros, like \&
                    while pos+i<len(s) and s[pos+i].isspace():
                        post_space += s[pos+i]
                        i += 1

                return LatexToken(tok='macro', arg=macro, pos=pos, len=i,
                                  pre_space=space, post_space=post_space)

            if (s[pos] == '%'):
                # latex comment
                m = re.search(r'(\n|\r|\n\r)\s*', s[pos:])
                mlen = None
                if m is not None:
                    arglen = m.start() # relative to pos already
                    mlen = m.end() # relative to pos already
                    mspace = m.group()
                else:
                    arglen = len(s)-pos# [  ==len(s[pos:])  ]
                    mlen = arglen
                    mspace = ''
                return LatexToken(tok='comment', arg=s[pos+1:pos+arglen], pos=pos, len=mlen,
                                  pre_space=space, post_space=mspace)

            openbracechars = '{'
            closebracechars = '}'
            if not brackets_are_chars:
                openbracechars += '['
                closebracechars += ']'

            if s[pos] in openbracechars:
                return LatexToken(tok='brace_open', arg=s[pos], pos=pos, len=1, pre_space=space)

            if s[pos] in closebracechars:
                return LatexToken(tok='brace_close', arg=s[pos], pos=pos, len=1, pre_space=space)

            # check if it is an inline math char, if we care about inline math.
            if (s[pos] == '$' and self.keep_inline_math):
                # check that we don't have double-$$, which would be a display environment.
                if not (pos+1 < len(s) and s[pos+1] == '$'):
                    return LatexToken(tok='mathmode_inline', arg=s[pos], pos=pos, len=1, pre_space=space)
                # otherwise, proceed to 'char' type.

            return LatexToken(tok='char', arg=s[pos], pos=pos, len=1, pre_space=space)
def get_token ( self , pos , brackets_are_chars = True , environments = True , keep_inline_math = None ) : s = self . s # shorthand with _PushPropOverride ( self , 'keep_inline_math' , keep_inline_math ) : space = '' while ( pos < len ( s ) and s [ pos ] . isspace ( ) ) : space += s [ pos ] pos += 1 if ( space . endswith ( '\n\n' ) ) : # two \n's indicate new paragraph. # Adding pre-space is overkill here I think. return LatexToken ( tok = 'char' , arg = '\n\n' , pos = pos - 2 , len = 2 , pre_space = '' ) if ( pos >= len ( s ) ) : raise LatexWalkerEndOfStream ( ) if ( s [ pos ] == '\\' ) : # escape sequence i = 2 macro = s [ pos + 1 ] # next char is necessarily part of macro # following chars part of macro only if all are alphabetical isalphamacro = False if ( s [ pos + 1 ] . isalpha ( ) ) : isalphamacro = True while pos + i < len ( s ) and s [ pos + i ] . isalpha ( ) : macro += s [ pos + i ] i += 1 # possibly followed by a star if ( pos + i < len ( s ) and s [ pos + i ] == '*' ) : macro += '*' i += 1 # see if we have a begin/end environment if ( environments and ( macro == 'begin' or macro == 'end' ) ) : # \begin{environment} or \end{environment} envmatch = re . match ( r'^\s*\{([\w*]+)\}' , s [ pos + i : ] ) if ( envmatch is None ) : raise LatexWalkerParseError ( s = s , pos = pos , msg = "Bad \\%s macro: expected {environment}" % ( macro ) ) return LatexToken ( tok = ( 'begin_environment' if macro == 'begin' else 'end_environment' ) , arg = envmatch . group ( 1 ) , pos = pos , len = i + envmatch . end ( ) , # !!envmatch.end() counts from pos+i pre_space = space ) # get the following whitespace, and store it in the macro's post_space post_space = '' if isalphamacro : # important, LaTeX does not consume space after non-alpha macros, like \& while pos + i < len ( s ) and s [ pos + i ] . isspace ( ) : post_space += s [ pos + i ] i += 1 return LatexToken ( tok = 'macro' , arg = macro , pos = pos , len = i , pre_space = space , post_space = post_space ) if ( s [ pos ] == '%' ) : # latex comment m = re . search ( r'(\n|\r|\n\r)\s*' , s [ pos : ] ) mlen = None if m is not None : arglen = m . start ( ) # relative to pos already mlen = m . end ( ) # relative to pos already mspace = m . group ( ) else : arglen = len ( s ) - pos # [  ==len(s[pos:])  ] mlen = arglen mspace = '' return LatexToken ( tok = 'comment' , arg = s [ pos + 1 : pos + arglen ] , pos = pos , len = mlen , pre_space = space , post_space = mspace ) openbracechars = '{' closebracechars = '}' if not brackets_are_chars : openbracechars += '[' closebracechars += ']' if s [ pos ] in openbracechars : return LatexToken ( tok = 'brace_open' , arg = s [ pos ] , pos = pos , len = 1 , pre_space = space ) if s [ pos ] in closebracechars : return LatexToken ( tok = 'brace_close' , arg = s [ pos ] , pos = pos , len = 1 , pre_space = space ) # check if it is an inline math char, if we care about inline math. if ( s [ pos ] == '$' and self . keep_inline_math ) : # check that we don't have double-$$, which would be a display environment. if not ( pos + 1 < len ( s ) and s [ pos + 1 ] == '$' ) : return LatexToken ( tok = 'mathmode_inline' , arg = s [ pos ] , pos = pos , len = 1 , pre_space = space ) # otherwise, proceed to 'char' type. return LatexToken ( tok = 'char' , arg = s [ pos ] , pos = pos , len = 1 , pre_space = space )
Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
Parses the latex content given to the constructor ( and stored in self . s ) starting at position pos to parse a single token as defined by : py : class : LatexToken .