tools/traffic_annotation/scripts/annotation_tokenizer.py
def _get_token(self):
    """Return the token here, or None on failure."""
    # Skip initial whitespace.
    pos = self._skip_whitespace()

    # Find the token here, if there's one.
    token = None

    for (token_type, regex) in TOKEN_REGEXEN:
      re_match = regex.match(self.body, pos)
      if re_match:
        token_content = next(g for g in re_match.groups() if g is not None)
        token = Token(token_type, token_content, re_match.end())
        break

    return token
Return the token here, or None on failure.
Return the token here, or None on failure.
Return the token here or None on failure .
def _get_token(self):
    pos = self._skip_whitespace()
    token = None
    for (token_type, regex) in TOKEN_REGEXEN:
      re_match = regex.match(self.body, pos)
      if re_match:
        token_content = next(g for g in re_match.groups() if g is not None)
        token = Token(token_type, token_content, re_match.end())
        break
    return token
def _get_token ( self ) : pos = self . _skip_whitespace ( ) token = None for ( token_type , regex ) in TOKEN_REGEXEN : re_match = regex . match ( self . body , pos ) if re_match : token_content = next ( g for g in re_match . groups ( ) if g is not None ) token = Token ( token_type , token_content , re_match . end ( ) ) break return token
Return the token here, or None on failure.
Return the token here or None on failure .
"""Return the token here, or None on failure.""" # Skip initial whitespace. # Find the token here, if there's one.
self