text: from itertools import zip_longest, tee
  from typing import Type, Tuple
  
  import pytest
  import codecs
  
  from pytest import raises
  
  from srctools.test.test_property_parser import parse_test as prop_parse_test
  from srctools.property_parser import KeyValError
  from srctools.tokenizer import (
      Token,
      Tokenizer,
      BaseTokenizer,
      Cy_Tokenizer, Py_Tokenizer,
      Cy_BaseTokenizer, Py_BaseTokenizer,
      Cy_IterTokenizer, Py_IterTokenizer,
      escape_text, _py_escape_text,
      TokenSyntaxError, _PUSHBACK_VALS as TOK_VALS,
  )
  
  T = Token
  
  # The correct result of parsing prop_parse_test.
  # Either the token, or token + value (which must be correct).
  prop_parse_tokens = [
      T.NEWLINE,
      T.NEWLINE,
      T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Root1"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Key"), (T.STRING, "Value"), T.NEWLINE,
  
      (T.STRING, "Extra"), (T.STRING, "Spaces"), T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Block"),
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "Empty"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      T.BRACE_CLOSE, T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "Block"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "bare"), T.NEWLINE,
      T.BRACE_OPEN, (T.STRING, "block"), (T.STRING, "he\tre"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "Root2"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "Name with \" in it"), (T.STRING, "Value with \" inside"), T.NEWLINE,
      (T.STRING, "multiline"), (T.STRING, 'text\n\tcan continue\nfor many "lines" of\n  possibly indented\n\ntext'), T.NEWLINE,
      (T.STRING, "Escapes"), (T.STRING, '\t \n \\d'), T.NEWLINE,
      (T.STRING, "Oneliner"), T.BRACE_OPEN, (T.STRING, 'name'), (T.STRING, 'value'), T.BRACE_CLOSE, T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "CommentChecks"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "after "), (T.STRING, "value"), T.NEWLINE,
      (T.STRING, "FlagBlocks"), (T.STRING, "This"), (T.PROP_FLAG, "test_disabled"), T.NEWLINE,
      (T.STRING, "Flag"), (T.STRING, "allowed"), (T.PROP_FLAG, "!test_disabled"), T.NEWLINE,
      (T.STRING, "FlagAllows"), (T.STRING, "This"), (T.PROP_FLAG, "test_enabled"), T.NEWLINE,
      (T.STRING, "Flag"), (T.STRING, "blocksthis"), (T.PROP_FLAG, "!test_enabled"), T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "shouldbe"), T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "toreplace"), (T.PROP_FLAG, "test_enabled"), T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "alsothis"), (T.PROP_FLAG, "test_enabled"), T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "shouldbe2"), T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "toreplace2"), (T.PROP_FLAG, "!test_disabled"), T.NEWLINE,
      (T.STRING, "Replaced"), (T.STRING, "alsothis2"), (T.PROP_FLAG, "!test_disabled"), T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Replaced"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "shouldbe3"), (T.STRING, "replaced3"), T.NEWLINE,
      (T.STRING, "prop2"), (T.STRING, "blah"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "Replaced"), (T.PROP_FLAG, "test_enabled"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "lambda"), (T.STRING, "should"), T.NEWLINE,
      (T.STRING, "replace"), (T.STRING, "above"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      T.NEWLINE,
      (T.STRING, "Replaced"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "shouldbe4"), (T.STRING, "replaced4"), T.NEWLINE,
      (T.STRING, "prop2"), (T.STRING, "blah"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "Replaced"), (T.PROP_FLAG, "!test_disabled"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "lambda2"), (T.STRING, "should2"), T.NEWLINE,
      (T.STRING, "replace2"), (T.STRING, "above2"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "otherval"), (T.STRING, "blah"), T.NEWLINE,
      (T.STRING, "shouldnotreplace"), (T.PROP_FLAG, "test_enabled"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "key"), (T.STRING, "value1"), T.NEWLINE,
      (T.STRING, "key"), (T.STRING, "value2"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      (T.STRING, "skipped"), (T.PROP_FLAG, "test_disabled"), T.NEWLINE,
      T.BRACE_OPEN, T.NEWLINE,
      (T.STRING, "ignore"), (T.STRING, "value"), T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
      T.BRACE_CLOSE, T.NEWLINE,
  ]
  
  # Additional text not valid as a property.
  noprop_parse_test = """
  #letter_abcdefghijklmnopqrstuvwxyz_ABCDEFGHIJKLMNOPQRSTUVWXYZ
  #ﬁmport test
  #EXclßÀde value
  #caseA\u0345\u03a3test
  { ]]{ }}}[[ {{] + = :: "test" + "ing" == vaLUE
  """
  
  noprop_parse_tokens = [
      T.NEWLINE,
      (T.DIRECTIVE, "letter_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz"), T.NEWLINE,
      (T.DIRECTIVE, "fimport"), (T.STRING, "test"), T.NEWLINE,
      (T.DIRECTIVE, "exclssàde"), (T.STRING, "value"), T.NEWLINE,
      (T.DIRECTIVE, "casea\u03b9\u03c3test"), T.NEWLINE,
      T.BRACE_OPEN, T.BRACK_CLOSE, T.BRACK_CLOSE, T.BRACE_OPEN, T.BRACE_CLOSE, T.BRACE_CLOSE, T.BRACE_CLOSE,
      T.BRACK_OPEN, T.BRACK_OPEN, T.BRACE_OPEN, T.BRACE_OPEN, T.BRACK_CLOSE, T.PLUS,
      T.EQUALS, T.COLON, T.COLON, (T.STRING, "test"), T.PLUS, (T.STRING, "ing"),
      T.EQUALS, T.EQUALS, (T.STRING, "vaLUE"), T.NEWLINE
  ]
  
  
  if Cy_Tokenizer is not Py_Tokenizer:
      parms = [Cy_Tokenizer, Py_Tokenizer]
      ids = ['Cython', 'Python']
  else:
      import srctools.tokenizer
      print('No _tokenizer! ' + str(vars(srctools.tokenizer)))
      parms = [Py_Tokenizer]
      ids = ['Python']
  
  
  @pytest.fixture(params=parms, ids=ids)
  def py_c_token(request):
      """Run the test twice, for the Python and C versions."""
      yield request.param
  
  del parms, ids
  
  
  def check_tokens(tokenizer, tokens):
      """Check the tokenizer produces the given tokens.
  
      The arguments are either (token, value) tuples or tokens.
      """
      # Don't show in pytest tracebacks.
      __tracebackhide__ = True
  
      sentinel = object()
      tokenizer_iter, tokenizer_backup = tee(tokenizer, 2)
      tok_test_iter = iter(tokens)
      for i, (token, comp_token) in enumerate(zip_longest(tokenizer_iter, tok_test_iter, fillvalue=sentinel), start=1):
          # Check if either is too short - we need zip_longest() for that.
          if token is sentinel:
              pytest.fail('{}: Tokenizer ended early - needed {}, got {}!'.format(
                  i,
                  [comp_token] + list(tok_test_iter),
                  list(tokenizer_backup),
              ))
          if comp_token is sentinel:
              pytest.fail('{}: Tokenizer had too many values - extra = {}!'.format(
                  i,
                  [token] + list(tokenizer_iter),
              ))
          assert len(token) == 2
          assert isinstance(token, tuple)
          if isinstance(comp_token, tuple):
              comp_type, comp_value = comp_token
              assert comp_type is token[0], "got {}, expected {} @ pos {}={}".format(token[0], comp_type, i, tokens[i-2: i+1])
              assert comp_value == token[1], "got {!r}, expected {!r} @ pos {}={}".format(token[1], comp_value, i, tokens[i-2: i+1])
          else:
              assert token[0] is comp_token, "got {}, expected {} @ pos {}={}".format(token[0], comp_token, i, tokens[i-2: i+1])
  
  
  def test_prop_tokens(py_c_token):
      """Test the tokenizer returns the correct sequence of tokens for this test string."""
      Tokenizer = py_c_token
  
      tok = Tokenizer(prop_parse_test, '', string_bracket=True)
      check_tokens(tok, prop_parse_tokens)
  
      # Test a list of lines.
      test_list = prop_parse_test.splitlines(keepends=True)
      # Break this line up semi-randomly, to help test the chunk code.
      test_list[27:28] = [''] + list(test_list[27].partition('"')) + ['', '', '']
      # They should be the same text though!
      assert ''.join(test_list) == prop_parse_test, "Bad test code!"
  
      # Test an iterator.
      tok = Tokenizer(iter(prop_parse_test.splitlines(keepends=True)), '', string_bracket=True)
      check_tokens(tok, prop_parse_tokens)
  
      tok = Tokenizer(test_list, '', string_bracket=True)
      check_tokens(tok, prop_parse_tokens)
  
      # Test a special case - empty chunks at the end.
      test_list += ['', '', '']
  
      tok = Tokenizer(test_list, '', string_bracket=True)
      check_tokens(tok, prop_parse_tokens)
  
  
  def test_nonprop_tokens(py_c_token):
      """Test the tokenizer returns the correct sequence of tokens for non-Property strings."""
      Tokenizer = py_c_token
  
      tok = Tokenizer(noprop_parse_test, '')
      check_tokens(tok, noprop_parse_tokens)
  
      # Test a list of lines.
      test_list = noprop_parse_test.splitlines(keepends=True)
  
      tok = Tokenizer(test_list, '')
      check_tokens(tok, noprop_parse_tokens)
  
      # Test a special case - empty chunks at the end.
      test_list += ['', '', '']
  
      tok = Tokenizer(test_list, '')
      check_tokens(tok, noprop_parse_tokens)
  
  
  def test_pushback(py_c_token):
      """Test pushing back tokens."""
      Tokenizer = py_c_token
      tok = Tokenizer(prop_parse_test, '', string_bracket=True)
      tokens = []
      for i, (tok_type, tok_value) in enumerate(tok):
          if i % 3 == 0:
              tok.push_back(tok_type, tok_value)
          else:
              tokens.append((tok_type, tok_value))
      check_tokens(tokens, prop_parse_tokens)
  
      tok = Tokenizer(noprop_parse_test, '')
      tokens = []
      for i, (tok_type, tok_value) in enumerate(tok):
          if i % 3 == 0:
              tok.push_back(tok_type, tok_value)
          else:
              tokens.append((tok_type, tok_value))
      check_tokens(tokens, noprop_parse_tokens)
  
  
  @pytest.mark.parametrize('token, val', [
      (Token.EOF, ''),
      (Token.NEWLINE, '\n'),
      (Token.BRACE_OPEN, '{'),
      (Token.BRACE_CLOSE, '}'),
      (Token.BRACK_OPEN, '['),
      (Token.BRACK_CLOSE, ']'),
      (Token.COLON, ':'),
      (Token.EQUALS, '='),
      (Token.PLUS, '+'),
      (Token.COMMA, ','),
  ])
  def test_pushback_opvalues(py_c_token, token: Token, val: str) -> None:
      """Test the operator tokens pushback the correct fixed value."""
      tok: Tokenizer = py_c_token(['test data'], string_bracket=False)
      tok.push_back(token, val)
      assert tok() == (token, val)
  
      tok.push_back(Token.STRING, 'push')
      with pytest.raises(ValueError):  # Can only push back one at a time.
          tok.push_back(Token.STRING, 'two_at_once')
      assert tok() == (Token.STRING, 'push')
  
      # Value is ignored for these token types.
      tok.push_back(token, 'another_val')
      assert tok() == (token, val)
  
  
  def test_call_next(py_c_token) -> None:
      """Test that tok() functions, and it can be mixed with iteration."""
      tok: Tokenizer = py_c_token('''{ "test" } "test" { + } ''', 'file')
  
      tok_type, tok_value = tok()
      assert tok_type is Token.BRACE_OPEN and tok_value == '{'
  
      it1 = iter(tok)
  
      assert next(it1) == (Token.STRING, "test")
      assert tok() == (Token.BRACE_CLOSE, '}')
      assert next(it1) == (Token.STRING, "test")
      assert next(it1) == (Token.BRACE_OPEN, '{')
      assert tok() == (Token.PLUS, '+')
      # Another iterator doesn't restart.
      assert next(iter(tok)) == (Token.BRACE_CLOSE, '}')
      assert tok() == (Token.EOF, '')
  
      with pytest.raises(StopIteration):
          next(it1)
  
  
  def test_star_comments(py_c_token):
      """Test disallowing /* */ comments."""
      Tokenizer = py_c_token
  
      text = '''\
      "blah"
          {
          "a" "b"
      /*
          "c" "d"
          }
      "second"
          {
      */
          }
      '''
  
      with pytest.raises(TokenSyntaxError):
          # Default = false
          for tok, tok_value in py_c_token(text):
              pass
  
      with pytest.raises(TokenSyntaxError):
          for tok, tok_value in py_c_token(text, allow_star_comments=False):
              pass
  
  
      check_tokens(py_c_token(text, allow_star_comments=True), [
          (Token.STRING, "blah"), Token.NEWLINE,
          Token.BRACE_OPEN, Token.NEWLINE,
          (Token.STRING, "a"), (Token.STRING, "b"), Token.NEWLINE,
          Token.NEWLINE,
          Token.BRACE_CLOSE, Token.NEWLINE,
      ])
  
      # Test with one string per chunk:
      for tok, tok_value in py_c_token(list(text), allow_star_comments=True):
          pass
  
      # Check line number is correct.
      tokenizer = py_c_token(text, allow_star_comments=True)
      for tok, tok_value in tokenizer:
          if tok is Token.BRACE_CLOSE:
              assert 10 == tokenizer.line_num
  
      # Check unterminated comments are invalid.
      with pytest.raises(TokenSyntaxError):
          for tok, tok_value in py_c_token(text.replace('*/', ''), allow_star_comments=True):
              pass
  
      # Test some edge cases with multiple asterisks:
      for tok, tok_value in py_c_token('"blah"\n/**/', allow_star_comments=True):
          pass
  
      for tok, tok_value in py_c_token('"blah"\n/*\n **/', allow_star_comments=True):
          pass
  
  
  def test_bom(py_c_token):
      """Test skipping a UTF8 BOM at the beginning."""
      Tokenizer = py_c_token
  
      bom = codecs.BOM_UTF8.decode('utf8')
  
      text = bom + '''\
  "blah"
    {
    "tes__t " "2"
      }
  '''.replace('__', bom)  # Check the BOM can be inside the contents.
  
      tokens = [
          (T.STRING, "blah"), T.NEWLINE,
          T.BRACE_OPEN, T.NEWLINE,
          (T.STRING, "tes" + bom + "t "), (T.STRING, "2"), T.NEWLINE,
          T.BRACE_CLOSE, T.NEWLINE,
      ]
  
      # Check without chunks.
      tok = Tokenizer(text, '')
      check_tokens(tok, tokens)
  
      # And with chunks.
      tok = Tokenizer(list(text), '')
      check_tokens(tok, tokens)
  
      # Test edge cases - characters with the first 1 or 2 bytes of the 3-byte
      # BOM should be kept intact.\xef\xbb\xbf
      matches_1 = b'\xef\xbe\xae'.decode('utf8') + '_test'
      matches_2 = b'\xef\xbb\xae'.decode('utf8') + '_test'
      assert matches_1.encode('utf8')[:1] == codecs.BOM_UTF8[:1]
      assert matches_2.encode('utf8')[:2] == codecs.BOM_UTF8[:2]
  
      assert next(Tokenizer(bom + 'test')) == (Token.STRING, 'test')
      assert next(Tokenizer(matches_1)) == (Token.STRING, matches_1)
      assert next(Tokenizer(matches_2)) == (Token.STRING, matches_2)
  
      # Test strings that are less than 3 bytes long.
      check_tokens(Tokenizer(['e']), [(Token.STRING, 'e')])
      check_tokens(Tokenizer(['e', 'x']), [(Token.STRING, 'ex')])
      check_tokens(Tokenizer(['e', ' ', 'f']), [(Token.STRING, 'e'), (Token.STRING, 'f')])
  
  
  def test_constructor(py_c_token):
      """Test various argument syntax for the tokenizer."""
      Tokenizer = py_c_token
  
      Tokenizer('blah')
      Tokenizer('blah', None)
      Tokenizer('blah', '', TokenSyntaxError)
      Tokenizer('blah', '', KeyValError, True)
      Tokenizer('blah', error=KeyValError)
      Tokenizer(['blah', 'blah'], string_bracket=True)
  
  
  def test_tok_filename(py_c_token):
      """Test that objects other than a direct string can be passed as filename."""
      Tokenizer = py_c_token
  
      class AFilePath:
          """Test path object support."""
          def __fspath__(self) -> str:
              return "path/to/file.vdf"
  
      class SubStr(str):
          """Subclasses should be accepted."""
  
      assert Tokenizer('blah', AFilePath()).filename == 'path/to/file.vdf'
      tok = Tokenizer('blah', SubStr('test/path.txt'))
      assert tok.filename == 'test/path.txt'
      assert isinstance(tok.filename, str)
  
      assert Tokenizer('file', b'binary/filename\xE3\x00.txt').filename == 'binary/filename\\xe3\\x00.txt'
  
  
  def test_obj_config(py_c_token):
      """Test getting and setting configuration attributes."""
      Tokenizer = py_c_token
  
      assert Tokenizer('').string_bracket is False
      assert Tokenizer('').allow_escapes is True
  
      assert Tokenizer('', string_bracket=1).string_bracket is True
      assert Tokenizer('', string_bracket=True).string_bracket is True
      assert Tokenizer('', string_bracket=False).string_bracket is False
      assert Tokenizer('', allow_escapes=[]).allow_escapes is False
      assert Tokenizer('', allow_escapes=True).allow_escapes is True
      assert Tokenizer('', allow_escapes=False).allow_escapes is False
  
      tok = Tokenizer('')
      assert tok.string_bracket is False
      tok.string_bracket = True
      assert tok.string_bracket is True
      tok.string_bracket = False
      assert tok.string_bracket is False
  
      tok = Tokenizer('')
      assert tok.allow_escapes is True
      tok.allow_escapes = False
      assert tok.allow_escapes is False
      tok.allow_escapes = True
      assert tok.allow_escapes is True
  
  
  @pytest.mark.parametrize('inp, out', [
      ('', ''),
      ("hello world", "hello world"),
      ("\thello_world", r"\thello_world"),
      ("\\thello_world", r"\\thello_world"),
      ("\\ttest\nvalue\t\\r\t\n", r"\\ttest\nvalue\t\\r\t\n"),
      # BMP characters, and some multiplane chars.
      ('test: ╒══╕', r'test: ╒══╕'),
      ("♜♞🤐♝♛🥌 chess: ♚♝♞♜", "♜♞🤐♝♛🥌 chess: ♚♝♞♜"),
      ('\t"╒═\\═╕"\n', r'\t\"╒═\\═╕\"\n'),
      ("\t♜♞\\🤐♝♛🥌♚♝\\\\♞\n♜", r"\t♜♞\\🤐♝♛🥌♚♝\\\\♞\n♜"),
  ])
  @pytest.mark.parametrize('func', [_py_escape_text, escape_text], ids=['Py', 'Cy'])
  def test_escape_text(inp: str, out: str, func) -> None:
      """Test the Python and C escape_text() functions."""
      assert func(inp) == out
      if inp == out:  # If the same, reuses the string.
          assert func(inp) is inp
  
  
  def test_brackets(py_c_token):
      """Test the [] handling produces the right results."""
      check_tokens(py_c_token('"blah" [ !!text45() ]', string_bracket=True), [
          (Token.STRING, "blah"),
          (Token.PROP_FLAG, " !!text45() ")
      ])
  
      check_tokens(py_c_token('"blah" [ !!text45() ]', string_bracket=False), [
          (Token.STRING, "blah"),
          Token.BRACK_OPEN,
          (Token.STRING, "!!text45"),
          (Token.PAREN_ARGS, ''),
          Token.BRACK_CLOSE,
      ])
  
      # Without, we don't care about brackets.
      check_tokens(py_c_token('[ unclosed {}', string_bracket=False), [
          Token.BRACK_OPEN,
          (Token.STRING, 'unclosed'),
          Token.BRACE_OPEN,
          Token.BRACE_CLOSE,
      ])
  
      # Corner case, check EOF while reading the string.
      check_tokens(py_c_token('[ unclosed', string_bracket=False), [
          Token.BRACK_OPEN,
          (Token.STRING, 'unclosed'),
      ])
  
      # Without, we don't care about brackets.
      check_tokens(py_c_token('unopened ]', string_bracket=False), [
          (Token.STRING, 'unopened'),
          Token.BRACK_CLOSE,
      ])
  
  def test_invalid_bracket(py_c_token):
      """Test detecting various invalid combinations of [] brackets."""
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('[ unclosed', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('unopened ]', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('[ok] bad ]', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('[ no [ nesting ] ]', string_bracket=True):
              pass
  
  
  def test_invalid_paren(py_c_token):
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('( unclosed', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('unopened )', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('(ok) bad )', string_bracket=True):
              pass
  
      with raises(TokenSyntaxError):
          for tok, tok_value in py_c_token('( no ( nesting ) )', string_bracket=True):
              pass
  
  
  def test_token_syntax_error() -> None:
      """Test the TokenSyntaxError class."""
      # There's no C version - if we're erroring, we don't care about
      # performance much.
  
      # This pretty much just needs to return the right repr() and str(),
      # and be an Exception.
  
      assert issubclass(TokenSyntaxError, Exception)
  
      err = TokenSyntaxError('test message', None, None)
      assert repr(err) == "TokenSyntaxError('test message', None, None)"
      assert err.mess == 'test message'
      assert err.file is None
      assert err.line_num is None
      assert str(err) == '''test message'''
  
      err = TokenSyntaxError('test message', 'a file', None)
      assert repr(err) == "TokenSyntaxError('test message', 'a file', None)"
      assert err.mess == 'test message'
      assert err.file == 'a file'
      assert err.line_num is None
      assert str(err) == '''test message
  Error occurred with file "a file".'''
  
      err = TokenSyntaxError('test message', 'a file', 45)
      assert repr(err) == "TokenSyntaxError('test message', 'a file', 45)"
      assert err.mess == 'test message'
      assert err.file == 'a file'
      assert err.line_num == 45
      assert str(err) == '''test message
  Error occurred on line 45, with file "a file".'''
  
      err = TokenSyntaxError('test message', None, 250)
      assert repr(err) == "TokenSyntaxError('test message', None, 250)"
      assert err.mess == 'test message'
      assert err.file is None
      assert err.line_num == 250
      assert str(err) == '''test message
  Error occurred on line 250.'''
  
  
  def test_tok_error(py_c_token):
      """Test the tok.error() helper."""
      tok: Tokenizer = py_c_token(['test'], 'filename.py')
      tok.line_num = 45
      assert tok.error('basic') == TokenSyntaxError('basic', 'filename.py', 45)
      assert tok.error('Error with } and { brackets') == TokenSyntaxError('Error with } and { brackets', 'filename.py', 45)
      tok.line_num = 9999
      assert tok.error('Arg {0}, {2} and {1} formatted', 'a', 'b', 'c') == TokenSyntaxError('Arg a, c and b formatted', 'filename.py', 9999)
      tok.filename = None
      assert tok.error('Param: {:.6f}, {!r}, {}', 1/3, "test", test_tok_error) == TokenSyntaxError(f"Param: {1/3:.6f}, 'test', {test_tok_error}", None, 9999)
  
  
  @pytest.mark.parametrize('token', [
      Token.STRING,
      Token.PROP_FLAG,
      Token.PAREN_ARGS,
      Token.DIRECTIVE,
  ])
  def test_tok_error_hasval(py_c_token, token):
      """Test the tok.error() handler with token types that accept values."""
      tok: Tokenizer = py_c_token(['test'], 'fname')
      tok.line_num = 23
      assert tok.error(token) == TokenSyntaxError(f'Unexpected token {token.name}!', 'fname', 23)
      assert tok.error(token, 'value') == TokenSyntaxError(f'Unexpected token {token.name}(value)!', 'fname', 23)
      with pytest.raises(TypeError):
          tok.error(token, 'val1', 'val2')
  
  
  @pytest.mark.parametrize('token', [
      Token.EOF, Token.NEWLINE,
      Token.BRACE_OPEN, Token.BRACE_CLOSE,
      Token.BRACK_OPEN, Token.BRACK_CLOSE,
      Token.COLON, Token.EQUALS, Token.PLUS,
  ])
  def test_tok_error_noval(py_c_token, token):
      """Test the tok.error() handler with token types that have the same value always."""
      tok: Tokenizer = py_c_token(['test'], 'fname')
      tok.line_num = 23
      assert tok.error(token) == TokenSyntaxError(f'Unexpected token {token.name}!', 'fname', 23)
      assert tok.error(token, 'value') == TokenSyntaxError(f'Unexpected token {token.name}!', 'fname', 23)
      with pytest.raises(TypeError):
          tok.error(token, 'val1', 'val2')
  
  
  def test_unicode_error_wrapping(py_c_token):
      """Test that Unicode errors are wrapped into TokenSyntaxError."""
      def raises_unicode():
          yield "line of_"
          yield "text\n"
          raise UnicodeDecodeError('utf8', bytes(100), 1, 2, 'reason')
  
      tok = py_c_token(raises_unicode())
      assert tok() == (Token.STRING, "line")
      assert tok() == (Token.STRING, "of_text")
      with pytest.raises(TokenSyntaxError) as exc_info:
          list(tok)
      assert isinstance(exc_info.value.__cause__, UnicodeDecodeError)
  
  
  def test_early_binary_arg(py_c_token):
      """Test that passing bytes values is caught before looping."""
      with pytest.raises(TypeError):
          py_c_token(b'test')
  
  def test_block_iter(py_c_token):
      """Test the Tokenizer.block() helper."""
      # First two correct usages:
      tok = py_c_token('''\
      "test"
      {
      "blah" "value"
      
      "another" "value"
      }
      ''')
      assert tok() == (Token.STRING, "test")
      bl = tok.block("tester")
      assert next(bl) == "blah"
      assert next(bl) == "value"
      assert next(bl) == "another"
      assert next(bl) == "value"
      with raises(StopIteration):
          next(bl)
  
      tok = py_c_token(' "blah" "value" } ')
      bl = tok.block("tester", False)
      assert next(bl) == "blah"
      assert next(bl) == "value"
      with raises(StopIteration):
          next(bl)
  
      # Completes correctly with no values.
      assert list(py_c_token('{}').block('')) == []
  
      # We can remove tokens halfway through on the original tokenizer.
      tok = py_c_token(' { \n\n"legal" { + } block } ')
      bl = tok.block("test")
      assert next(bl) == 'legal'
      assert tok() == (Token.BRACE_OPEN, '{')
      assert tok() == (Token.PLUS, '+')
      assert tok() == (Token.BRACE_CLOSE, '}')
      assert next(bl) == 'block'
      with raises(StopIteration):
          next(bl)
  
      # Now errors.
  
      # Not an open brace, also it must defer to the first next() call.
      b = py_c_token(' hi ').block('blah', consume_brace=True)
      with raises(TokenSyntaxError):
          next(b)
  
      # Also with implicit consume_brace
      b = py_c_token(' hi ').block('blah')
      with raises(TokenSyntaxError):
          next(b)
  
      # Open brace where there shouldn't.
      b = py_c_token('{').block('blah', consume_brace=False)
      with raises(TokenSyntaxError):
          next(b)
  
      # Two open braces, only consume one.
      b = py_c_token('{ {').block('blah')
      with raises(TokenSyntaxError):
          next(b)
  
      # And one in the middle.
      b = py_c_token('{ "test" { "never-here" } ').block('blah')
      assert next(b) == "test"
      with raises(TokenSyntaxError):
          next(b)
  
      # Running off the end uses the block in the result.
      b = py_c_token('{ blah "blah" ').block('SpecialBlockName')
      assert next(b) == "blah"
      assert next(b) == "blah"
      with raises(TokenSyntaxError, match='SpecialBlockName'):
          next(b)
  
  
  @pytest.mark.parametrize(
      'Tok',
      [Cy_BaseTokenizer, Py_BaseTokenizer],
      ids=['Cython', 'Python'],
  )
  def test_subclass_base(Tok: Type[BaseTokenizer]) -> None:
      """Test subclassing of the base tokenizer."""
      class Sub(Tok):
          def __init__(self, tok):
              super().__init__('filename', TokenSyntaxError)
              self.__tokens = iter(tok)
  
          def _get_token(self) -> Tuple[Token, str]:
              try:
                  tok = next(self.__tokens)
              except StopIteration:
                  return Token.EOF, ''
              if isinstance(tok, tuple):
                  return tok
              else:
                  return tok, TOK_VALS[tok]
  
      check_tokens(Sub(noprop_parse_tokens), noprop_parse_tokens)
  
      # Do some peeks, pushbacks, etc to check they work.
      tok = Sub(prop_parse_tokens)
      it = iter(tok)
      assert next(it) == (Token.NEWLINE, '\n')
      assert next(tok.skipping_newlines()) == (Token.STRING, 'Root1')
      tok.push_back(Token.DIRECTIVE, '#test')
      assert tok() == (Token.DIRECTIVE, '#test')
      assert tok() == (Token.NEWLINE, '\n')
      assert tok.peek() == (Token.BRACE_OPEN, '{')
      assert tok.peek() == (Token.BRACE_OPEN, '{')
      assert next(iter(tok)) == (Token.BRACE_OPEN, '{')
      skip = tok.skipping_newlines()
      assert next(skip) == (Token.STRING, 'Key')
      assert next(it) == (Token.STRING, 'Value')
      assert next(skip) == (Token.STRING, 'Extra')
      assert next(skip) == (Token.STRING, 'Spaces')
      assert next(tok.skipping_newlines()) == (Token.STRING, 'Block')
  