class_docstr:   def testSimpleTokenCases(self):
      text = ('identifier1:"string1"
       
  
  '
              'identifier2 : 
   
  123  
    identifier3 :'string'
  '
              'identifiER_4 : 1.1e+2 ID5:-0.23 ID6:'aaaa\'bbbb'
  '
              'ID7 : "aa\"bb"
  
  
  
   ID8: {A:inf B:-inf C:true D:false}
  '
              'ID9: 22 ID10: -111111111111111111 ID11: -22
  '
              'ID12: 2222222222222222222 ID13: 1.23456f ID14: 1.2e+2f '
              'false_bool:  0 true_BOOL:t 
   true_bool1:  1 false_BOOL1:f ' )
      tokenizer = text_format._Tokenizer(text)
      methods = [(tokenizer.ConsumeIdentifier, 'identifier1'),
                 ':',
                 (tokenizer.ConsumeString, 'string1'),
                 (tokenizer.ConsumeIdentifier, 'identifier2'),
                 ':',
                 (tokenizer.ConsumeInt32, 123),
                 (tokenizer.ConsumeIdentifier, 'identifier3'),
                 ':',
                 (tokenizer.ConsumeString, 'string'),
                 (tokenizer.ConsumeIdentifier, 'identifiER_4'),
                 ':',
                 (tokenizer.ConsumeFloat, 1.1e+2),
                 (tokenizer.ConsumeIdentifier, 'ID5'),
                 ':',
                 (tokenizer.ConsumeFloat, -0.23),
                 (tokenizer.ConsumeIdentifier, 'ID6'),
                 ':',
                 (tokenizer.ConsumeString, 'aaaa'bbbb'),
                 (tokenizer.ConsumeIdentifier, 'ID7'),
                 ':',
                 (tokenizer.ConsumeString, 'aa"bb'),
                 (tokenizer.ConsumeIdentifier, 'ID8'),
                 ':',
                 '{',
                 (tokenizer.ConsumeIdentifier, 'A'),
                 ':',
                 (tokenizer.ConsumeFloat, float('inf')),
                 (tokenizer.ConsumeIdentifier, 'B'),
                 ':',
                 (tokenizer.ConsumeFloat, -float('inf')),
                 (tokenizer.ConsumeIdentifier, 'C'),
                 ':',
                 (tokenizer.ConsumeBool, True),
                 (tokenizer.ConsumeIdentifier, 'D'),
                 ':',
                 (tokenizer.ConsumeBool, False),
                 '}',
                 (tokenizer.ConsumeIdentifier, 'ID9'),
                 ':',
                 (tokenizer.ConsumeUint32, 22),
                 (tokenizer.ConsumeIdentifier, 'ID10'),
                 ':',
                 (tokenizer.ConsumeInt64, -111111111111111111),
                 (tokenizer.ConsumeIdentifier, 'ID11'),
                 ':',
                 (tokenizer.ConsumeInt32, -22),
                 (tokenizer.ConsumeIdentifier, 'ID12'),
                 ':',
                 (tokenizer.ConsumeUint64, 2222222222222222222),
                 (tokenizer.ConsumeIdentifier, 'ID13'),
                 ':',
                 (tokenizer.ConsumeFloat, 1.23456),
                 (tokenizer.ConsumeIdentifier, 'ID14'),
                 ':',
                 (tokenizer.ConsumeFloat, 1.2e+2),
                 (tokenizer.ConsumeIdentifier, 'false_bool'),
                 ':',
                 (tokenizer.ConsumeBool, False),
                 (tokenizer.ConsumeIdentifier, 'true_BOOL'),
                 ':',
                 (tokenizer.ConsumeBool, True),
                 (tokenizer.ConsumeIdentifier, 'true_bool1'),
                 ':',
                 (tokenizer.ConsumeBool, True),
                 (tokenizer.ConsumeIdentifier, 'false_BOOL1'),
                 ':',
                 (tokenizer.ConsumeBool, False)]
  
      i = 0
      while not tokenizer.AtEnd():
        m = methods[i]
        if type(m) == str:
          token = tokenizer.token
          self.assertEqual(token, m)
          tokenizer.NextToken()
        else:
          self.assertEqual(m[1], m[0]())
        i += 1
  
    def testConsumeIntegers(self):
      # This test only tests the failures in the integer parsing methods as well
      # as the '0' special cases.
      int64_max = (1 << 63) - 1
      uint32_max = (1 << 32) - 1
      text = '-1 %d %d' % (uint32_max + 1, int64_max + 1)
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
      self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint64)
      self.assertEqual(-1, tokenizer.ConsumeInt32())
  
      self.assertRaises(text_format.ParseError, tokenizer.ConsumeUint32)
      self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt32)
      self.assertEqual(uint32_max + 1, tokenizer.ConsumeInt64())
  
      self.assertRaises(text_format.ParseError, tokenizer.ConsumeInt64)
      self.assertEqual(int64_max + 1, tokenizer.ConsumeUint64())
      self.assertTrue(tokenizer.AtEnd())
  
      text = '-0 -0 0 0'
      tokenizer = text_format._Tokenizer(text)
      self.assertEqual(0, tokenizer.ConsumeUint32())
      self.assertEqual(0, tokenizer.ConsumeUint64())
      self.assertEqual(0, tokenizer.ConsumeUint32())
      self.assertEqual(0, tokenizer.ConsumeUint64())
      self.assertTrue(tokenizer.AtEnd())
      
class_docstr_tok: ["def","testSimpleTokenCases","(","self",")",":","text","=","(","'identifier1",":","''","string1","''","'","'identifier2",":","123","identifier3",":","'string'","'","'identifiER_4",":","1.1e+2","ID5",":","-0.23","ID6",":","'aaaa\\'bbbb'","'","'ID7",":","``","aa\\","''","bb","''","ID8",":","{","A",":","inf","B",":","-inf","C",":","true","D",":","false","}","'","'ID9",":","22","ID10",":","-111111111111111111","ID11",":","-22","'","'ID12",":","2222222222222222222","ID13",":","1.23456f","ID14",":","1.2e+2f","'","'false_bool",":","0","true_BOOL",":","t","true_bool1",":","1","false_BOOL1",":","f","'",")","tokenizer","=","text_format._Tokenizer","(","text",")","methods","=","[","(","tokenizer.ConsumeIdentifier",",","'identifier1","'",")",",","'",":","'",",","(","tokenizer.ConsumeString",",","'string1","'",")",",","(","tokenizer.ConsumeIdentifier",",","'identifier2","'",")",",","'",":","'",",","(","tokenizer.ConsumeInt32",",","123",")",",","(","tokenizer.ConsumeIdentifier",",","'identifier3","'",")",",","'",":","'",",","(","tokenizer.ConsumeString",",","'string","'",")",",","(","tokenizer.ConsumeIdentifier",",","'identifiER_4","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","1.1e+2",")",",","(","tokenizer.ConsumeIdentifier",",","'ID5","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","-0.23",")",",","(","tokenizer.ConsumeIdentifier",",","'ID6","'",")",",","'",":","'",",","(","tokenizer.ConsumeString",",","'aaaa'bbbb","'",")",",","(","tokenizer.ConsumeIdentifier",",","'ID7","'",")",",","'",":","'",",","(","tokenizer.ConsumeString",",","'aa","''","bb","'",")",",","(","tokenizer.ConsumeIdentifier",",","'ID8","'",")",",","'",":","'",",","'","{","'",",","(","tokenizer.ConsumeIdentifier",",","'","A","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","float","(","'inf","'",")",")",",","(","tokenizer.ConsumeIdentifier",",","'","B","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","-float","(","'inf","'",")",")",",","(","tokenizer.ConsumeIdentifier",",","'","C","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","True",")",",","(","tokenizer.ConsumeIdentifier",",","'D","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","False",")",",","'","}","'",",","(","tokenizer.ConsumeIdentifier",",","'ID9","'",")",",","'",":","'",",","(","tokenizer.ConsumeUint32",",","22",")",",","(","tokenizer.ConsumeIdentifier",",","'ID10","'",")",",","'",":","'",",","(","tokenizer.ConsumeInt64",",","-111111111111111111",")",",","(","tokenizer.ConsumeIdentifier",",","'ID11","'",")",",","'",":","'",",","(","tokenizer.ConsumeInt32",",","-22",")",",","(","tokenizer.ConsumeIdentifier",",","'ID12","'",")",",","'",":","'",",","(","tokenizer.ConsumeUint64",",","2222222222222222222",")",",","(","tokenizer.ConsumeIdentifier",",","'ID13","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","1.23456",")",",","(","tokenizer.ConsumeIdentifier",",","'ID14","'",")",",","'",":","'",",","(","tokenizer.ConsumeFloat",",","1.2e+2",")",",","(","tokenizer.ConsumeIdentifier",",","'false_bool","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","False",")",",","(","tokenizer.ConsumeIdentifier",",","'true_BOOL","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","True",")",",","(","tokenizer.ConsumeIdentifier",",","'true_bool1","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","True",")",",","(","tokenizer.ConsumeIdentifier",",","'false_BOOL1","'",")",",","'",":","'",",","(","tokenizer.ConsumeBool",",","False",")","]","i","=","0","while","not","tokenizer.AtEnd","(",")",":","m","=","methods","[","i","]","if","type","(","m",")","==","str",":","token","=","tokenizer.token","self.assertEqual","(","token",",","m",")","tokenizer.NextToken","(",")","else",":","self.assertEqual","(","m","[","1","]",",","m","[","0","]","(",")",")","i","+=","1","def","testConsumeIntegers","(","self",")",":","#","This","test","only","tests","the","failures","in","the","integer","parsing","methods","as","well","#","as","the","'","0","'","special","cases",".","int64_max","=","(","1","<","<","63",")","-","1","uint32_max","=","(","1","<","<","32",")","-","1","text","=","'-1","%","d","%","d","'","%","(","uint32_max","+","1",",","int64_max","+","1",")","tokenizer","=","text_format._Tokenizer","(","text",")","self.assertRaises","(","text_format.ParseError",",","tokenizer.ConsumeUint32",")","self.assertRaises","(","text_format.ParseError",",","tokenizer.ConsumeUint64",")","self.assertEqual","(","-1",",","tokenizer.ConsumeInt32","(",")",")","self.assertRaises","(","text_format.ParseError",",","tokenizer.ConsumeUint32",")","self.assertRaises","(","text_format.ParseError",",","tokenizer.ConsumeInt32",")","self.assertEqual","(","uint32_max","+","1",",","tokenizer.ConsumeInt64","(",")",")","self.assertRaises","(","text_format.ParseError",",","tokenizer.ConsumeInt64",")","self.assertEqual","(","int64_max","+","1",",","tokenizer.ConsumeUint64","(",")",")","self.assertTrue","(","tokenizer.AtEnd","(",")",")","text","=","'-0","-0","0","0'","tokenizer","=","text_format._Tokenizer","(","text",")","self.assertEqual","(","0",",","tokenizer.ConsumeUint32","(",")",")","self.assertEqual","(","0",",","tokenizer.ConsumeUint64","(",")",")","self.assertEqual","(","0",",","tokenizer.ConsumeUint32","(",")",")","self.assertEqual","(","0",",","tokenizer.ConsumeUint64","(",")",")","self.assertTrue","(","tokenizer.AtEnd","(",")",")"]
code: def testConsumeByteString(self):
      text = "\"string1'"
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(
          text_format.ParseError, tokenizer.ConsumeByteString
      )
  
      text = 'string1"'
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(
          text_format.ParseError, tokenizer.ConsumeByteString
      )
  
      text = '\n"\\xt"'
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(
          text_format.ParseError, tokenizer.ConsumeByteString
      )
  
      text = '\n"\\"'
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(
          text_format.ParseError, tokenizer.ConsumeByteString
      )
  
      text = '\n"\\x"'
      tokenizer = text_format._Tokenizer(text)
      self.assertRaises(
          text_format.ParseError, tokenizer.ConsumeByteString
      )
  
imports: import difflib
  import re
  import unittest
  from google.protobuf import text_format
  from google.protobuf.internal import test_util
  from google.protobuf import unittest_pb2
  from google.protobuf import unittest_mset_pb2
label_desc: Use image node_id 1 for calling the TokenizerTest obj's underlying member method code with example usage: obj.testConsumeByteString() without return types