


'''simple docstring'''
import unittest

from parameterized import parameterized

from transformers import AutoTokenizer, GPTNeoXConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      GPTNeoXForCausalLM,
      GPTNeoXForQuestionAnswering,
      GPTNeoXForSequenceClassification,
      GPTNeoXForTokenClassification,
      GPTNeoXModel,
  )
class      lowercase			:

  """simple docstring"""

  def __init__(	self					,     UpperCamelCase_					,     UpperCamelCase_=13					,     UpperCamelCase_=7					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=True					,     UpperCamelCase_=99					,     UpperCamelCase_=64					,     UpperCamelCase_=5					,     UpperCamelCase_=4					,     UpperCamelCase_=37					,     UpperCamelCase_="gelu"					,     UpperCamelCase_=0.1					,     UpperCamelCase_=0.1					,     UpperCamelCase_=512					,     UpperCamelCase_=16					,     UpperCamelCase_=2					,     UpperCamelCase_=0.02					,     UpperCamelCase_=3					,     UpperCamelCase_=4					,     UpperCamelCase_=None					,     ):

       '''simple docstring'''


       UpperCamelCase__     :Any							      =				parent
       UpperCamelCase__     :Any							      =				batch_size
       UpperCamelCase__     :List[Any]							      =				seq_length
       UpperCamelCase__     :Optional[int]							      =				is_training
       UpperCamelCase__     :str							      =				use_input_mask
       UpperCamelCase__     :Union[str, Any]							      =				use_token_type_ids
       UpperCamelCase__     :Union[str, Any]							      =				use_labels
       UpperCamelCase__     :List[str]							      =				vocab_size
       UpperCamelCase__     :Union[str, Any]							      =				hidden_size
       UpperCamelCase__     :Tuple							      =				num_hidden_layers
       UpperCamelCase__     :List[str]							      =				num_attention_heads
       UpperCamelCase__     :Any							      =				intermediate_size
       UpperCamelCase__     :List[Any]							      =				hidden_act
       UpperCamelCase__     :Optional[Any]							      =				hidden_dropout_prob
       UpperCamelCase__     :Union[str, Any]							      =				attention_probs_dropout_prob
       UpperCamelCase__     :List[Any]							      =				max_position_embeddings
       UpperCamelCase__     :int							      =				type_vocab_size
       UpperCamelCase__     :Union[str, Any]							      =				type_sequence_label_size
       UpperCamelCase__     :Dict							      =				initializer_range
       UpperCamelCase__     :Any							      =				num_labels
       UpperCamelCase__     :List[str]							      =				num_choices
       UpperCamelCase__     :List[Any]							      =				scope
       UpperCamelCase__     :Dict							      =				vocab_size - 1




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Any							      =				ids_tensor([self.batch_size, self.seq_length]					,     self.vocab_size							)

       UpperCamelCase__     :Dict							      =				None
       if self.use_input_mask:
            UpperCamelCase__     :List[Any]							      =				random_attention_mask([self.batch_size, self.seq_length]							)

       UpperCamelCase__     :Any							      =				None
       if self.use_labels:
            UpperCamelCase__     :Dict							      =				ids_tensor([self.batch_size, self.seq_length]					,     self.num_labels							)

       UpperCamelCase__     :Dict							      =				self.get_config()

       return config, input_ids, input_mask, token_labels




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       return GPTNeoXConfig(
           vocab_size=self.vocab_size					,     hidden_size=self.hidden_size					,     num_hidden_layers=self.num_hidden_layers					,     num_attention_heads=self.num_attention_heads					,     intermediate_size=self.intermediate_size					,     hidden_act=self.hidden_act					,     hidden_dropout_prob=self.hidden_dropout_prob					,     attention_probs_dropout_prob=self.attention_probs_dropout_prob					,     max_position_embeddings=self.max_position_embeddings					,     type_vocab_size=self.type_vocab_size					,     is_decoder=lowercase__					,     initializer_range=self.initializer_range					,     pad_token_id=self.pad_token_id					,     )




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Optional[int]							      =				self.prepare_config_and_inputs()

       UpperCamelCase__     :List[str]							      =				True

       return config, input_ids, input_mask, token_labels




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :List[Any]							      =				GPTNeoXModel(config=lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :List[str]							      =				model(lowercase__					,     attention_mask=lowercase__							)
       UpperCamelCase__     :List[str]							      =				model(lowercase__							)
       self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Union[str, Any]							      =				True
       UpperCamelCase__     :int							      =				GPTNeoXModel(lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :str							      =				model(lowercase__					,     attention_mask=lowercase__							)
       self.parent.assertEqual(result.last_hidden_state.shape					,     (self.batch_size, self.seq_length, self.hidden_size)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Optional[int]							      =				GPTNeoXForCausalLM(config=lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :List[str]							      =				model(lowercase__					,     attention_mask=lowercase__					,     labels=lowercase__							)
       self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.vocab_size)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Optional[Any]							      =				self.num_labels
       UpperCamelCase__     :Dict							      =				GPTNeoXForQuestionAnswering(lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :int							      =				model(lowercase__					,     attention_mask=lowercase__							)
       self.parent.assertEqual(result.start_logits.shape					,     (self.batch_size, self.seq_length)							)
       self.parent.assertEqual(result.end_logits.shape					,     (self.batch_size, self.seq_length)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Union[str, Any]							      =				self.num_labels
       UpperCamelCase__     :Any							      =				GPTNeoXForSequenceClassification(lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :List[str]							      =				ids_tensor([self.batch_size]					,     self.type_sequence_label_size							)
       UpperCamelCase__     :Union[str, Any]							      =				model(lowercase__					,     attention_mask=lowercase__					,     labels=lowercase__							)
       self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.num_labels)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :List[Any]							      =				self.num_labels
       UpperCamelCase__     :List[Any]							      =				GPTNeoXForTokenClassification(lowercase__							)
       model.to(lowercase__							)
       model.eval()
       UpperCamelCase__     :Any							      =				model(lowercase__					,     attention_mask=lowercase__					,     labels=lowercase__							)
       self.parent.assertEqual(result.logits.shape					,     (self.batch_size, self.seq_length, self.num_labels)							)




  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Dict							      =				True
       UpperCamelCase__     :Dict							      =				GPTNeoXForCausalLM(config=lowercase__							)
       model.to(lowercase__							)
       model.eval()

       # first forward pass
       UpperCamelCase__     :Tuple							      =				model(lowercase__					,     attention_mask=lowercase__					,     use_cache=lowercase__							)
       UpperCamelCase__     :Optional[int]							      =				outputs.past_key_values

       # create hypothetical multiple next token and extent to next_input_ids
       UpperCamelCase__     :Any							      =				ids_tensor((self.batch_size, 3)					,     config.vocab_size							)
       UpperCamelCase__     :List[Any]							      =				ids_tensor((self.batch_size, 3)					,     vocab_size=2							)

       # append to next input_ids and
       UpperCamelCase__     :int							      =				torch.cat([input_ids, next_tokens]					,     dim=-1							)
       UpperCamelCase__     :List[Any]							      =				torch.cat([input_mask, next_mask]					,     dim=-1							)

       UpperCamelCase__     :Any							      =				model(lowercase__					,     attention_mask=lowercase__					,     output_hidden_states=lowercase__							)
       UpperCamelCase__     :Dict							      =				output_from_no_past['''hidden_states'''][0]
       UpperCamelCase__     :int							      =				model(
           lowercase__					,     attention_mask=lowercase__					,     past_key_values=lowercase__					,     output_hidden_states=lowercase__					,     )['''hidden_states'''][0]

       # select random slice
       UpperCamelCase__     :Union[str, Any]							      =				ids_tensor((1,)					,     output_from_past.shape[-1]							).item()
       UpperCamelCase__     :str							      =				output_from_no_past[:, -3:, random_slice_idx].detach()
       UpperCamelCase__     :Optional[int]							      =				output_from_past[:, :, random_slice_idx].detach()

       self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]							)

       # test that outputs are equal for slice
       self.parent.assertTrue(torch.allclose(lowercase__					,     lowercase__					,     atol=1e-3							)							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :List[Any]							      =				self.prepare_config_and_inputs()
       UpperCamelCase__     :int							      =				config_and_inputs
       UpperCamelCase__     :int							      =				{'''input_ids''': input_ids, '''attention_mask''': input_mask}
       return config, inputs_dict



@require_torch
class      lowercase			(							a__			,	a__			,	a__			,	unittest.TestCase      ):

  """simple docstring"""

  _a									=       (
      (
          GPTNeoXModel,
          GPTNeoXForCausalLM,
          GPTNeoXForQuestionAnswering,
          GPTNeoXForSequenceClassification,
          GPTNeoXForTokenClassification,
      )
      if is_torch_available()
      else ()
  )
  _a									=       (GPTNeoXForCausalLM,) if is_torch_available() else ()
  _a									=       (
      {
          """feature-extraction""": GPTNeoXModel,
          """question-answering""": GPTNeoXForQuestionAnswering,
          """text-classification""": GPTNeoXForSequenceClassification,
          """text-generation""": GPTNeoXForCausalLM,
          """token-classification""": GPTNeoXForTokenClassification,
          """zero-shot""": GPTNeoXForSequenceClassification,
      }
      if is_torch_available()
      else {}
  )
  _a									=       False
  _a									=       False
  _a									=       False
  _a									=       False




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Optional[int]							      =				GPTNeoXModelTester(self							)
       UpperCamelCase__     :Any							      =				ConfigTester(self					,     config_class=lowercase__					,     hidden_size=64					,     num_attention_heads=8							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       self.config_tester.run_common_tests()




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Any							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(lowercase__					,     lowercase__					,     lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Union[str, Any]							      =				self.model_tester.prepare_config_and_inputs_for_decoder()
       self.model_tester.create_and_check_model_as_decoder(lowercase__					,     lowercase__					,     lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :int							      =				self.model_tester.prepare_config_and_inputs_for_decoder()

       UpperCamelCase__     :Dict							      =				None

       self.model_tester.create_and_check_model_as_decoder(lowercase__					,     lowercase__					,     lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :Optional[Any]							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_decoder_model_past_large_inputs(lowercase__					,     lowercase__					,     lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :str							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_causal_lm(*lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :List[str]							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :str							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*lowercase__							)




  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :List[str]							      =				self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*lowercase__							)




  @unittest.skip(reason='''Feed forward chunking is not implemented'''							)
  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       pass




  @parameterized.expand([('''linear''',), ('''dynamic''',)]							)
  def 					lowerCAmelCase__       (	self					,     UpperCamelCase_							):

       '''simple docstring'''


       UpperCamelCase__     :Any							      =				self.model_tester.prepare_config_and_inputs_for_common()
       UpperCamelCase__     :List[str]							      =				ids_tensor([1, 10]					,     config.vocab_size							)
       UpperCamelCase__     :List[str]							      =				ids_tensor([1, int(config.max_position_embeddings * 1.5							)]					,     config.vocab_size							)

       set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
       UpperCamelCase__     :Tuple							      =				GPTNeoXModel(lowercase__							)
       original_model.to(lowercase__							)
       original_model.eval()
       UpperCamelCase__     :Tuple							      =				original_model(lowercase__							).last_hidden_state
       UpperCamelCase__     :Any							      =				original_model(lowercase__							).last_hidden_state

       set_seed(42							)  # Fixed seed at init time so the two models get the same random weights
       UpperCamelCase__     :str							      =				{'''type''': scaling_type, '''factor''': 10.0}
       UpperCamelCase__     :int							      =				GPTNeoXModel(lowercase__							)
       scaled_model.to(lowercase__							)
       scaled_model.eval()
       UpperCamelCase__     :List[str]							      =				scaled_model(lowercase__							).last_hidden_state
       UpperCamelCase__     :Dict							      =				scaled_model(lowercase__							).last_hidden_state

       # Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
       # maximum sequence length, so the outputs for the short input should match.
       if scaling_type == "dynamic":
            self.assertTrue(torch.allclose(lowercase__					,     lowercase__					,     atol=1e-5							)							)
       else:
            self.assertFalse(torch.allclose(lowercase__					,     lowercase__					,     atol=1e-5							)							)

       # The output should be different for long inputs
       self.assertFalse(torch.allclose(lowercase__					,     lowercase__					,     atol=1e-5							)							)



@require_torch
class      lowercase			(							unittest.TestCase      ):

  """simple docstring"""

  @slow
  def 					lowerCAmelCase__       (	self							):

       '''simple docstring'''


       UpperCamelCase__     :List[Any]							      =				AutoTokenizer.from_pretrained('''EleutherAI/pythia-410m-deduped'''							)
       for checkpointing in [True, False]:
            UpperCamelCase__     :Tuple							      =				GPTNeoXForCausalLM.from_pretrained('''EleutherAI/pythia-410m-deduped'''							)

            if checkpointing:
                 model.gradient_checkpointing_enable()
            else:
                 model.gradient_checkpointing_disable()
            model.to(lowercase__							)

            UpperCamelCase__     :List[Any]							      =				tokenizer('''My favorite food is'''					,     return_tensors='''pt'''							).to(lowercase__							)
            # The hub repo. is updated on 2023-04-04, resulting in poor outputs.
            # See: https://github.com/huggingface/transformers/pull/24193
            UpperCamelCase__     :Union[str, Any]							      =				'''My favorite food is a good old-fashioned, old-fashioned, old-fashioned.\n\nI\'m not sure'''

            UpperCamelCase__     :Optional[Any]							      =				model.generate(**lowercase__					,     do_sample=lowercase__					,     max_new_tokens=20							)
            UpperCamelCase__     :List[Any]							      =				tokenizer.batch_decode(lowercase__							)[0]

            self.assertEqual(lowercase__					,     lowercase__							)



'''simple docstring'''
import json
import os
import unittest

from transformers.models.roc_bert.tokenization_roc_bert import (
    VOCAB_FILES_NAMES,
    RoCBertBasicTokenizer,
    RoCBertTokenizer,
    RoCBertWordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class      lowercase			(							A__			,	unittest.TestCase      ):

		"""simple docstring"""

		_a									=       RoCBertTokenizer
		_a									=       None
		_a									=       False
		_a									=       True
		_a									=       filter_non_english




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''你''', '''好''', '''是''', '''谁''', '''a''', '''b''', '''c''', '''d''']
							UpperCamelCase__     :Optional[Any]							      =				{}
							UpperCamelCase__     :str							      =				{}
							for i, value in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :List[Any]							      =				i
												UpperCamelCase__     :List[str]							      =				i
							UpperCamelCase__     :Any							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''vocab_file''']							)
							UpperCamelCase__     :Optional[Any]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_shape_file''']							)
							UpperCamelCase__     :List[str]							      =				os.path.join(self.tmpdirname					,     VOCAB_FILES_NAMES['''word_pronunciation_file''']							)
							with open(self.vocab_file					,     '''w'''					,     encoding='''utf-8'''							) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]							)							)
							with open(self.word_shape_file					,     '''w'''					,     encoding='''utf-8'''							) as word_shape_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)
							with open(self.word_pronunciation_file					,     '''w'''					,     encoding='''utf-8'''							) as word_pronunciation_writer:
												json.dump(UpperCamelCase_					,     UpperCamelCase_					,     ensure_ascii=UpperCamelCase_							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Tuple							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize('''你好[SEP]你是谁'''							)
							self.assertListEqual(UpperCamelCase_					,     ['''你''', '''好''', '''[SEP]''', '''你''', '''是''', '''谁''']							)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)
							self.assertListEqual(tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)					,     [5, 6, 2, 5, 7, 8]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				RoCBertBasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''							)					,     ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''h\u00E9llo''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Any							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']							)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''							)					,     ['''hello''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Dict							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     strip_accents=UpperCamelCase_							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''							)					,     ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :str							      =				RoCBertBasicTokenizer(do_lower_case=UpperCamelCase_					,     never_split=['''[UNK]''']							)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''							)					,     ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :List[str]							      =				['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__     :List[str]							      =				{}
							for i, token in enumerate(UpperCamelCase_							):
												UpperCamelCase__     :Tuple							      =				i
							UpperCamelCase__     :Any							      =				RoCBertWordpieceTokenizer(vocab=UpperCamelCase_					,     unk_token='''[UNK]'''							)

							self.assertListEqual(tokenizer.tokenize(''''''							)					,     []							)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''							)					,     ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']							)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''							)					,     ['''[UNK]''', '''runn''', '''##ing''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''							)							)
							self.assertTrue(_is_whitespace('''\t'''							)							)
							self.assertTrue(_is_whitespace('''\r'''							)							)
							self.assertTrue(_is_whitespace('''\n'''							)							)
							self.assertTrue(_is_whitespace('''\u00A0'''							)							)

							self.assertFalse(_is_whitespace('''A'''							)							)
							self.assertFalse(_is_whitespace('''-'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''							)							)

							self.assertFalse(_is_control('''A'''							)							)
							self.assertFalse(_is_control(''' '''							)							)
							self.assertFalse(_is_control('''\t'''							)							)
							self.assertFalse(_is_control('''\r'''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''							)							)
							self.assertTrue(_is_punctuation('''$'''							)							)
							self.assertTrue(_is_punctuation('''`'''							)							)
							self.assertTrue(_is_punctuation('''.'''							)							)

							self.assertFalse(_is_punctuation('''A'''							)							)
							self.assertFalse(_is_punctuation(''' '''							)							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Union[str, Any]							      =				self.get_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)

							if self.test_rust_tokenizer:
												UpperCamelCase__     :List[str]							      =				self.get_rust_tokenizer()
												self.assertListEqual(
												    [rust_tokenizer.tokenize(UpperCamelCase_							) for t in ['''Test''', '''\xad''', '''test''']]					,     [['''[UNK]'''], [], ['''[UNK]''']]							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Optional[Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__     :List[str]							      =				tokenizer_r.encode_plus(
																	    UpperCamelCase_					,     return_attention_mask=UpperCamelCase_					,     return_token_type_ids=UpperCamelCase_					,     return_offsets_mapping=UpperCamelCase_					,     add_special_tokens=UpperCamelCase_					,     )

																	UpperCamelCase__     :int							      =				tokenizer_r.do_lower_case if hasattr(UpperCamelCase_					,     '''do_lower_case'''							) else False
																	UpperCamelCase__     :str							      =				(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]					,     tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']							)							)
																	self.assertEqual([e[0] for e in expected_results]					,     tokens['''offset_mapping''']							)




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :Optional[Any]							      =				['''的''', '''人''', '''有''']
							UpperCamelCase__     :Tuple							      =				''''''.join(UpperCamelCase_							)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''							):
																	UpperCamelCase__     :Union[str, Any]							      =				True
																	UpperCamelCase__     :str							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :Optional[Any]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :str							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				False
																	UpperCamelCase__     :Union[str, Any]							      =				self.rust_tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)
																	UpperCamelCase__     :Dict							      =				self.tokenizer_class.from_pretrained(UpperCamelCase_					,     **UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer_r.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)
																	UpperCamelCase__     :List[Any]							      =				tokenizer_p.encode(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Any							      =				tokenizer_r.convert_ids_to_tokens(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer_p.convert_ids_to_tokens(UpperCamelCase_							)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__     :Optional[Any]							      =				[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(UpperCamelCase_							)
																	]
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)
																	self.assertListEqual(UpperCamelCase_					,     UpperCamelCase_							)




		@slow
		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.tokenizer_class(self.vocab_file					,     self.word_shape_file					,     self.word_pronunciation_file							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.encode('''你好'''					,     add_special_tokens=UpperCamelCase_							)
							UpperCamelCase__     :Dict							      =				tokenizer.encode('''你是谁'''					,     add_special_tokens=UpperCamelCase_							)

							UpperCamelCase__     :Union[str, Any]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_							)
							UpperCamelCase__     :Optional[int]							      =				tokenizer.build_inputs_with_special_tokens(UpperCamelCase_					,     UpperCamelCase_							)

							assert encoded_sentence == [1] + text + [2]
							assert encoded_pair == [1] + text + [2] + text_a + [2]




		def 					lowerCAmelCase__       (	self							):

							'''simple docstring'''


							UpperCamelCase__     :int							      =				self.get_tokenizers(do_lower_case=UpperCamelCase_							)
							for tokenizer in tokenizers:
												with self.subTest(F'''{tokenizer.__class__.__name__}'''							):
																	UpperCamelCase__     :Optional[int]							      =				'''你好，你是谁'''
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.tokenize(UpperCamelCase_							)
																	UpperCamelCase__     :Union[str, Any]							      =				tokenizer.convert_tokens_to_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Any							      =				tokenizer.convert_tokens_to_shape_ids(UpperCamelCase_							)
																	UpperCamelCase__     :int							      =				tokenizer.convert_tokens_to_pronunciation_ids(UpperCamelCase_							)
																	UpperCamelCase__     :Optional[int]							      =				tokenizer.prepare_for_model(
																	    UpperCamelCase_					,     UpperCamelCase_					,     UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	UpperCamelCase__     :Optional[int]							      =				tokenizer.encode_plus(UpperCamelCase_					,     add_special_tokens=UpperCamelCase_							)

																	self.assertEqual(UpperCamelCase_					,     UpperCamelCase_							)