





"""simple docstring"""





import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin




class 							snake_case							(			__UpperCAmelCase      ,  unittest.TestCase ):

      '''simple docstring'''



      _A     :       List[Any]         =       ProphetNetTokenizer
      _A     :       Optional[int]         =       False

      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        super().setUp()

        __UpperCAmelCase    :						Optional[Any]       =						[
            '''[UNK]''',
            '''[CLS]''',
            '''[SEP]''',
            '''[PAD]''',
            '''[MASK]''',
            '''want''',
            '''##want''',
            '''##ed''',
            '''wa''',
            '''un''',
            '''runn''',
            '''##ing''',
            ''',''',
            '''low''',
            '''lowest''',
        ]
        __UpperCAmelCase    :						int       =						os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES['''vocab_file'''] )
        with open(self.vocab_file						,				'''w'''						,				encoding='''utf-8''' ) as vocab_writer:
          vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens] ) )

      def       A_					(							self       :      Tuple						,				__lowercase       :      Optional[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						'''UNwant\u00E9d,running'''
        __UpperCAmelCase    :						List[Any]       =						'''unwanted, running'''
        return input_text, output_text

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						self.tokenizer_class(self.vocab_file )

        __UpperCAmelCase    :						Any       =						tokenizer.tokenize('''UNwant\u00E9d,running''' )
        self.assertListEqual(__lowercase						,				['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing'''] )
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase )						,				[9, 6, 7, 12, 10, 11] )

      def       A_					(							self       :      Tuple ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						BasicTokenizer()

        self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz''' )						,				['''ah''', '''\u535A''', '''\u63A8''', '''zz'''] )

      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Any       =						BasicTokenizer(do_lower_case=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )						,				['''hello''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )						,				['''hello'''] )

      def       A_					(							self       :      int ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						BasicTokenizer(do_lower_case=__lowercase						,				strip_accents=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )						,				['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )						,				['''h\u00E9llo'''] )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						int       =						BasicTokenizer(do_lower_case=__lowercase						,				strip_accents=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )						,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )						,				['''hello'''] )

      def       A_					(							self       :      Dict ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[Any]       =						BasicTokenizer(do_lower_case=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )						,				['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?'''] )
        self.assertListEqual(tokenizer.tokenize('''H\u00E9llo''' )						,				['''hello'''] )

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						BasicTokenizer(do_lower_case=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  ''' )						,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

      def       A_					(							self       :      Any ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						BasicTokenizer(do_lower_case=__lowercase						,				strip_accents=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )						,				['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

      def       A_					(							self       :      Union[str, Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						BasicTokenizer(do_lower_case=__lowercase						,				strip_accents=__lowercase )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  ''' )						,				['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?'''] )

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						BasicTokenizer(do_lower_case=__lowercase						,				never_split=['''[UNK]'''] )

        self.assertListEqual(
            tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]''' )						,				['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]'''] )

      def       A_					(							self       :      str ):


        '''simple docstring'''




        __UpperCAmelCase    :						Optional[int]       =						['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

        __UpperCAmelCase    :						List[Any]       =						{}
        for i, token in enumerate(__lowercase ):
          __UpperCAmelCase    :						Optional[Any]       =						i
        __UpperCAmelCase    :						Optional[int]       =						WordpieceTokenizer(vocab=__lowercase						,				unk_token='''[UNK]''' )

        self.assertListEqual(tokenizer.tokenize('''''' )						,				[] )

        self.assertListEqual(tokenizer.tokenize('''unwanted running''' )						,				['''un''', '''##want''', '''##ed''', '''runn''', '''##ing'''] )

        self.assertListEqual(tokenizer.tokenize('''unwantedX running''' )						,				['''[UNK]''', '''runn''', '''##ing'''] )

      @require_torch
      def       A_					(							self       :      List[Any] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Tuple       =						self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased''' )

        __UpperCAmelCase    :						Tuple       =						['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
        __UpperCAmelCase    :						str       =						[1_037, 2_146, 20_423, 2_005, 7_680, 7_849, 3_989, 1_012, 102]
        __UpperCAmelCase    :						List[str]       =						tokenizer(__lowercase						,				padding=__lowercase						,				return_tensors='''pt''' )
        self.assertIsInstance(__lowercase						,				__lowercase )
        __UpperCAmelCase    :						Optional[int]       =						list(batch.input_ids.numpy()[0] )
        self.assertListEqual(__lowercase						,				__lowercase )

        self.assertEqual((2, 9)						,				batch.input_ids.shape )
        self.assertEqual((2, 9)						,				batch.attention_mask.shape )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.assertTrue(_is_whitespace(''' ''' ) )
        self.assertTrue(_is_whitespace('''\t''' ) )
        self.assertTrue(_is_whitespace('''\r''' ) )
        self.assertTrue(_is_whitespace('''\n''' ) )
        self.assertTrue(_is_whitespace('''\u00A0''' ) )

        self.assertFalse(_is_whitespace('''A''' ) )
        self.assertFalse(_is_whitespace('''-''' ) )

      def       A_					(							self       :      Optional[Any] ):


        '''simple docstring'''




        self.assertTrue(_is_control('''\u0005''' ) )

        self.assertFalse(_is_control('''A''' ) )
        self.assertFalse(_is_control(''' ''' ) )
        self.assertFalse(_is_control('''\t''' ) )
        self.assertFalse(_is_control('''\r''' ) )

      def       A_					(							self       :      Optional[int] ):


        '''simple docstring'''




        self.assertTrue(_is_punctuation('''-''' ) )
        self.assertTrue(_is_punctuation('''$''' ) )
        self.assertTrue(_is_punctuation('''`''' ) )
        self.assertTrue(_is_punctuation('''.''' ) )

        self.assertFalse(_is_punctuation('''A''' ) )
        self.assertFalse(_is_punctuation(''' ''' ) )







      @slow
      def       A_					(							self       :      List[str] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Dict       =						self.tokenizer_class.from_pretrained('''microsoft/prophetnet-large-uncased''' )

        __UpperCAmelCase    :						Optional[Any]       =						tokenizer.encode('''sequence builders'''						,				add_special_tokens=__lowercase )
        __UpperCAmelCase    :						List[str]       =						tokenizer.encode('''multi-sequence build'''						,				add_special_tokens=__lowercase )

        __UpperCAmelCase    :						str       =						tokenizer.build_inputs_with_special_tokens(__lowercase )
        __UpperCAmelCase    :						Union[str, Any]       =						tokenizer.build_inputs_with_special_tokens(__lowercase						,				__lowercase )

        assert encoded_sentence == text + [102]
        assert encoded_pair == text + [102] + text_a + [102]






"""simple docstring"""





def       lowerCamelCase_       (   UpperCAmelCase_  ,			UpperCAmelCase_  )						->bool:





  """simple docstring"""


  return numa ^ numa < 0


if __name__ == "__main__":
  import doctest

  doctest.testmod()