


'''simple docstring'''






import os
import unittest

from transformers import BatchEncoding
from transformers.models.bert.tokenization_bert import (
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.prophetnet.tokenization_prophetnet import VOCAB_FILES_NAMES, ProphetNetTokenizer
from transformers.testing_utils import require_torch, slow

from ...test_tokenization_common import TokenizerTesterMixin




class  a     (     _SCREAMING_SNAKE_CASE  ,				unittest.TestCase       ):
 _lowerCAmelCase 					=	ProphetNetTokenizer
 _lowerCAmelCase 					=	False
 def 							__UpperCAmelCase		( self						)					->							List[str]:
      super().setUp()

      _a			      =							[
          '[UNK]',
          '[CLS]',
          '[SEP]',
          '[PAD]',
          '[MASK]',
          'want',
          '##want',
          '##ed',
          'wa',
          'un',
          'runn',
          '##ing',
          ',',
          'low',
          'lowest',
      ]
      _a			      =							os.path.join(self.tmpdirname      ,				VOCAB_FILES_NAMES['vocab_file']						)
      with open(self.vocab_file      ,				'w'      ,				encoding='utf-8'						) as vocab_writer:
           vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Any:
      _a			      =							'UNwant\u00E9d,running'
      _a			      =							'unwanted, running'
      return input_text, output_text
 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      _a			      =							self.tokenizer_class(self.vocab_file						)

      _a			      =							tokenizer.tokenize('UNwant\u00E9d,running'						)
      self.assertListEqual(__magic_name__      ,				['un', '##want', '##ed', ',', 'runn', '##ing']						)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__						)      ,				[9, 6, 7, 12, 10, 11]						)
 def 							__UpperCAmelCase		( self						)					->							int:
      _a			      =							BasicTokenizer()

      self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'						)      ,				['ah', '\u535A', '\u63A8', 'zz']						)
 def 							__UpperCAmelCase		( self						)					->							Dict:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)      ,				['hello', '!', 'how', 'are', 'you', '?']						)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,				['hello']						)
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__      ,				strip_accents=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,				['hällo', '!', 'how', 'are', 'you', '?']						)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,				['h\u00E9llo']						)
 def 							__UpperCAmelCase		( self						)					->							str:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__      ,				strip_accents=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,				['hallo', '!', 'how', 'are', 'you', '?']						)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,				['hello']						)
 def 							__UpperCAmelCase		( self						)					->							Any:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,				['hallo', '!', 'how', 'are', 'you', '?']						)
      self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,				['hello']						)
 def 							__UpperCAmelCase		( self						)					->							str:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '						)      ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?']						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__      ,				strip_accents=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,				['HäLLo', '!', 'how', 'Are', 'yoU', '?']						)
 def 							__UpperCAmelCase		( self						)					->							Any:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__      ,				strip_accents=__magic_name__						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '						)      ,				['HaLLo', '!', 'how', 'Are', 'yoU', '?']						)
 def 							__UpperCAmelCase		( self						)					->							List[Any]:
      _a			      =							BasicTokenizer(do_lower_case=__magic_name__      ,				never_split=['[UNK]']						)

      self.assertListEqual(
          tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'						)      ,				['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      _a			      =							['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

      _a			      =							{}
      for i, token in enumerate(__magic_name__						):
           _a			      =							i
      _a			      =							WordpieceTokenizer(vocab=__magic_name__      ,				unk_token='[UNK]'						)

      self.assertListEqual(tokenizer.tokenize(''						)      ,				[]						)

      self.assertListEqual(tokenizer.tokenize('unwanted running'						)      ,				['un', '##want', '##ed', 'runn', '##ing']						)

      self.assertListEqual(tokenizer.tokenize('unwantedX running'						)      ,				['[UNK]', 'runn', '##ing']						)
 @require_torch
 def 							__UpperCAmelCase		( self						)					->							str:
      _a			      =							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

      _a			      =							['A long paragraph for summarization.', 'Another paragraph for summarization.']
      _a			      =							[10_37, 21_46, 2_04_23, 20_05, 76_80, 78_49, 39_89, 10_12, 1_02]
      _a			      =							tokenizer(__magic_name__      ,				padding=__magic_name__      ,				return_tensors='pt'						)
      self.assertIsInstance(__magic_name__      ,				__magic_name__						)
      _a			      =							list(batch.input_ids.numpy()[0]						)
      self.assertListEqual(__magic_name__      ,				__magic_name__						)

      self.assertEqual((2, 9)      ,				batch.input_ids.shape						)
      self.assertEqual((2, 9)      ,				batch.attention_mask.shape						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      self.assertTrue(_is_whitespace(' '						)						)
      self.assertTrue(_is_whitespace('\t'						)						)
      self.assertTrue(_is_whitespace('\r'						)						)
      self.assertTrue(_is_whitespace('\n'						)						)
      self.assertTrue(_is_whitespace('\u00A0'						)						)

      self.assertFalse(_is_whitespace('A'						)						)
      self.assertFalse(_is_whitespace('-'						)						)
 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      self.assertTrue(_is_control('\u0005'						)						)

      self.assertFalse(_is_control('A'						)						)
      self.assertFalse(_is_control(' '						)						)
      self.assertFalse(_is_control('\t'						)						)
      self.assertFalse(_is_control('\r'						)						)
 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      self.assertTrue(_is_punctuation('-'						)						)
      self.assertTrue(_is_punctuation('$'						)						)
      self.assertTrue(_is_punctuation('`'						)						)
      self.assertTrue(_is_punctuation('.'						)						)

      self.assertFalse(_is_punctuation('A'						)						)
      self.assertFalse(_is_punctuation(' '						)						)




 @slow
 def 							__UpperCAmelCase		( self						)					->							Dict:
      _a			      =							self.tokenizer_class.from_pretrained('microsoft/prophetnet-large-uncased'						)

      _a			      =							tokenizer.encode('sequence builders'      ,				add_special_tokens=__magic_name__						)
      _a			      =							tokenizer.encode('multi-sequence build'      ,				add_special_tokens=__magic_name__						)

      _a			      =							tokenizer.build_inputs_with_special_tokens(__magic_name__						)
      _a			      =							tokenizer.build_inputs_with_special_tokens(__magic_name__      ,				__magic_name__						)

      assert encoded_sentence == text + [1_02]
      assert encoded_pair == text + [1_02] + text_a + [1_02]





'''simple docstring'''






class  a     :
 def __init__( self						)					->							List[Any]:
      _a			      =							''
      _a			      =							''
      _a			      =							[]
 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							int:
      if m == -1:
           return n + 1
      elif n == -1:
           return m + 1
      elif self.dp[m][n] > -1:
           return self.dp[m][n]
      else:
           if self.worda[m] == self.worda[n]:
                _a			      =							self.__min_dist_top_down_dp(m - 1      ,				n - 1						)
           else:
                _a			      =							self.__min_dist_top_down_dp(__magic_name__      ,				n - 1						)
                _a			      =							self.__min_dist_top_down_dp(m - 1      ,				__magic_name__						)
                _a			      =							self.__min_dist_top_down_dp(m - 1      ,				n - 1						)
                _a			      =							1 + min(__magic_name__      ,				__magic_name__      ,				__magic_name__						)

           return self.dp[m][n]
 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							int:
      _a			      =							worda
      _a			      =							worda
      _a			      =							[[-1 for _ in range(len(__magic_name__						)						)] for _ in range(len(__magic_name__						)						)]

      return self.__min_dist_top_down_dp(len(__magic_name__						) - 1      ,				len(__magic_name__						) - 1						)




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							int:
      _a			      =							worda
      _a			      =							worda
      _a			      =							len(__magic_name__						)
      _a			      =							len(__magic_name__						)
      _a			      =							[[0 for _ in range(n + 1						)] for _ in range(m + 1						)]

      for i in range(m + 1						):
           for j in range(n + 1						):
                if i == 0:  # first string is empty
                     _a			      =							j
                elif j == 0:  # second string is empty
                     _a			      =							i
                elif worda[i - 1] == worda[j - 1]:  # last characters are equal
                     _a			      =							self.dp[i - 1][j - 1]
                else:
                     _a			      =							self.dp[i][j - 1]
                     _a			      =							self.dp[i - 1][j]
                     _a			      =							self.dp[i - 1][j - 1]
                     _a			      =							1 + min(__magic_name__      ,				__magic_name__      ,				__magic_name__						)
      return self.dp[m][n]


if __name__ == "__main__":
 a_					:    Dict          =    EditDistance()

 print("****************** Testing Edit Distance DP Algorithm ******************")
 print()

 a_					:    Tuple          =    input("Enter the first string: ").strip()
 a_					:    Dict          =    input("Enter the second string: ").strip()

 print()
 print(f'''The minimum edit distance is: {solver.min_dist_top_down(Sa, Sa)}''')
 print(f'''The minimum edit distance is: {solver.min_dist_bottom_up(Sa, Sa)}''')
 print()
 print("*************** End of Testing Edit Distance DP Algorithm ***************")

