


"""simple docstring"""


import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad

class 				_SCREAMING_SNAKE_CASE				(							unittest.TestCase  ):
   """simple docstring"""



   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    lowercase__       :       List[str]				=						10


   def 					UpperCAmelCase__(						self						)      ->							Any:
    lowercase__       :       List[str]				=						[1, 2, 3, 4]
    lowercase__       :       Union[str, Any]				=						[1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
    self.assertEqual(truncate_or_pad(lowerCamelCase__ ,     self.block_size ,     0						) ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							str:
    lowercase__       :       Optional[int]				=						[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    lowercase__       :       Dict				=						[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    self.assertEqual(truncate_or_pad(lowerCamelCase__ ,     self.block_size ,     0						) ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							int:
    lowercase__       :       Dict				=						[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]
    lowercase__       :       Union[str, Any]				=						[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    self.assertEqual(truncate_or_pad(lowerCamelCase__ ,     self.block_size ,     0						) ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							List[Any]:
    lowercase__       :       Optional[Any]				=						"""It was the year of Our Lord one thousand seven hundred and
        seventy-five.\n\nSpiritual revelations were conceded to England at that
        favoured period, as at this."""
    lowercase__	,       lowercase__       :       int				=						process_story(lowerCamelCase__						)
    self.assertEqual(lowerCamelCase__ ,     []						)


   def 					UpperCAmelCase__(						self						)      ->							Any:
    lowercase__       :       Tuple				=						""""""
    lowercase__	,       lowercase__       :       Optional[Any]				=						process_story(lowerCamelCase__						)
    self.assertEqual(lowerCamelCase__ ,     []						)
    self.assertEqual(lowerCamelCase__ ,     []						)


   def 					UpperCAmelCase__(						self						)      ->							Tuple:
    lowercase__       :       Union[str, Any]				=						(
        """It was the year of Our Lord one thousand seven hundred and """
        """seventy-five\n\nSpiritual revelations were conceded to England """
        """at that favoured period, as at this.\n@highlight\n\nIt was the best of times"""
    )
    lowercase__	,       lowercase__       :       Any				=						process_story(lowerCamelCase__						)

    lowercase__       :       int				=						[
        """It was the year of Our Lord one thousand seven hundred and seventy-five.""",
        """Spiritual revelations were conceded to England at that favoured period, as at this.""",
    ]
    self.assertEqual(lowerCamelCase__ ,     lowerCamelCase__						)

    lowercase__       :       Union[str, Any]				=						["""It was the best of times."""]
    self.assertEqual(lowerCamelCase__ ,     lowerCamelCase__						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    lowercase__       :       Tuple				=						torch.tensor([1, 2, 3, 4]						)
    lowercase__       :       Optional[Any]				=						torch.tensor([1, 1, 1, 1]						)
    np.testing.assert_array_equal(build_mask(lowerCamelCase__ ,     0						).numpy() ,     expected.numpy()						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    lowercase__       :       Any				=						torch.tensor([1, 2, 3, 4, 23, 23, 23]						)
    lowercase__       :       Any				=						torch.tensor([1, 1, 1, 1, 0, 0, 0]						)
    np.testing.assert_array_equal(build_mask(lowerCamelCase__ ,     23						).numpy() ,     expected.numpy()						)


   def 					UpperCAmelCase__(						self						)      ->							Optional[Any]:
    lowercase__       :       Optional[int]				=						torch.tensor([8, 2, 3, 4, 1, 1, 1]						)
    lowercase__       :       List[str]				=						torch.tensor([1, 1, 1, 1, 0, 0, 0]						)
    np.testing.assert_array_equal(build_mask(lowerCamelCase__ ,     1						).numpy() ,     expected.numpy()						)







   def 					UpperCAmelCase__(						self						)      ->							Optional[int]:
    lowercase__       :       int				=						101
    lowercase__       :       Dict				=						torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 101, 5, 6], [1, 101, 3, 4, 101, 6]]						)
    lowercase__       :       Optional[int]				=						torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]]						)

    lowercase__       :       Optional[int]				=						compute_token_type_ids(lowerCamelCase__ ,     lowerCamelCase__						)
    np.testing.assert_array_equal(lowerCamelCase__ ,     lowerCamelCase__						)



"""simple docstring"""


import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 				_SCREAMING_SNAKE_CASE				(							__UpperCAmelCase     ,   unittest.TestCase  ):
   """simple docstring"""



   _a   :       Optional[int]									=    LayoutLMTokenizer
   _a   :       Dict									=    LayoutLMTokenizerFast
   _a   :       Dict									=    True
   _a   :       Any									=    True


   def 					UpperCAmelCase__(						self						)      ->							List[str]:
    super().setUp()

    lowercase__       :       int				=						[
        """[UNK]""",
        """[CLS]""",
        """[SEP]""",
        """want""",
        """##want""",
        """##ed""",
        """wa""",
        """un""",
        """runn""",
        """##ing""",
        """,""",
        """low""",
        """lowest""",
    ]
    lowercase__       :       Dict				=						os.path.join(self.tmpdirname ,     VOCAB_FILES_NAMES["""vocab_file"""]						)
    with open(self.vocab_file ,     """w""" ,     encoding="""utf-8"""						) as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)


   def 					UpperCAmelCase__(						self ,     **lowerCamelCase__						)      ->							Optional[int]:
    return LayoutLMTokenizer.from_pretrained(self.tmpdirname ,     **lowerCamelCase__						)


   def 					UpperCAmelCase__(						self ,     lowerCamelCase__						)      ->							List[Any]:
    lowercase__       :       List[str]				=						"""UNwant\u00E9d,running"""
    lowercase__       :       Optional[int]				=						"""unwanted, running"""
    return input_text, output_text


   def 					UpperCAmelCase__(						self						)      ->							Optional[int]:
    lowercase__       :       str				=						self.tokenizer_class(self.vocab_file						)

    lowercase__       :       Optional[Any]				=						tokenizer.tokenize("""UNwant\u00E9d,running"""						)
    self.assertListEqual(lowerCamelCase__ ,     ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]						)
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase__						) ,     [7, 4, 5, 10, 8, 9]						)







   def 					UpperCAmelCase__(						self						)      ->							Any:
    pass