






'''simple docstring'''



import os
import unittest

from transformers import LayoutLMTokenizer, LayoutLMTokenizerFast
from transformers.models.layoutlm.tokenization_layoutlm import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase      ,      unittest.TestCase     ):
      __a  					=LayoutLMTokenizer
      __a  					=LayoutLMTokenizerFast
      __a  					=True
      __a  					=True




      def  __UpperCamelCase							(  self      )							->List[str]:



         '''simple docstring'''
         super().setUp()

         __a				     =    [
             '[UNK]',
             '[CLS]',
             '[SEP]',
             'want',
             '##want',
             '##ed',
             'wa',
             'un',
             'runn',
             '##ing',
             ',',
             'low',
             'lowest',
         ]
         __a				     =    os.path.join(self.tmpdirname ,   VOCAB_FILES_NAMES['vocab_file']      )
         with open(self.vocab_file ,   'w' ,   encoding='utf-8'      ) as vocab_writer:
            vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )




      def  __UpperCamelCase							(  self ,   **lowerCamelCase      )							->Any:



         '''simple docstring'''
         return LayoutLMTokenizer.from_pretrained(self.tmpdirname ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->Optional[int]:



         '''simple docstring'''
         __a				     =    'UNwant\u00E9d,running'
         __a				     =    'unwanted, running'
         return input_text, output_text




      def  __UpperCamelCase							(  self      )							->Dict:



         '''simple docstring'''
         __a				     =    self.tokenizer_class(self.vocab_file      )

         __a				     =    tokenizer.tokenize('UNwant\u00E9d,running'      )
         self.assertListEqual(lowerCamelCase ,   ['un', '##want', '##ed', ',', 'runn', '##ing']      )
         self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase      ) ,   [7, 4, 5, 10, 8, 9]      )






      def  __UpperCamelCase							(  self      )							->Any:



         '''simple docstring'''
         pass







'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


__UpperCamelCase      :       Optional[int]      			=      {
    """configuration_speech_to_text""": ["""SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """Speech2TextConfig"""],
    """processing_speech_to_text""": ["""Speech2TextProcessor"""],
}

try:
 if not is_sentencepiece_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Union[str, Any]      			=      ["""Speech2TextTokenizer"""]

try:
 if not is_speech_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Optional[Any]      			=      ["""Speech2TextFeatureExtractor"""]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       List[Any]      			=      [
     """TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFSpeech2TextForConditionalGeneration""",
     """TFSpeech2TextModel""",
     """TFSpeech2TextPreTrainedModel""",
 ]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Tuple      			=      [
     """SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """Speech2TextForConditionalGeneration""",
     """Speech2TextModel""",
     """Speech2TextPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
 from .processing_speech_to_text import SpeechaTextProcessor

 try:
  if not is_sentencepiece_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_speech_to_text import SpeechaTextTokenizer

 try:
  if not is_speech_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_speech_to_text import (
      TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFSpeechaTextForConditionalGeneration,
      TFSpeechaTextModel,
      TFSpeechaTextPreTrainedModel,
  )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_speech_to_text import (
      SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
      SpeechaTextForConditionalGeneration,
      SpeechaTextModel,
      SpeechaTextPreTrainedModel,
  )

else:
 import sys

 __UpperCamelCase      :       List[str]      			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)