

"""simple docstring"""






from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching





def 	lowercase      (   ) ->      Union[str, Any]:
 import os as original_os
 from os import path as original_path
 from os import rename as original_rename
 from os.path import dirname as original_dirname
 from os.path import join as original_join

 assert _test_patching.os is original_os
 assert _test_patching.path is original_path
 assert _test_patching.join is original_join

 assert _test_patching.renamed_os is original_os
 assert _test_patching.renamed_path is original_path
 assert _test_patching.renamed_join is original_join

 __magic_name__     =      '''__test_patch_submodule_mock__'''
 with patch_submodule(_test_patching      ,   '''os.path.join'''      ,   __UpperCamelCase						):
  # Every way to access os.path.join must be patched, and the rest must stay untouched

  # check os.path.join
  assert isinstance(_test_patching.os      ,   _PatchedModuleObj						)
  assert isinstance(_test_patching.os.path      ,   _PatchedModuleObj						)
  assert _test_patching.os.path.join is mock

  # check path.join
  assert isinstance(_test_patching.path      ,   _PatchedModuleObj						)
  assert _test_patching.path.join is mock

  # check join
  assert _test_patching.join is mock

  # check that the other attributes are untouched
  assert _test_patching.os.rename is original_rename
  assert _test_patching.path.dirname is original_dirname
  assert _test_patching.os.path.dirname is original_dirname

  # Even renamed modules or objects must be patched

  # check renamed_os.path.join
  assert isinstance(_test_patching.renamed_os      ,   _PatchedModuleObj						)
  assert isinstance(_test_patching.renamed_os.path      ,   _PatchedModuleObj						)
  assert _test_patching.renamed_os.path.join is mock

  # check renamed_path.join
  assert isinstance(_test_patching.renamed_path      ,   _PatchedModuleObj						)
  assert _test_patching.renamed_path.join is mock

  # check renamed_join
  assert _test_patching.renamed_join is mock

  # check that the other attributes are untouched
  assert _test_patching.renamed_os.rename is original_rename
  assert _test_patching.renamed_path.dirname is original_dirname
  assert _test_patching.renamed_os.path.dirname is original_dirname

 # check that everthing is back to normal when the patch is over

 assert _test_patching.os is original_os
 assert _test_patching.path is original_path
 assert _test_patching.join is original_join

 assert _test_patching.renamed_os is original_os
 assert _test_patching.renamed_path is original_path
 assert _test_patching.renamed_join is original_join





def 	lowercase      (   ) ->      List[str]:
 assert _test_patching.open is open

 __magic_name__     =      '''__test_patch_submodule_builtin_mock__'''
 # _test_patching has "open" in its globals
 assert _test_patching.open is open
 with patch_submodule(_test_patching      ,   '''open'''      ,   __UpperCamelCase						):
  assert _test_patching.open is mock

 # check that everthing is back to normal when the patch is over

 assert _test_patching.open is open





def 	lowercase      (   ) ->      Tuple:
 # pandas.read_csv is not present in _test_patching
 __magic_name__     =      '''__test_patch_submodule_missing_mock__'''
 with patch_submodule(_test_patching      ,   '''pandas.read_csv'''      ,   __UpperCamelCase						):
  pass





def 	lowercase      (   ) ->      Tuple:
 # builtin should always be mocked even if they're not in the globals
 # in case they're loaded at one point
 __magic_name__     =      '''__test_patch_submodule_missing_builtin_mock__'''
 # _test_patching doesn't have "len" in its globals
 assert getattr(_test_patching      ,   '''len'''      ,   __UpperCamelCase						) is None
 with patch_submodule(_test_patching      ,   '''len'''      ,   __UpperCamelCase						):
  assert _test_patching.len is mock
 assert _test_patching.len is len





def 	lowercase      (   ) ->      int:
 __magic_name__     =      '''__test_patch_submodule_start_and_stop_mock__'''
 __magic_name__     =      patch_submodule(_test_patching      ,   '''open'''      ,   __UpperCamelCase						)
 assert _test_patching.open is open
 patch.start()
 assert _test_patching.open is mock
 patch.stop()
 assert _test_patching.open is open





def 	lowercase      (   ) ->      str:
 from os import rename as original_rename
 from os.path import dirname as original_dirname
 from os.path import join as original_join

 __magic_name__     =      '''__test_patch_submodule_successive_join__'''
 __magic_name__     =      '''__test_patch_submodule_successive_dirname__'''
 __magic_name__     =      '''__test_patch_submodule_successive_rename__'''
 assert _test_patching.os.path.join is original_join
 assert _test_patching.os.path.dirname is original_dirname
 assert _test_patching.os.rename is original_rename

 with patch_submodule(_test_patching      ,   '''os.path.join'''      ,   __UpperCamelCase						):
  with patch_submodule(_test_patching      ,   '''os.rename'''      ,   __UpperCamelCase						):
   with patch_submodule(_test_patching      ,   '''os.path.dirname'''      ,   __UpperCamelCase						):
    assert _test_patching.os.path.join is mock_join
    assert _test_patching.os.path.dirname is mock_dirname
    assert _test_patching.os.rename is mock_rename

    # try another order
 with patch_submodule(_test_patching      ,   '''os.rename'''      ,   __UpperCamelCase						):
  with patch_submodule(_test_patching      ,   '''os.path.join'''      ,   __UpperCamelCase						):
   with patch_submodule(_test_patching      ,   '''os.path.dirname'''      ,   __UpperCamelCase						):
    assert _test_patching.os.path.join is mock_join
    assert _test_patching.os.path.dirname is mock_dirname
    assert _test_patching.os.rename is mock_rename

 assert _test_patching.os.path.join is original_join
 assert _test_patching.os.path.dirname is original_dirname
 assert _test_patching.os.rename is original_rename





def 	lowercase      (   ) ->      Optional[int]:
 __magic_name__     =      '''__test_patch_submodule_doesnt_exist_mock__'''
 with patch_submodule(_test_patching      ,   '''__module_that_doesn_exist__.__attribute_that_doesn_exist__'''      ,   __UpperCamelCase						):
  pass
 with patch_submodule(_test_patching      ,   '''os.__attribute_that_doesn_exist__'''      ,   __UpperCamelCase						):
  pass




"""simple docstring"""






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class        _lowercase					(     __UpperCAmelCase   , unittest.TestCase     ):
     _lowerCamelCase     =       FunnelTokenizer
     _lowerCamelCase     =       FunnelTokenizerFast
     _lowerCamelCase     =       True
     _lowerCamelCase     =       True







     def 	lowerCAmelCase__    (      self       ):
      super().setUp()

      __magic_name__     =      [
          '''<unk>''',
          '''<cls>''',
          '''<sep>''',
          '''want''',
          '''##want''',
          '''##ed''',
          '''wa''',
          '''un''',
          '''runn''',
          '''##ing''',
          ''',''',
          '''low''',
          '''lowest''',
      ]
      __magic_name__     =      os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file''']       )
      with open(self.vocab_file ,						'''w''' ,						encoding='''utf-8'''       ) as vocab_writer:
       vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]       )       )







     def 	lowerCAmelCase__    (      self ,						**UpperCamelCase_       ):
      return FunnelTokenizer.from_pretrained(self.tmpdirname ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						**UpperCamelCase_       ):
      return FunnelTokenizerFast.from_pretrained(self.tmpdirname ,						**UpperCamelCase_       )







     def 	lowerCAmelCase__    (      self ,						UpperCamelCase_       ):
      __magic_name__     =      '''UNwant\u00E9d,running'''
      __magic_name__     =      '''unwanted, running'''
      return input_text, output_text







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.tokenizer_class(self.vocab_file       )

      __magic_name__     =      tokenizer.tokenize('''UNwant\u00E9d,running'''       )
      self.assertListEqual(UpperCamelCase_ ,						['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']       )
      self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase_       ) ,						[7, 4, 5, 10, 8, 9]       )







     def 	lowerCAmelCase__    (      self       ):
      __magic_name__     =      self.get_tokenizers(do_lower_case=UpperCamelCase_       )
      for tokenizer in tokenizers:
       __magic_name__     =      tokenizer('''UNwant\u00E9d,running'''       )
       __magic_name__     =      len(inputs['''input_ids''']       ) - 1
       self.assertListEqual(inputs['''token_type_ids'''] ,						[2] + [0] * sentence_len       )

       __magic_name__     =      tokenizer('''UNwant\u00E9d,running''' ,						'''UNwant\u00E9d,running'''       )
       self.assertListEqual(inputs['''token_type_ids'''] ,						[2] + [0] * sentence_len + [1] * sentence_len       )

