




'''simple docstring'''







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast

class 	_snake_case(			unittest.TestCase      ):






 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Any:






    """simple docstring"""


    A__       =     0






 @slow
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Union[str, Any]:






    """simple docstring"""


    for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
       A__       =     AutoTokenizer.from_pretrained(a    )
       self.assertIsNotNone(a    )
       self.assertIsInstance(a    ,		(BertTokenizer, BertTokenizerFast)    )
       self.assertGreater(len(a    )    ,		0    )

    for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
       A__       =     AutoTokenizer.from_pretrained(a    )
       self.assertIsNotNone(a    )
       self.assertIsInstance(a    ,		(GPTaTokenizer, GPTaTokenizerFast)    )
       self.assertGreater(len(a    )    ,		0    )






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   Dict:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained(a    )
    self.assertIsInstance(a    ,		(BertTokenizer, BertTokenizerFast)    )
    self.assertEqual(tokenizer.vocab_size    ,		12    )






 def 						_UpperCamelCase (self       :  str    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained(a    )
    self.assertIsInstance(a    ,		(RobertaTokenizer, RobertaTokenizerFast)    )
    self.assertEqual(tokenizer.vocab_size    ,		20    )






 def 						_UpperCamelCase (self       :  List[str]    )       ->   Optional[Any]:






    """simple docstring"""


    A__       =     AutoConfig.from_pretrained(a    )
    self.assertIsInstance(a    ,		a    )
    # Check that tokenizer_type ≠ model_type
    A__       =     AutoTokenizer.from_pretrained(a    ,		config=a    )
    self.assertIsInstance(a    ,		(BertTokenizer, BertTokenizerFast)    )
    self.assertEqual(tokenizer.vocab_size    ,		12    )






 def 						_UpperCamelCase (self       :  Any    )       ->   Optional[Any]:






    """simple docstring"""


    with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy('./tests/fixtures/vocab.txt'    ,		os.path.join(a    ,		'vocab.txt'    )    )

       A__       =     AutoTokenizer.from_pretrained(a    ,		tokenizer_type='bert'    ,		use_fast=a    )
       self.assertIsInstance(a    ,		a    )

    with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy('./tests/fixtures/vocab.json'    ,		os.path.join(a    ,		'vocab.json'    )    )
       shutil.copy('./tests/fixtures/merges.txt'    ,		os.path.join(a    ,		'merges.txt'    )    )

       A__       =     AutoTokenizer.from_pretrained(a    ,		tokenizer_type='gpt2'    ,		use_fast=a    )
       self.assertIsInstance(a    ,		a    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   int:






    """simple docstring"""


    with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy('./tests/fixtures/vocab.txt'    ,		os.path.join(a    ,		'vocab.txt'    )    )

       A__       =     AutoTokenizer.from_pretrained(a    ,		tokenizer_type='bert'    )
       self.assertIsInstance(a    ,		a    )

    with tempfile.TemporaryDirectory() as tmp_dir:
       shutil.copy('./tests/fixtures/vocab.json'    ,		os.path.join(a    ,		'vocab.json'    )    )
       shutil.copy('./tests/fixtures/merges.txt'    ,		os.path.join(a    ,		'merges.txt'    )    )

       A__       =     AutoTokenizer.from_pretrained(a    ,		tokenizer_type='gpt2'    )
       self.assertIsInstance(a    ,		a    )






 def 						_UpperCamelCase (self       :  List[str]    )       ->   List[str]:






    """simple docstring"""


    with pytest.raises(a    ):
       AutoTokenizer.from_pretrained('./'    ,		tokenizer_type='xxx'    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  List[Any]    )       ->   str:






    """simple docstring"""


    for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
       A__       =     tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased'    )
       self.assertIsInstance(a    ,		(BertTokenizer, BertTokenizerFast)    )

       if isinstance(a    ,		a    ):
          self.assertEqual(tokenizer.basic_tokenizer.do_lower_case    ,		a    )
       else:
          self.assertEqual(tokenizer.do_lower_case    ,		a    )

       self.assertEqual(tokenizer.model_max_length    ,		5_12    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Any    )       ->   Optional[int]:






    """simple docstring"""


    for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
       with self.assertRaisesRegex(
           a    ,		'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'    ,		):
          A__       =     tokenizer_class.from_pretrained('julien-c/herlolip-not-exists'    )






 def 						_UpperCamelCase (self       :  int    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     TOKENIZER_MAPPING.values()
    A__       =     []

    for slow_tok, fast_tok in tokenizers:
       if slow_tok is not None:
          tokenizer_names.append(slow_tok.__name__    )

       if fast_tok is not None:
          tokenizer_names.append(fast_tok.__name__    )

    for tokenizer_name in tokenizer_names:
       # must find the right class
       tokenizer_class_from_name(a    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  int    )       ->   Union[str, Any]:






    """simple docstring"""


    self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'    ,		use_fast=a    )    ,		a    )
    self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'    )    ,		a    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Dict:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained('distilbert-base-uncased'    ,		do_lower_case=a    )
    A__       =     'Hello, world. How are you?'
    A__       =     tokenizer.tokenize(a    )
    self.assertEqual('[UNK]'    ,		tokens[0]    )

    A__       =     AutoTokenizer.from_pretrained('microsoft/mpnet-base'    ,		do_lower_case=a    )
    A__       =     tokenizer.tokenize(a    )
    self.assertEqual('[UNK]'    ,		tokens[0]    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Tuple    )       ->   Any:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config'    )
    self.assertEqual(type(a    )    ,		a    )
    self.assertEqual(tokenizer.model_max_length    ,		5_12    )
    self.assertEqual(tokenizer.vocab_size    ,		3_00_00    )
    self.assertEqual(tokenizer.unk_token    ,		'[UNK]'    )
    self.assertEqual(tokenizer.padding_side    ,		'right'    )
    self.assertEqual(tokenizer.truncation_side    ,		'right'    )






 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   int:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained(a    )
    self.assertIsInstance(a    ,		(BertTokenizer, BertTokenizerFast)    )
    with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(a    )
       A__       =     AutoTokenizer.from_pretrained(a    )

    self.assertIsInstance(a    ,		tokenizer.__class__    )
    self.assertEqual(tokenizera.vocab_size    ,		12    )






 def 						_UpperCamelCase (self       :  Any    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained('ctrl'    )
    # There is no fast CTRL so this always gives us a slow tokenizer.
    self.assertIsInstance(a    ,		a    )






 def 						_UpperCamelCase (self       :  List[str]    )       ->   int:






    """simple docstring"""


    A__       =     get_tokenizer_config('bert-base-cased'    )
    A__       =     config.pop('_commit_hash'    ,		a    )
    # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
    self.assertEqual(a    ,		{'do_lower_case': False}    )

    # This model does not have a tokenizer_config so we get back an empty dict.
    A__       =     get_tokenizer_config(a    )
    self.assertDictEqual(a    ,		{}    )

    # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
    A__       =     AutoTokenizer.from_pretrained(a    )
    with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(a    )
       A__       =     get_tokenizer_config(a    )

    # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
    self.assertEqual(config['tokenizer_class']    ,		'BertTokenizer'    )






 def 						_UpperCamelCase (self       :  int    )       ->   List[Any]:






    """simple docstring"""


    try:
       AutoConfig.register('custom'    ,		a    )

       AutoTokenizer.register(a    ,		slow_tokenizer_class=a    )
       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(a    ):
          AutoTokenizer.register(a    ,		slow_tokenizer_class=a    )

       A__       =     CustomTokenizer.from_pretrained(a    )
       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(a    )

          A__       =     AutoTokenizer.from_pretrained(a    )
          self.assertIsInstance(a    ,		a    )

    finally:
       if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Tuple    )       ->   int:






    """simple docstring"""


    try:
       AutoConfig.register('custom'    ,		a    )

       # Can register in two steps
       AutoTokenizer.register(a    ,		slow_tokenizer_class=a    )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,		(CustomTokenizer, None)    )
       AutoTokenizer.register(a    ,		fast_tokenizer_class=a    )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,		(CustomTokenizer, CustomTokenizerFast)    )

       del TOKENIZER_MAPPING._extra_content[CustomConfig]
       # Can register in one step
       AutoTokenizer.register(
           a    ,		slow_tokenizer_class=a    ,		fast_tokenizer_class=a    )
       self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,		(CustomTokenizer, CustomTokenizerFast)    )

       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(a    ):
          AutoTokenizer.register(a    ,		fast_tokenizer_class=a    )

       # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
       # and that model does not have a tokenizer.json
       with tempfile.TemporaryDirectory() as tmp_dir:
          A__       =     BertTokenizerFast.from_pretrained(a    )
          bert_tokenizer.save_pretrained(a    )
          A__       =     CustomTokenizerFast.from_pretrained(a    )

       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(a    )

          A__       =     AutoTokenizer.from_pretrained(a    )
          self.assertIsInstance(a    ,		a    )

          A__       =     AutoTokenizer.from_pretrained(a    ,		use_fast=a    )
          self.assertIsInstance(a    ,		a    )

    finally:
       if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]






 def 						_UpperCamelCase (self       :  str    )       ->   Optional[int]:






    """simple docstring"""


    with self.assertRaises(a    ):
       A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    )
    # If remote code is disabled, we can't load this config.
    with self.assertRaises(a    ):
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    )

    A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    )
    self.assertTrue(tokenizer.special_attribute_present    )
    # Test tokenizer can be reloaded.
    with tempfile.TemporaryDirectory() as tmp_dir:
       tokenizer.save_pretrained(a    )
       A__       =     AutoTokenizer.from_pretrained(a    ,		trust_remote_code=a    )
    self.assertTrue(reloaded_tokenizer.special_attribute_present    )

    if is_tokenizers_available():
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )
       self.assertEqual(reloaded_tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )

       # Test we can also load the slow version
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    ,		use_fast=a    )
       self.assertTrue(tokenizer.special_attribute_present    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(a    )
          A__       =     AutoTokenizer.from_pretrained(a    ,		trust_remote_code=a    ,		use_fast=a    )
       self.assertEqual(reloaded_tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       self.assertTrue(reloaded_tokenizer.special_attribute_present    )
    else:
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       self.assertEqual(reloaded_tokenizer.__class__.__name__    ,		'NewTokenizer'    )






 @require_tokenizers
 def 						_UpperCamelCase (self       :  Union[str, Any]    )       ->   int:






    """simple docstring"""


    class 	_snake_case(			UpperCAmelCase      ):
     __snake_case:							Optional[Any]						=					False




    class 	_snake_case(			UpperCAmelCase      ):
     __snake_case:							str						=					NewTokenizer
     __snake_case:							Optional[int]						=					False

    try:
       AutoConfig.register('custom'    ,		a    )
       AutoTokenizer.register(a    ,		slow_tokenizer_class=a    )
       AutoTokenizer.register(a    ,		fast_tokenizer_class=a    )
       # If remote code is not set, the default is to use local
       A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )
       self.assertFalse(tokenizer.special_attribute_present    )
       A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,		use_fast=a    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       self.assertFalse(tokenizer.special_attribute_present    )

       # If remote code is disabled, we load the local one.
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )
       self.assertFalse(tokenizer.special_attribute_present    )
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    ,		use_fast=a    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       self.assertFalse(tokenizer.special_attribute_present    )

       # If remote is enabled, we load from the Hub
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )
       self.assertTrue(tokenizer.special_attribute_present    )
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer'    ,		trust_remote_code=a    ,		use_fast=a    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
       self.assertTrue(tokenizer.special_attribute_present    )

    finally:
       if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
       if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]






 def 						_UpperCamelCase (self       :  List[Any]    )       ->   int:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained(
        'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,		trust_remote_code=a    )
    self.assertTrue(tokenizer.special_attribute_present    )
    if is_tokenizers_available():
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizerFast'    )

       # Test we can also load the slow version
       A__       =     AutoTokenizer.from_pretrained(
           'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,		trust_remote_code=a    ,		use_fast=a    )
       self.assertTrue(tokenizer.special_attribute_present    )
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )
    else:
       self.assertEqual(tokenizer.__class__.__name__    ,		'NewTokenizer'    )






 def 						_UpperCamelCase (self       :  Dict    )       ->   Optional[Any]:






    """simple docstring"""


    with self.assertRaisesRegex(
        a    ,		'bert-base is not a local folder and is not a valid model identifier'    ):
       A__       =     AutoTokenizer.from_pretrained('bert-base'    )






 def 						_UpperCamelCase (self       :  Any    )       ->   Any:






    """simple docstring"""


    with self.assertRaisesRegex(
        a    ,		R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'    ):
       A__       =     AutoTokenizer.from_pretrained(a    ,		revision='aaaaaa'    )






 def 						_UpperCamelCase (self       :  List[Any]    )       ->   List[Any]:






    """simple docstring"""


    A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )
    with RequestCounter() as counter:
       A__       =     AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'    )
       self.assertEqual(counter.get_request_count    ,		0    )
       self.assertEqual(counter.head_request_count    ,		1    )
       self.assertEqual(counter.other_request_count    ,		0    )








'''simple docstring'''







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

lowerCAmelCase_     						=				'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
    raise ImportWarning(
        '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
    )

if version.parse(pyarrow.__version__).major < 8:
    raise ImportWarning(
        '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
        '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
    )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

lowerCAmelCase_     						=				concatenate_datasets
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadManager
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


