


import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config






@pytest.fixture(scope='session'     )
def   _lowercase(					):
							a__      =10
							a__      =datasets.Features(
							    {
							        'tokens': datasets.Sequence(datasets.Value('string'     )     ),
							        'labels': datasets.Sequence(datasets.ClassLabel(names=['negative', 'positive']     )     ),
							        'answers': datasets.Sequence(
							            {
							                'text': datasets.Value('string'     ),
							                'answer_start': datasets.Value('int32'     ),
							            }     ),
							        'id': datasets.Value('int64'     ),
							    }     )
							a__      =datasets.Dataset.from_dict(
							    {
							        'tokens': [['foo'] * 5] * n,
							        'labels': [[1] * 5] * n,
							        'answers': [{'answer_start': [97], 'text': ['1976']}] * 10,
							        'id': list(range(__a     )     ),
							    }    ,     features=__a    ,     )
							return dataset






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						Dict     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'file.arrow'     )
							dataset.map(cache_file_name=__a     )
							return filename


# FILE_CONTENT + files


_lowerCAmelCase:						Tuple   				=       '\\n    Text data.\n    Second line of data.'






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt'
							a__      =FILE_CONTENT
							with open(__a    ,     'w'     ) as f:
														f.write(__a     )
							return filename






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[str]     ):
							import bza

							a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.bz2'
							a__      =bytes(__a    ,     'utf-8'     )
							with bza.open(__a    ,     'wb'     ) as f:
														f.write(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							import gzip

							a__      =str(tmp_path_factory.mktemp('data'     ) / 'file.txt.gz'     )
							a__      =bytes(__a    ,     'utf-8'     )
							with gzip.open(__a    ,     'wb'     ) as f:
														f.write(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Tuple     ):
							if datasets.config.LZ4_AVAILABLE:
														import lza.frame

														a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.lz4'
														a__      =bytes(__a    ,     'utf-8'     )
														with lza.frame.open(__a    ,     'wb'     ) as f:
																					f.write(__a     )
														return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]    ,     __a				:						Tuple     ):
							if datasets.config.PY7ZR_AVAILABLE:
														import pyazr

														a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.7z'
														with pyazr.SevenZipFile(__a    ,     'w'     ) as archive:
																					archive.write(__a    ,     arcname=os.path.basename(__a     )     )
														return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						str    ,     __a				:						Optional[int]     ):
							import tarfile

							a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.tar'
							with tarfile.TarFile(__a    ,     'w'     ) as f:
														f.add(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Union[str, Any]     ):
							import lzma

							a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.xz'
							a__      =bytes(__a    ,     'utf-8'     )
							with lzma.open(__a    ,     'wb'     ) as f:
														f.write(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						str    ,     __a				:						Any     ):
							import zipfile

							a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Tuple     ):
							if datasets.config.ZSTANDARD_AVAILABLE:
														import zstandard as zstd

														a__      =tmp_path_factory.mktemp('data'     ) / 'file.txt.zst'
														a__      =bytes(__a    ,     'utf-8'     )
														with zstd.open(__a    ,     'wb'     ) as f:
																					f.write(__a     )
														return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						str     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'file.xml'
							a__      =textwrap.dedent(
							    '\\n    <?xml version="1.0" encoding="UTF-8" ?>\n    <tmx version="1.4">\n      <header segtype="sentence" srclang="ca" />\n      <body>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 1</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 2</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 3</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 4</seg></tuv>\n        </tu>\n        <tu>\n          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>\n          <tuv xml:lang="en"><seg>Content 5</seg></tuv>\n        </tu>\n      </body>\n    </tmx>'     )
							with open(__a    ,     'w'     ) as f:
														f.write(__a     )
							return filename


_lowerCAmelCase:						str   				=       [
    {'col_1': '0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': '1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': '2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': '3', 'col_2': 3, 'col_3': 3.0},
]
_lowerCAmelCase:						Any   				=       [
    {'col_1': '4', 'col_2': 4, 'col_3': 4.0},
    {'col_1': '5', 'col_2': 5, 'col_3': 5.0},
]
_lowerCAmelCase:						Optional[Any]   				=       {
    'col_1': ['0', '1', '2', '3'],
    'col_2': [0, 1, 2, 3],
    'col_3': [0.0, 1.0, 2.0, 3.0],
}

_lowerCAmelCase:						List[Any]   				=       [
    {'col_3': 0.0, 'col_1': '0', 'col_2': 0},
    {'col_3': 1.0, 'col_1': '1', 'col_2': 1},
]

_lowerCAmelCase:						Optional[int]   				=       [
    {'col_1': 's0', 'col_2': 0, 'col_3': 0.0},
    {'col_1': 's1', 'col_2': 1, 'col_3': 1.0},
    {'col_1': 's2', 'col_2': 2, 'col_3': 2.0},
    {'col_1': 's3', 'col_2': 3, 'col_3': 3.0},
]






@pytest.fixture(scope='session'     )
def   _lowercase(					):
							return DATA_DICT_OF_LISTS






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]     ):
							a__      =datasets.Dataset.from_dict(__a     )
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.arrow'     )
							dataset.map(cache_file_name=__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.sqlite'     )
							with contextlib.closing(sqlitea.connect(__a     )     ) as con:
														a__      =con.cursor()
														cur.execute('CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)'     )
														for item in DATA:
																					cur.execute('INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'    ,     tuple(item.values()     )     )
														con.commit()
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.csv'     )
							with open(__a    ,     'w'    ,     newline=''     ) as f:
														a__      =csv.DictWriter(__a    ,     fieldnames=['col_1', 'col_2', 'col_3']     )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset2.csv'     )
							with open(__a    ,     'w'    ,     newline=''     ) as f:
														a__      =csv.DictWriter(__a    ,     fieldnames=['col_1', 'col_2', 'col_3']     )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						Tuple     ):
							import bza

							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.csv.bz2'
							with open(__a    ,     'rb'     ) as f:
														a__      =f.read()
							# data = bytes(FILE_CONTENT, "utf-8")
							with bza.open(__a    ,     'wb'     ) as f:
														f.write(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						Optional[Any]    ,     __a				:						int     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.csv.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Dict    ,     __a				:						str    ,     __a				:						Tuple     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.csv.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(csv_path.replace('.csv'    ,     '.CSV'     )     )     )
														f.write(__a    ,     arcname=os.path.basename(csva_path.replace('.csv'    ,     '.CSV'     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Any    ,     __a				:						str    ,     __a				:						Union[str, Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset_with_dir.csv.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.parquet'     )
							a__      =pa.schema(
							    {
							        'col_1': pa.string(),
							        'col_2': pa.intaa(),
							        'col_3': pa.floataa(),
							    }     )
							with open(__a    ,     'wb'     ) as f:
														a__      =pq.ParquetWriter(__a    ,     schema=__a     )
														a__      =pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(__a     )     )] for k in DATA[0]}    ,     schema=__a     )
														writer.write_table(__a     )
														writer.close()
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Dict     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.json'     )
							a__      ={'data': DATA}
							with open(__a    ,     'w'     ) as f:
														json.dump(__a    ,     __a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						str     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.json'     )
							a__      ={'data': DATA_DICT_OF_LISTS}
							with open(__a    ,     'w'     ) as f:
														json.dump(__a    ,     __a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.jsonl'     )
							with open(__a    ,     'w'     ) as f:
														for item in DATA:
																					f.write(json.dumps(__a     ) + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset2.jsonl'     )
							with open(__a    ,     'w'     ) as f:
														for item in DATA:
																					f.write(json.dumps(__a     ) + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Tuple     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset_312.jsonl'     )
							with open(__a    ,     'w'     ) as f:
														for item in DATA_312:
																					f.write(json.dumps(__a     ) + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Dict     ):
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset-str.jsonl'     )
							with open(__a    ,     'w'     ) as f:
														for item in DATA_STR:
																					f.write(json.dumps(__a     ) + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[str]    ,     __a				:						Union[str, Any]     ):
							import gzip

							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.txt.gz'     )
							with open(__a    ,     'rb'     ) as orig_file:
														with gzip.open(__a    ,     'wb'     ) as zipped_file:
																					zipped_file.writelines(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]    ,     __a				:						Tuple     ):
							import gzip

							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.jsonl.gz'     )
							with open(__a    ,     'rb'     ) as orig_file:
														with gzip.open(__a    ,     'wb'     ) as zipped_file:
																					zipped_file.writelines(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						Optional[int]    ,     __a				:						List[Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.jsonl.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Any    ,     __a				:						Optional[Any]    ,     __a				:						Tuple    ,     __a				:						Dict     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset_nested.jsonl.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.join('nested'    ,     os.path.basename(__a     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[int]    ,     __a				:						Optional[int]    ,     __a				:						Union[str, Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset_with_dir.jsonl.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]    ,     __a				:						Union[str, Any]    ,     __a				:						Any     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.jsonl.tar'
							with tarfile.TarFile(__a    ,     'w'     ) as f:
														f.add(__a    ,     arcname=os.path.basename(__a     )     )
														f.add(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						int    ,     __a				:						Union[str, Any]    ,     __a				:						str    ,     __a				:						Union[str, Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset_nested.jsonl.tar'
							with tarfile.TarFile(__a    ,     'w'     ) as f:
														f.add(__a    ,     arcname=os.path.join('nested'    ,     os.path.basename(__a     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Any     ):
							a__      =['0', '1', '2', '3']
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset.txt'     )
							with open(__a    ,     'w'     ) as f:
														for item in data:
																					f.write(item + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Any     ):
							a__      =['0', '1', '2', '3']
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset2.txt'     )
							with open(__a    ,     'w'     ) as f:
														for item in data:
																					f.write(item + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Any     ):
							a__      =['0', '1', '2', '3']
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.abc'
							with open(__a    ,     'w'     ) as f:
														for item in data:
																					f.write(item + '\n'     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[str]    ,     __a				:						Optional[Any]    ,     __a				:						Dict     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.text.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						int    ,     __a				:						Union[str, Any]    ,     __a				:						Optional[int]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset_with_dir.text.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
														f.write(__a    ,     arcname=os.path.join('main_dir'    ,     os.path.basename(__a     )     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]    ,     __a				:						Union[str, Any]    ,     __a				:						Union[str, Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.ext.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename('unsupported.ext'     )     )
														f.write(__a    ,     arcname=os.path.basename('unsupported_2.ext'     )     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Dict     ):
							a__      ='\n'.join(['First', 'Second\u2029with Unicode new line', 'Third']     )
							a__      =str(tmp_path_factory.mktemp('data'     ) / 'dataset_with_unicode_new_lines.txt'     )
							with open(__a    ,     'w'    ,     encoding='utf-8'     ) as f:
														f.write(__a     )
							return path






@pytest.fixture(scope='session'     )
def   _lowercase(					):
							return os.path.join('tests'    ,     'features'    ,     'data'    ,     'test_image_rgb.jpg'     )






@pytest.fixture(scope='session'     )
def   _lowercase(					):
							return os.path.join('tests'    ,     'features'    ,     'data'    ,     'test_audio_44100.wav'     )






@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						Optional[Any]    ,     __a				:						Optional[Any]     ):
							a__      =tmp_path_factory.mktemp('data'     ) / 'dataset.img.zip'
							with zipfile.ZipFile(__a    ,     'w'     ) as f:
														f.write(__a    ,     arcname=os.path.basename(__a     )     )
														f.write(__a    ,     arcname=os.path.basename(__a     ).replace('.jpg'    ,     '2.jpg'     )     )
							return path







@pytest.fixture(scope='session'     )
def   _lowercase(					__a				:						List[Any]     ):
							a__      =tmp_path_factory.mktemp('data_dir'     )

							(data_dir / "subdir").mkdir()
							with open(data_dir / 'subdir' / 'train.txt'    ,     'w'     ) as f:
														f.write('foo\n' * 10     )
							with open(data_dir / 'subdir' / 'test.txt'    ,     'w'     ) as f:
														f.write('bar\n' * 10     )
							# hidden file
							with open(data_dir / 'subdir' / '.test.txt'    ,     'w'     ) as f:
														f.write('bar\n' * 10     )

							# hidden directory
							(data_dir / ".subdir").mkdir()
							with open(data_dir / '.subdir' / 'train.txt'    ,     'w'     ) as f:
														f.write('foo\n' * 10     )
							with open(data_dir / '.subdir' / 'test.txt'    ,     'w'     ) as f:
														f.write('bar\n' * 10     )

							return data_dir


"""simple docstring"""


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class 							a					(		unittest.TestCase ):


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      _UpperCAmelCase    			=   0


    @slow
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(GPTaTokenizer, GPTaTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(RobertaTokenizer, RobertaTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					20       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   AutoConfig.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )
      # Check that tokenizer_type ≠ model_type
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					config=__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Tuple       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with pytest.raises(__lowerCAmelCase       ):
        AutoTokenizer.from_pretrained("""./""" ,					tokenizer_type="""xxx"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )

        if isinstance(__lowerCAmelCase ,					__lowerCAmelCase       ):
          self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,					__lowerCAmelCase       )
        else:
          self.assertEqual(tokenizer.do_lower_case ,					__lowerCAmelCase       )

        self.assertEqual(tokenizer.model_max_length ,					512       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        with self.assertRaisesRegex(
            __lowerCAmelCase ,					"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier""" ,					):
          _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      # tests: https://github.com/huggingface/transformers/pull/13251
      # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
      # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
      _UpperCAmelCase    			=   TOKENIZER_MAPPING.values()
      _UpperCAmelCase    			=   []

      for slow_tok, fast_tok in tokenizers:
        if slow_tok is not None:
          tokenizer_names.append(slow_tok.__name__       )

        if fast_tok is not None:
          tokenizer_names.append(fast_tok.__name__       )

      for tokenizer_name in tokenizer_names:
        # must find the right class
        tokenizer_class_from_name(__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased""" ,					use_fast=__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ) ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	int       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""distilbert-base-uncased""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   """Hello, world. How are you?"""
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""microsoft/mpnet-base""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""       )
      self.assertEqual(type(__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertEqual(tokenizer.model_max_length ,					512       )
      self.assertEqual(tokenizer.vocab_size ,					3_0000       )
      self.assertEqual(tokenizer.unk_token ,					"""[UNK]"""       )
      self.assertEqual(tokenizer.padding_side ,					"""right"""       )
      self.assertEqual(tokenizer.truncation_side ,					"""right"""       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )

      self.assertIsInstance(__lowerCAmelCase ,					tokenizer.__class__       )
      self.assertEqual(tokenizera.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""ctrl"""       )
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	str       ):
      # Check we can load the tokenizer config of an online model.
      _UpperCAmelCase    			=   get_tokenizer_config("""bert-base-cased"""       )
      _UpperCAmelCase    			=   config.pop("""_commit_hash""" ,					__lowerCAmelCase       )
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(__lowerCAmelCase ,					{"""do_lower_case""": False}       )

      # This model does not have a tokenizer_config so we get back an empty dict.
      _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )
      self.assertDictEqual(__lowerCAmelCase ,					{}       )

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["""tokenizer_class"""] ,					"""BertTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )

        _UpperCAmelCase    			=   CustomTokenizer.from_pretrained(__lowerCAmelCase       )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        # Can register in two steps
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, None)       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        del TOKENIZER_MAPPING._extra_content[CustomConfig]
        # Can register in one step
        AutoTokenizer.register(
            __lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )

        # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
        # and that model does not have a tokenizer.json
        with tempfile.TemporaryDirectory() as tmp_dir:
          _UpperCAmelCase    			=   BertTokenizerFast.from_pretrained(__lowerCAmelCase       )
          bert_tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   CustomTokenizerFast.from_pretrained(__lowerCAmelCase       )

        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # If remote code is not set, we will time out when asking whether to load the model.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(reloaded_tokenizer.special_attribute_present       )

      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(reloaded_tokenizer.special_attribute_present       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):


      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Tuple  				=				False



      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Optional[int]  				=				NewTokenizer
          _snake_case  :	Any  				=				False

      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        # If remote code is not set, the default is to use local
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote code is disabled, we load the local one.
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote is enabled, we load from the Hub
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertTrue(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(tokenizer.special_attribute_present       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
          """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					"""bert-base is not a local folder and is not a valid model identifier"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""bert-base"""       )


    def   lowerCAmelCase_	(       self						:	int       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					revision="""aaaaaa"""       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # Make sure we have cached the tokenizer.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
      with RequestCounter() as counter:
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
        self.assertEqual(counter.get_request_count ,					0       )
        self.assertEqual(counter.head_request_count ,					1       )
        self.assertEqual(counter.other_request_count ,					0       )
