



"""simple docstring"""






import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin






class 			__snake_case			(      _lowercase    ,	unittest.TestCase):
  snake_case__							:  List[str]         =		CTRLTokenizer
  snake_case__							:  List[str]         =		False
  snake_case__							:  Any         =		False




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         super().setUp()

         # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
         _lowerCamelCase		:							str            =					['''adapt''', '''re@@''', '''a@@''', '''apt''', '''c@@''', '''t''', '''<unk>''']
         _lowerCamelCase		:							Dict            =					dict(zip(__lowerCAmelCase		,   range(len(__lowerCAmelCase						)						)						)						)
         _lowerCamelCase		:							Optional[Any]            =					['''#version: 0.2''', '''a p''', '''ap t</w>''', '''r e''', '''a d''', '''ad apt</w>''', '''''']
         _lowerCamelCase		:							Any            =					{'''unk_token''': '''<unk>'''}

         _lowerCamelCase		:							Optional[int]            =					os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['''vocab_file''']						)
         _lowerCamelCase		:							Optional[int]            =					os.path.join(self.tmpdirname		,   VOCAB_FILES_NAMES['''merges_file''']						)
         with open(self.vocab_file		,   '''w'''		,   encoding='''utf-8'''						) as fp:
                fp.write(json.dumps(__lowerCAmelCase						) + '''\n'''						)
         with open(self.merges_file		,   '''w'''		,   encoding='''utf-8'''						) as fp:
                fp.write('''\n'''.join(__lowerCAmelCase						)						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Optional[int]		,   **__lowerCAmelCase	:		Optional[Any]						):






         """simple docstring"""






         kwargs.update(self.special_tokens_map						)
         return CTRLTokenizer.from_pretrained(self.tmpdirname		,   **__lowerCAmelCase						)




  def 		SCREAMING_SNAKE_CASE							(					self	:		Any		,   __lowerCAmelCase	:		str						):






         """simple docstring"""






         _lowerCamelCase		:							Dict            =					'''adapt react readapt apt'''
         _lowerCamelCase		:							Dict            =					'''adapt react readapt apt'''
         return input_text, output_text




  def 		SCREAMING_SNAKE_CASE							(					self	:		List[str]						):






         """simple docstring"""






         _lowerCamelCase		:							str            =					CTRLTokenizer(self.vocab_file		,   self.merges_file		,   **self.special_tokens_map						)
         _lowerCamelCase		:							List[Any]            =					'''adapt react readapt apt'''
         _lowerCamelCase		:							Union[str, Any]            =					'''adapt re@@ a@@ c@@ t re@@ adapt apt'''.split()
         _lowerCamelCase		:							Union[str, Any]            =					tokenizer.tokenize(__lowerCAmelCase						)
         self.assertListEqual(__lowerCAmelCase		,   __lowerCAmelCase						)

         _lowerCamelCase		:							Optional[Any]            =					tokens + [tokenizer.unk_token]

         _lowerCamelCase		:							Any            =					[0, 1, 2, 4, 5, 1, 0, 3, 6]
         self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCAmelCase						)		,   __lowerCAmelCase						)








'''simple docstring'''







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

lowerCAmelCase_     						=				'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
    raise ImportWarning(
        '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
    )

if version.parse(pyarrow.__version__).major < 8:
    raise ImportWarning(
        '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
        '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
    )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

lowerCAmelCase_     						=				concatenate_datasets
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadManager
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


