




'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
    from .tokenization_fnet import FNetTokenizer
else:
    lowerCAmelCase_     						=				None

lowerCAmelCase_     						=				logging.get_logger(__name__)
lowerCAmelCase_     						=				{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCAmelCase_     						=				{
    '''vocab_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/spiece.model''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''google/fnet-base''': '''https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json''',
        '''google/fnet-large''': '''https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json''',
    },
}

lowerCAmelCase_     						=				{
    '''google/fnet-base''': 5_1_2,
    '''google/fnet-large''': 5_1_2,
}

lowerCAmelCase_     						=				'''▁'''

class 	_snake_case(			UpperCAmelCase      ):
 __snake_case:							Union[str, Any]						=					VOCAB_FILES_NAMES
 __snake_case:							Optional[Any]						=					PRETRAINED_VOCAB_FILES_MAP
 __snake_case:							Union[str, Any]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:							List[str]						=					['''input_ids''', '''token_type_ids''']
 __snake_case:							List[Any]						=					FNetTokenizer






 def __init__(self       :  List[Any]    ,		a       :  str=None    ,		a       :  str=None    ,		a       :  Tuple=False    ,		a       :  Any=True    ,		a       :  int=True    ,		a       :  List[str]="<unk>"    ,		a       :  Dict="[SEP]"    ,		a       :  Optional[Any]="<pad>"    ,		a       :  Optional[Any]="[CLS]"    ,		a       :  Dict="[MASK]"    ,		**a       :  List[str]    ,		)       ->   int:






    """simple docstring"""


    A__       =     (
        AddedToken(a    ,		lstrip=a    ,		rstrip=a    ,		normalized=a    )
        if isinstance(a    ,		a    )
        else mask_token
    )

    super().__init__(
        a    ,		tokenizer_file=a    ,		do_lower_case=a    ,		remove_space=a    ,		keep_accents=a    ,		unk_token=a    ,		sep_token=a    ,		pad_token=a    ,		cls_token=a    ,		mask_token=a    ,		**a    ,		)

    A__       =     do_lower_case
    A__       =     remove_space
    A__       =     keep_accents
    A__       =     vocab_file
    A__       =     False if not self.vocab_file else True






 def 						_UpperCamelCase (self       :  Tuple    ,		a       :  List[int]    ,		a       :  Optional[List[int]] = None    )       ->   List[int]:






    """simple docstring"""


    A__       =     [self.sep_token_id]
    A__       =     [self.cls_token_id]
    if token_ids_a is None:
       return cls + token_ids_a + sep
    return cls + token_ids_a + sep + token_ids_a + sep






 def 						_UpperCamelCase (self       :  List[str]    ,		a       :  List[int]    ,		a       :  Optional[List[int]] = None    )       ->   List[int]:






    """simple docstring"""


    A__       =     [self.sep_token_id]
    A__       =     [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep    ) * [0]
    return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






 def 						_UpperCamelCase (self       :  List[str]    ,		a       :  str    ,		a       :  Optional[str] = None    )       ->   Tuple[str]:






    """simple docstring"""


    if not os.path.isdir(a    ):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
       return
    A__       =     os.path.join(
        a    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

    if os.path.abspath(self.vocab_file    ) != os.path.abspath(a    ):
       copyfile(self.vocab_file    ,		a    )

    return (out_vocab_file,)








'''simple docstring'''







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

lowerCAmelCase_     						=				'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
    raise ImportWarning(
        '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
    )

if version.parse(pyarrow.__version__).major < 8:
    raise ImportWarning(
        '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
        '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
    )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

lowerCAmelCase_     						=				concatenate_datasets
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadManager
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


