




'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase_     						=				logging.get_logger(__name__)

lowerCAmelCase_     						=				'''▁'''

lowerCAmelCase_     						=				{'''vocab_file''': '''sentencepiece.bpe.model''', '''monolingual_vocab_file''': '''dict.txt'''}

lowerCAmelCase_     						=				{
    '''vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model''',
    },
    '''monolingual_vocab_file''': {
        '''vinai/bartpho-syllable''': '''https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt''',
    },
}

lowerCAmelCase_     						=				{'''vinai/bartpho-syllable''': 1_0_2_4}

class 	_snake_case(			UpperCAmelCase      ):
 __snake_case:							Union[str, Any]						=					VOCAB_FILES_NAMES
 __snake_case:							Dict						=					PRETRAINED_VOCAB_FILES_MAP
 __snake_case:							Dict						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __snake_case:							Optional[Any]						=					['''input_ids''', '''attention_mask''']






 def __init__(self       :  Optional[Any]    ,		a       :  List[str]    ,		a       :  Dict    ,		a       :  List[str]="<s>"    ,		a       :  List[Any]="</s>"    ,		a       :  List[Any]="</s>"    ,		a       :  Optional[int]="<s>"    ,		a       :  Dict="<unk>"    ,		a       :  Optional[int]="<pad>"    ,		a       :  Any="<mask>"    ,		a       :  Optional[Dict[str, Any]] = None    ,		**a       :  Optional[int]    ,		)       ->   None:






    """simple docstring"""


    A__       =     AddedToken(a    ,		lstrip=a    ,		rstrip=a    ) if isinstance(a    ,		a    ) else mask_token

    A__       =     {} if sp_model_kwargs is None else sp_model_kwargs

    super().__init__(
        bos_token=a    ,		eos_token=a    ,		unk_token=a    ,		sep_token=a    ,		cls_token=a    ,		pad_token=a    ,		mask_token=a    ,		sp_model_kwargs=self.sp_model_kwargs    ,		**a    ,		)

    A__       =     vocab_file
    A__       =     monolingual_vocab_file
    A__       =     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
    self.sp_model.Load(str(a    )    )

    # Load the reduced vocab

    # Keep order of special tokens for backward compatibility
    A__       =     {}
    A__       =     0
    for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
       if str(a    ) not in self.fairseq_tokens_to_ids:
          A__       =     cnt
          cnt += 1
    with open(a    ,		'r'    ,		encoding='utf-8'    ) as f:
       for line in f.readlines():
          A__       =     line.strip().split()[0]
          A__       =     len(self.fairseq_tokens_to_ids    )
    if str(a    ) not in self.fairseq_tokens_to_ids:
       A__       =     len(self.fairseq_tokens_to_ids    )

    A__       =     {v: k for k, v in self.fairseq_tokens_to_ids.items()}






 def __getstate__(self       :  List[Any]    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     self.__dict__.copy()
    A__       =     None
    A__       =     self.sp_model.serialized_model_proto()
    return state






 def __setstate__(self       :  Any    ,		a       :  int    )       ->   Any:






    """simple docstring"""


    A__       =     d

    # for backward compatibility
    if not hasattr(self    ,		'sp_model_kwargs'    ):
       A__       =     {}

    A__       =     spm.SentencePieceProcessor(**self.sp_model_kwargs    )
    self.sp_model.LoadFromSerializedProto(self.sp_model_proto    )






 def 						_UpperCamelCase (self       :  Union[str, Any]    ,		a       :  List[int]    ,		a       :  Optional[List[int]] = None    )       ->   List[int]:






    """simple docstring"""


    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    A__       =     [self.cls_token_id]
    A__       =     [self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep






 def 						_UpperCamelCase (self       :  Any    ,		a       :  List[int]    ,		a       :  Optional[List[int]] = None    ,		a       :  bool = False    )       ->   List[int]:






    """simple docstring"""


    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=a    ,		token_ids_a=a    ,		already_has_special_tokens=a    )

    if token_ids_a is None:
       return [1] + ([0] * len(a    )) + [1]
    return [1] + ([0] * len(a    )) + [1, 1] + ([0] * len(a    )) + [1]






 def 						_UpperCamelCase (self       :  Optional[int]    ,		a       :  List[int]    ,		a       :  Optional[List[int]] = None    )       ->   List[int]:






    """simple docstring"""


    A__       =     [self.sep_token_id]
    A__       =     [self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep    ) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]






 @property
 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   List[str]:






    """simple docstring"""


    return len(self.fairseq_ids_to_tokens    )






 def 						_UpperCamelCase (self       :  Tuple    )       ->   Any:






    """simple docstring"""


    A__       =     {self.convert_ids_to_tokens(a    ): i for i in range(self.vocab_size    )}
    vocab.update(self.added_tokens_encoder    )
    return vocab






 def 						_UpperCamelCase (self       :  Optional[Any]    ,		a       :  str    )       ->   List[str]:






    """simple docstring"""


    return self.sp_model.encode(a    ,		out_type=a    )






 def 						_UpperCamelCase (self       :  Dict    ,		a       :  Union[str, Any]    )       ->   str:






    """simple docstring"""


    if token in self.fairseq_tokens_to_ids:
       return self.fairseq_tokens_to_ids[token]
    else:
       return self.unk_token_id






 def 						_UpperCamelCase (self       :  Optional[int]    ,		a       :  int    )       ->   Tuple:






    """simple docstring"""


    return self.fairseq_ids_to_tokens[index]






 def 						_UpperCamelCase (self       :  Optional[Any]    ,		a       :  Tuple    )       ->   Union[str, Any]:






    """simple docstring"""


    A__       =     ''.join(a    ).replace(a    ,		' '    ).strip()
    return out_string






 def 						_UpperCamelCase (self       :  List[str]    ,		a       :  str    ,		a       :  Optional[str] = None    )       ->   Tuple[str]:






    """simple docstring"""


    if not os.path.isdir(a    ):
       logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
       return
    A__       =     os.path.join(
        a    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )
    A__       =     os.path.join(
        a    ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file']    ,		)

    if os.path.abspath(self.vocab_file    ) != os.path.abspath(a    ) and os.path.isfile(self.vocab_file    ):
       copyfile(self.vocab_file    ,		a    )
    elif not os.path.isfile(self.vocab_file    ):
       with open(a    ,		'wb'    ) as fi:
          A__       =     self.sp_model.serialized_model_proto()
          fi.write(a    )

    if os.path.abspath(self.monolingual_vocab_file    ) != os.path.abspath(
        a    ) and os.path.isfile(self.monolingual_vocab_file    ):
       copyfile(self.monolingual_vocab_file    ,		a    )
    elif not os.path.isfile(self.monolingual_vocab_file    ):
       with open(a    ,		'w'    ,		encoding='utf-8'    ) as fp:
          for token in self.fairseq_tokens_to_ids:
             if token not in self.all_special_tokens:
                fp.write(f"""{str(a    )} \n"""    )

    return out_vocab_file, out_monolingual_vocab_file








'''simple docstring'''







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

lowerCAmelCase_     						=				'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
    raise ImportWarning(
        '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
    )

if version.parse(pyarrow.__version__).major < 8:
    raise ImportWarning(
        '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
        '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
    )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

lowerCAmelCase_     						=				concatenate_datasets
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadManager
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadConfig
lowerCAmelCase_     						=				DownloadMode
lowerCAmelCase_     						=				DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager


