import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


A_:   int   =      logging.get_logger(__name__)
A_:   Any   =      {"""vocab_file""": """spiece.model"""}

A_:   str   =      {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

A_:   Union[str, Any]   =      {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

A_:   List[str]   =      """▁"""


class _lowercase       (							_UpperCAmelCase    ):





							"""simple docstring"""




							lowerCAmelCase__							=       VOCAB_FILES_NAMES
							lowerCAmelCase__							=       PRETRAINED_VOCAB_FILES_MAP
							lowerCAmelCase__							=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



							def __init__(     self  , UpperCAmelCase  , UpperCAmelCase=True  , UpperCAmelCase=True  , UpperCAmelCase=False  , UpperCAmelCase="[CLS]"  , UpperCAmelCase="[SEP]"  , UpperCAmelCase="<unk>"  , UpperCAmelCase="[SEP]"  , UpperCAmelCase="<pad>"  , UpperCAmelCase="[CLS]"  , UpperCAmelCase="[MASK]"  , UpperCAmelCase = None  , **UpperCAmelCase  , ):







														'''simple docstring'''






														_lowercase					=						(
														    AddedToken(__lowerCAmelCase  , lstrip=__lowerCAmelCase  , rstrip=__lowerCAmelCase  , normalized=__lowerCAmelCase	)
														    if isinstance(__lowerCAmelCase  , __lowerCAmelCase	)
														    else mask_token
														)

														_lowercase					=						{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=__lowerCAmelCase  , remove_space=__lowerCAmelCase  , keep_accents=__lowerCAmelCase  , bos_token=__lowerCAmelCase  , eos_token=__lowerCAmelCase  , unk_token=__lowerCAmelCase  , sep_token=__lowerCAmelCase  , pad_token=__lowerCAmelCase  , cls_token=__lowerCAmelCase  , mask_token=__lowerCAmelCase  , sp_model_kwargs=self.sp_model_kwargs  , **__lowerCAmelCase  , )

														_lowercase					=						do_lower_case
														_lowercase					=						remove_space
														_lowercase					=						keep_accents
														_lowercase					=						vocab_file

														_lowercase					=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(__lowerCAmelCase	)



							@property
							def      _UpperCAmelCase							(     self	):







														'''simple docstring'''






														return len(self.sp_model	)



							def      _UpperCAmelCase							(     self	):







														'''simple docstring'''






														_lowercase					=						{self.convert_ids_to_tokens(__lowerCAmelCase	): i for i in range(self.vocab_size	)}
														vocab.update(self.added_tokens_encoder	)
														return vocab



							def __getstate__(     self	):







														'''simple docstring'''






														_lowercase					=						self.__dict__.copy()
														_lowercase					=						None
														return state



							def __setstate__(     self  , UpperCAmelCase	):







														'''simple docstring'''






														_lowercase					=						d

														# for backward compatibility
														if not hasattr(self  , """sp_model_kwargs"""	):
																					_lowercase					=						{}

														_lowercase					=						spm.SentencePieceProcessor(**self.sp_model_kwargs	)
														self.sp_model.Load(self.vocab_file	)



							def      _UpperCAmelCase							(     self  , UpperCAmelCase	):







														'''simple docstring'''






														if self.remove_space:
																					_lowercase					=						''' '''.join(inputs.strip().split()	)
														else:
																					_lowercase					=						inputs
														_lowercase					=						outputs.replace("""``"""  , """\""""	).replace("""\'\'"""  , """\""""	)

														if not self.keep_accents:
																					_lowercase					=						unicodedata.normalize("""NFKD"""  , __lowerCAmelCase	)
																					_lowercase					=						''''''.join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase	)]	)
														if self.do_lower_case:
																					_lowercase					=						outputs.lower()

														return outputs



							def      _UpperCAmelCase							(     self  , UpperCAmelCase	):







														'''simple docstring'''






														_lowercase					=						self.preprocess_text(__lowerCAmelCase	)
														_lowercase					=						self.sp_model.encode(__lowerCAmelCase  , out_type=__lowerCAmelCase	)
														_lowercase					=						[]
														for piece in pieces:
																					if len(__lowerCAmelCase	) > 1 and piece[-1] == str(""","""	) and piece[-2].isdigit():
																												_lowercase					=						self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase  , """"""	)	)
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]	) == 1:
																																										_lowercase					=						cur_pieces[1:]
																																			else:
																																										_lowercase					=						cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]	)
																												new_pieces.extend(__lowerCAmelCase	)
																					else:
																												new_pieces.append(__lowerCAmelCase	)

														return new_pieces



							def      _UpperCAmelCase							(     self  , UpperCAmelCase	):







														'''simple docstring'''






														return self.sp_model.PieceToId(__lowerCAmelCase	)



							def      _UpperCAmelCase							(     self  , UpperCAmelCase	):







														'''simple docstring'''






														return self.sp_model.IdToPiece(__lowerCAmelCase	)



							def      _UpperCAmelCase							(     self  , UpperCAmelCase	):







														'''simple docstring'''






														_lowercase					=						[]
														_lowercase					=						''''''
														_lowercase					=						False
														for token in tokens:
																					# make sure that special tokens are not decoded using sentencepiece model
																					if token in self.all_special_tokens:
																												if not prev_is_special:
																																			out_string += " "
																												out_string += self.sp_model.decode(__lowerCAmelCase	) + token
																												_lowercase					=						True
																												_lowercase					=						[]
																					else:
																												current_sub_tokens.append(__lowerCAmelCase	)
																												_lowercase					=						False
														out_string += self.sp_model.decode(__lowerCAmelCase	)
														return out_string.strip()



							def      _UpperCAmelCase							(     self  , UpperCAmelCase  , UpperCAmelCase = None	):







														'''simple docstring'''






														_lowercase					=						[self.sep_token_id]
														_lowercase					=						[self.cls_token_id]
														if token_ids_a is None:
																					return cls + token_ids_a + sep
														return cls + token_ids_a + sep + token_ids_a + sep



							def      _UpperCAmelCase							(     self  , UpperCAmelCase  , UpperCAmelCase = None  , UpperCAmelCase = False	):







														'''simple docstring'''






														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase  , token_ids_a=__lowerCAmelCase  , already_has_special_tokens=__lowerCAmelCase	)

														if token_ids_a is not None:
																					return [1] + ([0] * len(__lowerCAmelCase	)) + [1] + ([0] * len(__lowerCAmelCase	)) + [1]
														return [1] + ([0] * len(__lowerCAmelCase	)) + [1]



							def      _UpperCAmelCase							(     self  , UpperCAmelCase  , UpperCAmelCase = None	):







														'''simple docstring'''






														_lowercase					=						[self.sep_token_id]
														_lowercase					=						[self.cls_token_id]

														if token_ids_a is None:
																					return len(cls + token_ids_a + sep	) * [0]
														return len(cls + token_ids_a + sep	) * [0] + len(token_ids_a + sep	) * [1]



							def      _UpperCAmelCase							(     self  , UpperCAmelCase  , UpperCAmelCase = None	):







														'''simple docstring'''






														if not os.path.isdir(__lowerCAmelCase	):
																					logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''	)
																					return
														_lowercase					=						os.path.join(
														    __lowerCAmelCase  , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

														if os.path.abspath(self.vocab_file	) != os.path.abspath(__lowerCAmelCase	) and os.path.isfile(self.vocab_file	):
																					copyfile(self.vocab_file  , __lowerCAmelCase	)
														elif not os.path.isfile(self.vocab_file	):
																					with open(__lowerCAmelCase  , """wb"""	) as fi:
																												_lowercase					=						self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase	)

														return (out_vocab_file,)










import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				assert isinstance(snake_case,	snake_case			)
				assert dataset.num_rows == 4
				assert dataset.num_columns == 3
				assert dataset.column_names == ["col_1", "col_2", "col_3"]
				for feature, expected_dtype in expected_features.items():
								assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('''keep_in_memory''',	[False, True]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Tuple   =   tmp_path / '''cache'''
				__magic_name__		:int   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								__magic_name__		:Optional[Any]   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case,	keep_in_memory=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:List[str]   =   tmp_path / '''cache'''
				__magic_name__		:int   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:Tuple   =   features.copy() if features else default_expected_features
				__magic_name__		:Union[str, Any]   =   (
				    Features({feature: Value(snake_case			) for feature, dtype in features.items()}			) if features is not None else None
				)
				__magic_name__		:int   =   ParquetDatasetReader(snake_case,	features=snake_case,	cache_dir=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''			), '''train''', '''test''']			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:str   =   tmp_path / '''cache'''
				__magic_name__		:List[Any]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:int   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case,	split=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)
				assert dataset.split == split if split else "train"

@pytest.mark.parametrize('''path_type''',	[str, list]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				if issubclass(snake_case,	snake_case			):
								__magic_name__		:Union[str, Any]   =   parquet_path
				elif issubclass(snake_case,	snake_case			):
								__magic_name__		:Union[str, Any]   =   [parquet_path]
				__magic_name__		:Optional[int]   =   tmp_path / '''cache'''
				__magic_name__		:Optional[int]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:str   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case			).read()
				_check_parquet_dataset(snake_case,	snake_case			)

def   __lowercase       (			snake_case,	snake_case,	snake_case=("train",)			):







				"""simple docstring"""






				assert isinstance(snake_case,	snake_case			)
				for split in splits:
								__magic_name__		:Optional[Any]   =   dataset_dict[split]
								assert dataset.num_rows == 4
								assert dataset.num_columns == 3
								assert dataset.column_names == ["col_1", "col_2", "col_3"]
								for feature, expected_dtype in expected_features.items():
												assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize('''keep_in_memory''',	[False, True]			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Any   =   tmp_path / '''cache'''
				__magic_name__		:Optional[int]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
								__magic_name__		:Tuple   =   ParquetDatasetReader(
								    {'''train''': parquet_path},	cache_dir=snake_case,	keep_in_memory=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case			)

@pytest.mark.parametrize(
    '''features''',	[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ],	)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:Optional[Any]   =   tmp_path / '''cache'''
				__magic_name__		:Dict   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:int   =   features.copy() if features else default_expected_features
				__magic_name__		:List[Any]   =   (
				    Features({feature: Value(snake_case			) for feature, dtype in features.items()}			) if features is not None else None
				)
				__magic_name__		:Optional[int]   =   ParquetDatasetReader({'''train''': parquet_path},	features=snake_case,	cache_dir=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case			)

@pytest.mark.parametrize('''split''',	[None, NamedSplit('''train'''			), '''train''', '''test''']			)
def   __lowercase       (			snake_case,	snake_case,	snake_case			):







				"""simple docstring"""






				if split:
								__magic_name__		:Dict   =   {split: parquet_path}
				else:
								__magic_name__		:Optional[int]   =   '''train'''
								__magic_name__		:Dict   =   {'''train''': parquet_path, '''test''': parquet_path}
				__magic_name__		:List[Any]   =   tmp_path / '''cache'''
				__magic_name__		:Optional[Any]   =   {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
				__magic_name__		:Optional[Any]   =   ParquetDatasetReader(snake_case,	cache_dir=snake_case			).read()
				_check_parquet_datasetdict(snake_case,	snake_case,	splits=list(path.keys()			)			)
				assert all(dataset[split].split == split for split in path.keys()			)

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:str   =   ParquetDatasetWriter(snake_case,	tmp_path / '''foo.parquet'''			)
				assert writer.write() > 0
				__magic_name__		:List[Any]   =   pq.ParquetFile(tmp_path / '''foo.parquet'''			)
				__magic_name__		:List[Any]   =   pf.read()
				assert dataset.data.table == output_table

def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				__magic_name__		:List[str]   =   str(shared_datadir / '''test_image_rgb.jpg'''			)
				__magic_name__		:Tuple   =   {'''image''': [image_path]}
				__magic_name__		:List[Any]   =   Features({'''image''': Image()}			)
				__magic_name__		:Tuple   =   Dataset.from_dict(snake_case,	features=snake_case			)
				__magic_name__		:Union[str, Any]   =   ParquetDatasetWriter(snake_case,	tmp_path / '''foo.parquet'''			)
				assert writer.write() > 0

				__magic_name__		:List[str]   =   Dataset.from_parquet(str(tmp_path / '''foo.parquet'''			)			)
				assert dataset.features == reloaded_dataset.features

				__magic_name__		:List[str]   =   ParquetDatasetReader(str(tmp_path / '''foo.parquet'''			),	streaming=snake_case			).read()
				assert dataset.features == reloaded_iterable_dataset.features

@pytest.mark.parametrize(
    '''feature, expected''',	[
        (Features({'''foo''': Value('''int32'''			)}			), None),
        (Features({'''image''': Image(), '''foo''': Value('''int32'''			)}			), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({'''nested''': Sequence(Audio()			)}			), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ],	)
def   __lowercase       (			snake_case,	snake_case			):







				"""simple docstring"""






				assert get_writer_batch_size(snake_case			) == expected






