


"""simple docstring"""


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCamelCase						:				Optional[int]             =      logging.get_logger(__name__)

UpperCamelCase						:				Any             =      "▁"

UpperCamelCase						:				int             =      {"vocab_file": "sentencepiece.bpe.model"}

UpperCamelCase						:				List[Any]             =      {
    "vocab_file": {
        "xlm-roberta-base": "https://huggingface.co/xlm-roberta-base/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large": "https://huggingface.co/xlm-roberta-large/resolve/main/sentencepiece.bpe.model",
        "xlm-roberta-large-finetuned-conll02-dutch": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-dutch/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll02-spanish": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll02-spanish/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-english": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-english/resolve/main/sentencepiece.bpe.model"
        ),
        "xlm-roberta-large-finetuned-conll03-german": (
            "https://huggingface.co/xlm-roberta-large-finetuned-conll03-german/resolve/main/sentencepiece.bpe.model"
        ),
    }
}

UpperCamelCase						:				Optional[int]             =      {
    "xlm-roberta-base": 512,
    "xlm-roberta-large": 512,
    "xlm-roberta-large-finetuned-conll02-dutch": 512,
    "xlm-roberta-large-finetuned-conll02-spanish": 512,
    "xlm-roberta-large-finetuned-conll03-english": 512,
    "xlm-roberta-large-finetuned-conll03-german": 512,
}



class 			lowerCamelCase__						(   UpperCAmelCase_			):
						lowerCAmelCase       =     VOCAB_FILES_NAMES
						lowerCAmelCase       =     PRETRAINED_VOCAB_FILES_MAP
						lowerCAmelCase       =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						lowerCAmelCase       =     ["""input_ids""", """attention_mask"""]







						def __init__(							self   :    str			,			_lowercase   :    str			,			_lowercase   :    Dict="<s>"			,			_lowercase   :    Union[str, Any]="</s>"			,			_lowercase   :    Tuple="</s>"			,			_lowercase   :    Optional[int]="<s>"			,			_lowercase   :    Optional[int]="<unk>"			,			_lowercase   :    Dict="<pad>"			,			_lowercase   :    Union[str, Any]="<mask>"			,			_lowercase   :    Optional[Dict[str, Any]] = None			,			**_lowercase   :    Optional[int]			,			):
								# Mask token behave like a normal word, i.e. include the space before it
								A					       = AddedToken(_lowercase			,			lstrip=_lowercase			,			rstrip=_lowercase		) if isinstance(_lowercase			,			_lowercase		) else mask_token

								A					       = {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    bos_token=_lowercase			,			eos_token=_lowercase			,			unk_token=_lowercase			,			sep_token=_lowercase			,			cls_token=_lowercase			,			pad_token=_lowercase			,			mask_token=_lowercase			,			sp_model_kwargs=self.sp_model_kwargs			,			**_lowercase			,			)

								A					       = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
								self.sp_model.Load(str(_lowercase		)		)
								A					       = vocab_file

								# Original fairseq vocab and spm vocab must be "aligned":
								# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
								# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
								# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
								# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

								# Mimic fairseq token-to-id alignment for the first 4 token
								A					       = {'<s>': 0, '<pad>': 1, '</s>': 2, '<unk>': 3}

								# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
								A					       = 1

								A					       = len(self.sp_model		) + self.fairseq_offset
								A					       = {v: k for k, v in self.fairseq_tokens_to_ids.items()}







						def __getstate__(							self   :    Tuple		):
								A					       = self.__dict__.copy()
								A					       = None
								A					       = self.sp_model.serialized_model_proto()
								return state







						def __setstate__(							self   :    Tuple			,			_lowercase   :    Optional[Any]		):
								A					       = d

								# for backward compatibility
								if not hasattr(self			,			'sp_model_kwargs'		):
										A					       = {}

								A					       = spm.SentencePieceProcessor(**self.sp_model_kwargs		)
								self.sp_model.LoadFromSerializedProto(self.sp_model_proto		)







						def __a						(							self   :    List[Any]			,			_lowercase   :    List[int]			,			_lowercase   :    Optional[List[int]] = None		):

								if token_ids_a is None:
										return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								A					       = [self.cls_token_id]
								A					       = [self.sep_token_id]
								return cls + token_ids_a + sep + sep + token_ids_a + sep







						def __a						(							self   :    Dict			,			_lowercase   :    List[int]			,			_lowercase   :    Optional[List[int]] = None			,			_lowercase   :    bool = False		):

								if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_lowercase			,			token_ids_a=_lowercase			,			already_has_special_tokens=_lowercase		)

								if token_ids_a is None:
										return [1] + ([0] * len(_lowercase		)) + [1]
								return [1] + ([0] * len(_lowercase		)) + [1, 1] + ([0] * len(_lowercase		)) + [1]







						def __a						(							self   :    int			,			_lowercase   :    List[int]			,			_lowercase   :    Optional[List[int]] = None		):
								A					       = [self.sep_token_id]
								A					       = [self.cls_token_id]

								if token_ids_a is None:
										return len(cls + token_ids_a + sep		) * [0]
								return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]







						@property
						def __a						(							self   :    Dict		):
								return len(self.sp_model		) + self.fairseq_offset + 1  # Add the <mask> token







						def __a						(							self   :    Tuple		):
								A					       = {self.convert_ids_to_tokens(_lowercase		): i for i in range(self.vocab_size		)}
								vocab.update(self.added_tokens_encoder		)
								return vocab







						def __a						(							self   :    Any			,			_lowercase   :    str		):
								return self.sp_model.encode(_lowercase			,			out_type=_lowercase		)







						def __a						(							self   :    Optional[Any]			,			_lowercase   :    Union[str, Any]		):
								if token in self.fairseq_tokens_to_ids:
										return self.fairseq_tokens_to_ids[token]
								A					       = self.sp_model.PieceToId(_lowercase		)

								# Need to return unknown token if the SP model returned 0
								return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







						def __a						(							self   :    List[str]			,			_lowercase   :    Tuple		):
								if index in self.fairseq_ids_to_tokens:
										return self.fairseq_ids_to_tokens[index]
								return self.sp_model.IdToPiece(index - self.fairseq_offset		)







						def __a						(							self   :    Optional[int]			,			_lowercase   :    int		):
								A					       = ''.join(_lowercase		).replace(_lowercase			,			' '		).strip()
								return out_string







						def __a						(							self   :    Tuple			,			_lowercase   :    str			,			_lowercase   :    Optional[str] = None		):
								if not os.path.isdir(_lowercase		):
										logger.error(f'Vocabulary path ({save_directory}) should be a directory'		)
										return
								A					       = os.path.join(
								    _lowercase			,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

								if os.path.abspath(self.vocab_file		) != os.path.abspath(_lowercase		) and os.path.isfile(self.vocab_file		):
										copyfile(self.vocab_file			,			_lowercase		)
								elif not os.path.isfile(self.vocab_file		):
										with open(_lowercase			,			'wb'		) as fi:
												A					       = self.sp_model.serialized_model_proto()
												fi.write(_lowercase		)

								return (out_vocab_file,)









"""simple docstring"""


import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible



class 			lowerCamelCase__						(   unittest.TestCase			):







						def __a						(							self   :    Dict		):
								A					       = [
								    'safety_checker/pytorch_model.bin',
								    'safety_checker/model.safetensors',
								    'vae/diffusion_pytorch_model.bin',
								    'vae/diffusion_pytorch_model.safetensors',
								    'text_encoder/pytorch_model.bin',
								    'text_encoder/model.safetensors',
								    'unet/diffusion_pytorch_model.bin',
								    'unet/diffusion_pytorch_model.safetensors',
								]
								self.assertTrue(is_safetensors_compatible(_lowercase		)		)







						def __a						(							self   :    Union[str, Any]		):
								A					       = [
								    'unet/diffusion_pytorch_model.bin',
								    'unet/diffusion_pytorch_model.safetensors',
								]
								self.assertTrue(is_safetensors_compatible(_lowercase		)		)







						def __a						(							self   :    int		):
								A					       = [
								    'safety_checker/pytorch_model.bin',
								    'safety_checker/model.safetensors',
								    'vae/diffusion_pytorch_model.bin',
								    'vae/diffusion_pytorch_model.safetensors',
								    'text_encoder/pytorch_model.bin',
								    'text_encoder/model.safetensors',
								    'unet/diffusion_pytorch_model.bin',
								    # Removed: 'unet/diffusion_pytorch_model.safetensors',
								]
								self.assertFalse(is_safetensors_compatible(_lowercase		)		)







						def __a						(							self   :    Dict		):
								A					       = [
								    'text_encoder/pytorch_model.bin',
								    'text_encoder/model.safetensors',
								]
								self.assertTrue(is_safetensors_compatible(_lowercase		)		)







						def __a						(							self   :    Union[str, Any]		):
								A					       = [
								    'safety_checker/pytorch_model.bin',
								    'safety_checker/model.safetensors',
								    'vae/diffusion_pytorch_model.bin',
								    'vae/diffusion_pytorch_model.safetensors',
								    'text_encoder/pytorch_model.bin',
								    # Removed: 'text_encoder/model.safetensors',
								    'unet/diffusion_pytorch_model.bin',
								    'unet/diffusion_pytorch_model.safetensors',
								]
								self.assertFalse(is_safetensors_compatible(_lowercase		)		)







						def __a						(							self   :    Any		):
								A					       = [
								    'safety_checker/pytorch_model.fp16.bin',
								    'safety_checker/model.fp16.safetensors',
								    'vae/diffusion_pytorch_model.fp16.bin',
								    'vae/diffusion_pytorch_model.fp16.safetensors',
								    'text_encoder/pytorch_model.fp16.bin',
								    'text_encoder/model.fp16.safetensors',
								    'unet/diffusion_pytorch_model.fp16.bin',
								    'unet/diffusion_pytorch_model.fp16.safetensors',
								]
								A					       = 'fp16'
								self.assertTrue(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    Union[str, Any]		):
								A					       = [
								    'unet/diffusion_pytorch_model.fp16.bin',
								    'unet/diffusion_pytorch_model.fp16.safetensors',
								]
								A					       = 'fp16'
								self.assertTrue(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    Optional[Any]		):
								# pass variant but use the non-variant filenames
								A					       = [
								    'unet/diffusion_pytorch_model.bin',
								    'unet/diffusion_pytorch_model.safetensors',
								]
								A					       = 'fp16'
								self.assertTrue(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    Dict		):
								A					       = [
								    'safety_checker/pytorch_model.fp16.bin',
								    'safety_checker/model.fp16.safetensors',
								    'vae/diffusion_pytorch_model.fp16.bin',
								    'vae/diffusion_pytorch_model.fp16.safetensors',
								    'text_encoder/pytorch_model.fp16.bin',
								    'text_encoder/model.fp16.safetensors',
								    'unet/diffusion_pytorch_model.fp16.bin',
								    # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
								]
								A					       = 'fp16'
								self.assertFalse(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    Dict		):
								A					       = [
								    'text_encoder/pytorch_model.fp16.bin',
								    'text_encoder/model.fp16.safetensors',
								]
								A					       = 'fp16'
								self.assertTrue(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    Any		):
								# pass variant but use the non-variant filenames
								A					       = [
								    'text_encoder/pytorch_model.bin',
								    'text_encoder/model.safetensors',
								]
								A					       = 'fp16'
								self.assertTrue(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)







						def __a						(							self   :    List[str]		):
								A					       = [
								    'safety_checker/pytorch_model.fp16.bin',
								    'safety_checker/model.fp16.safetensors',
								    'vae/diffusion_pytorch_model.fp16.bin',
								    'vae/diffusion_pytorch_model.fp16.safetensors',
								    'text_encoder/pytorch_model.fp16.bin',
								    # 'text_encoder/model.fp16.safetensors',
								    'unet/diffusion_pytorch_model.fp16.bin',
								    'unet/diffusion_pytorch_model.fp16.safetensors',
								]
								A					       = 'fp16'
								self.assertFalse(is_safetensors_compatible(_lowercase			,			variant=_lowercase		)		)





