


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)
SCREAMING_SNAKE_CASE							    =							{'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE							    =							{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE							    =							{
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      VOCAB_FILES_NAMES
     _lowerCamelCase   =      PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase   =      ["""input_ids""", """attention_mask"""]


     def __init__(	self  ,       __A  ,       __A=False  ,       __A=False  ,       __A=False  ,       __A=None  ,       __A=None  ,       __A=None  ,       __A=None  ,       __A = None  ,       **__A  ,       ):
        __a					=  {} if sp_model_kwargs is None else sp_model_kwargs

        __a					=  kwargs.get("""name_or_path"""    )
        if name_or_path is None:
           logger.warning(
               """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
               """ you are testing the model, this can safely be ignored"""    )
           __a					=  """None"""

        # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
        __a					=  """<|endoftext|>""" if eos_token is None else eos_token
        __a					=  """<unk>""" if unk_token is None else unk_token
        if "gpt-sw3-7b" in name_or_path:
           __a					=  unk_token if pad_token is None else pad_token
           __a					=  eos_token if bos_token is None else bos_token
        else:
           __a					=  """<pad>""" if pad_token is None else pad_token
           __a					=  """<s>""" if bos_token is None else bos_token

        super().__init__(
            do_lower_case=__A  ,       remove_space=__A  ,       keep_accents=__A  ,       bos_token=__A  ,       eos_token=__A  ,       unk_token=__A  ,       pad_token=__A  ,       sp_model_kwargs=self.sp_model_kwargs  ,       **__A  ,       )

        __a					=  do_lower_case
        __a					=  remove_space
        __a					=  keep_accents
        __a					=  vocab_file

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(__A    )

        # Used for whitespace normalization in input texts
        # fmt : off
        __a					=  {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
        # fmt : on

        # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
        __a					=  re.compile(
            f'''[{''.join(map(__A  ,       list(range(0  ,       9    )    ) + list(range(11  ,       32    )    ) + list(range(127  ,       160    )    ) + [160, 173, 8203]    )    )}]'''    )


     def __getstate__(	self    ):
        __a					=  self.__dict__.copy()
        __a					=  None
        return state


     def __setstate__(	self  ,       __A    ):
        __a					=  d

        # for backward compatibility
        if not hasattr(self  ,       """sp_model_kwargs"""    ):
           __a					=  {}

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(self.vocab_file    )


     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def 	snake_case_  (	self    ):
        return len(self.sp_model    )


     def 	snake_case_  (	self  ,       __A    ):
        __a					=  self.non_printing_characters_re.sub(""""""  ,       __A    )

        # Normalize whitespaces
        __a					=  """""".join([char if char not in self.whitespaces else """ """ for char in text]    )

        # NFC Unicode normalization
        __a					=  unicodedata.normalize("""NFC"""  ,       __A    )
        return text


     def 	snake_case_  (	self  ,       __A  ,       **__A    ):
        __a					=  self.preprocess_text(__A    )
        return self.sp_model.encode(__A  ,       out_type=__A    )


     def 	snake_case_  (	self  ,       __A    ):
        return self.sp_model.PieceToId(__A    )


     def 	snake_case_  (	self  ,       __A    ):
        return self.sp_model.IdToPiece(__A    )


     @staticmethod
     def 	snake_case_  (	__A    ):
        return out_string


     def 	snake_case_  (	self  ,       __A    ):
        __a					=  []
        __a					=  """"""
        __a					=  False
        for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
              # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
              if not prev_is_special:
                 out_string += " "

              out_string += self.sp_model.decode(__A    ) + token
              __a					=  True
              __a					=  []
           else:
              current_sub_tokens.append(__A    )
              __a					=  False
        out_string += self.sp_model.decode(__A    )

        return out_string


     def 	snake_case_  (	self    ):
        __a					=  {self.convert_ids_to_tokens(__A    ): i for i in range(self.vocab_size    )}
        vocab.update(self.added_tokens_encoder    )
        return vocab


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        if not os.path.isdir(__A    ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
           return
        __a					=  os.path.join(
            __A  ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

        if os.path.abspath(self.vocab_file    ) != os.path.abspath(__A    ) and os.path.isfile(self.vocab_file    ):
           copyfile(self.vocab_file  ,       __A    )
        elif not os.path.isfile(self.vocab_file    ):
           with open(__A  ,       """wb"""    ) as fi:
              __a					=  self.sp_model.serialized_model_proto()
              fi.write(__A    )

        return (out_vocab_file,)


     def 	snake_case_  (	self  ,       __A  ,       __A = False    ):

        if isinstance(__A  ,       __A    ):
           __a					=  self.preprocess_text(__A    )
           __a					=  self.sp_model.encode(__A    )
        else:
           __a					=  [self.preprocess_text(__A    ) for t in text]
           __a					=  self.sp_model.encode(__A    )

        if return_tensors is True or return_tensors == "pt":
           __a					=  torch.tensor(__A    )

        return token_ids


     def 	snake_case_  (	self  ,       __A    ):
        return self.sp_model.decode(__A    )


     def 	snake_case_  (	self  ,       __A    ):
        __a					=  [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
        __a					=  (
            f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(__A    ) + f'''{self.bos_token}Bot:'''
        )
        return self.encode(text=__A    )









import importlib.metadata
import operator
import re
import sys
from typing import Optional

from packaging import version


__A								=       {
    "<": operator.lt,
    "<=": operator.le,
    "==": operator.eq,
    "!=": operator.ne,
    ">=": operator.ge,
    ">": operator.gt,
}







def      lowerCamelCase_						(			UpperCamelCase__							: List[str]					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: Any					,	UpperCamelCase__							: Dict					,	UpperCamelCase__							: int )	->			List[Any]:





				"""simple docstring"""


				if got_ver is None or want_ver is None:
								raise ValueError(
								    F"""Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider"""
								    F""" reinstalling {pkg}.""" )
				if not ops[op](version.parse(UpperCamelCase__ )					,	version.parse(UpperCamelCase__ ) ):
								raise ImportError(
								    F"""{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}""" )







def      lowerCamelCase_						(			UpperCamelCase__							: str					,	UpperCamelCase__							: Optional[str] = None )	->			None:





				"""simple docstring"""


				__lowerCamelCase									=       F"""\n{hint}""" if hint is not None else ''

				# non-versioned check
				if re.match(R'^[\w_\-\d]+$'					,	UpperCamelCase__ ):
								__lowerCamelCase			,		__lowerCamelCase			,		__lowerCamelCase									=       requirement, None, None
				else:
								__lowerCamelCase									=       re.findall(R'^([^!=<>\s]+)([\s!=<>]{1,2}.+)'					,	UpperCamelCase__ )
								if not match:
												raise ValueError(
												    'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but'
												    F""" got {requirement}""" )
								__lowerCamelCase			,		__lowerCamelCase									=       match[0]
								__lowerCamelCase									=       want_full.split(',' )  # there could be multiple requirements
								__lowerCamelCase									=       {}
								for w in want_range:
												__lowerCamelCase									=       re.findall(R'^([\s!=<>]{1,2})(.+)'					,	UpperCamelCase__ )
												if not match:
																raise ValueError(
																    'requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23,'
																    F""" but got {requirement}""" )
												__lowerCamelCase			,		__lowerCamelCase									=       match[0]
												__lowerCamelCase									=       want_ver
												if op not in ops:
																raise ValueError(F"""{requirement}: need one of {list(ops.keys() )}, but got {op}""" )

    # special case
				if pkg == "python":
								__lowerCamelCase									=       '.'.join([str(UpperCamelCase__ ) for x in sys.version_info[:3]] )
								for op, want_ver in wanted.items():
												_compare_versions(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )
								return

				# check if any version is installed
				try:
								__lowerCamelCase									=       importlib.metadata.version(UpperCamelCase__ )
				except importlib.metadata.PackageNotFoundError:
								raise importlib.metadata.PackageNotFoundError(
								    F"""The '{requirement}' distribution was not found and is required by this application. {hint}""" )

				# check that the right version is installed if version number or a range was provided
				if want_ver is not None:
								for op, want_ver in wanted.items():
												_compare_versions(UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__					,	UpperCamelCase__ )







def      lowerCamelCase_						(			UpperCamelCase__							: Optional[int] )	->			Optional[int]:





				"""simple docstring"""


				__lowerCamelCase									=       'Try: pip install transformers -U or pip install -e \'.[dev]\' if you\'re working with git main'
				return require_version(UpperCamelCase__					,	UpperCamelCase__ )


