






"""simple docstring"""







import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


lowerCAmelCase_	:    Any            =      logging.get_logger(__name__)
lowerCAmelCase_	:    List[Any]            =      {'''vocab_file''': '''spiece.model'''}

lowerCAmelCase_	:    List[str]            =      {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

lowerCAmelCase_	:    List[str]            =      {
    '''AI-Sweden/gpt-sw3-126m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-350m''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_0_4_8,
    '''AI-Sweden/gpt-sw3-20b''': 2_0_4_8,
}





class   UpperCamelCase_ (       a_  ):
  _A       :					Optional[int]				=	VOCAB_FILES_NAMES
  _A       :					List[Any]				=	PRETRAINED_VOCAB_FILES_MAP
  _A       :					Dict				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _A       :					Optional[int]				=	['input_ids', 'attention_mask']



  def __init__(      self ,		snake_case__ ,		snake_case__=False ,		snake_case__=False ,		snake_case__=False ,		snake_case__=None ,		snake_case__=None ,		snake_case__=None ,		snake_case__=None ,		snake_case__ = None ,		**snake_case__ ,		)			->			None:



         """simple docstring"""





         UpperCAmelCase	    =			{} if sp_model_kwargs is None else sp_model_kwargs

         UpperCAmelCase	    =			kwargs.get("""name_or_path"""				)
         if name_or_path is None:
                logger.warning(
                    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                    """ you are testing the model, this can safely be ignored"""				)
                UpperCAmelCase	    =			"""None"""

         # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
         UpperCAmelCase	    =			"""<|endoftext|>""" if eos_token is None else eos_token
         UpperCAmelCase	    =			"""<unk>""" if unk_token is None else unk_token
         if "gpt-sw3-7b" in name_or_path:
                UpperCAmelCase	    =			unk_token if pad_token is None else pad_token
                UpperCAmelCase	    =			eos_token if bos_token is None else bos_token
         else:
                UpperCAmelCase	    =			"""<pad>""" if pad_token is None else pad_token
                UpperCAmelCase	    =			"""<s>""" if bos_token is None else bos_token

         super().__init__(
             do_lower_case=snake_case__ ,		remove_space=snake_case__ ,		keep_accents=snake_case__ ,		bos_token=snake_case__ ,		eos_token=snake_case__ ,		unk_token=snake_case__ ,		pad_token=snake_case__ ,		sp_model_kwargs=self.sp_model_kwargs ,		**snake_case__ ,		)

         UpperCAmelCase	    =			do_lower_case
         UpperCAmelCase	    =			remove_space
         UpperCAmelCase	    =			keep_accents
         UpperCAmelCase	    =			vocab_file

         UpperCAmelCase	    =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.Load(snake_case__				)

         # Used for whitespace normalization in input texts
         # fmt : off
         UpperCAmelCase	    =			{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
         # fmt : on

         # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
         UpperCAmelCase	    =			re.compile(
             f'''[{''.join(map(snake_case__ ,		list(range(0 ,		9				)				) + list(range(11 ,		32				)				) + list(range(1_27 ,		1_60				)				) + [1_60, 1_73, 82_03]				)				)}]'''				)



  def __getstate__(      self				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			self.__dict__.copy()
         UpperCAmelCase	    =			None
         return state



  def __setstate__(      self ,		snake_case__				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			d

         # for backward compatibility
         if not hasattr(self ,		"""sp_model_kwargs"""				):
                UpperCAmelCase	    =			{}

         UpperCAmelCase	    =			spm.SentencePieceProcessor(**self.sp_model_kwargs				)
         self.sp_model.Load(self.vocab_file				)



  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         return len(self.sp_model				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			str:



         """simple docstring"""





         UpperCAmelCase	    =			self.non_printing_characters_re.sub("""""" ,		snake_case__				)

         # Normalize whitespaces
         UpperCAmelCase	    =			"""""".join([char if char not in self.whitespaces else """ """ for char in text]				)

         # NFC Unicode normalization
         UpperCAmelCase	    =			unicodedata.normalize("""NFC""" ,		snake_case__				)
         return text



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		**snake_case__				)			->			List[str]:



         """simple docstring"""





         UpperCAmelCase	    =			self.preprocess_text(snake_case__				)
         return self.sp_model.encode(snake_case__ ,		out_type=snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			int:



         """simple docstring"""





         return self.sp_model.PieceToId(snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			str:



         """simple docstring"""





         return self.sp_model.IdToPiece(snake_case__				)



  @staticmethod
  def 		UpperCamelCase_				(      snake_case__				)			->			str:



         """simple docstring"""





         return out_string



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			str:



         """simple docstring"""





         UpperCAmelCase	    =			[]
         UpperCAmelCase	    =			""""""
         UpperCAmelCase	    =			False
         for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                       # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                       if not prev_is_special:
                              out_string += " "

                       out_string += self.sp_model.decode(snake_case__				) + token
                       UpperCAmelCase	    =			True
                       UpperCAmelCase	    =			[]
                else:
                       current_sub_tokens.append(snake_case__				)
                       UpperCAmelCase	    =			False
         out_string += self.sp_model.decode(snake_case__				)

         return out_string



  def 		UpperCamelCase_				(      self				)			->			Dict[str, int]:



         """simple docstring"""





         UpperCAmelCase	    =			{self.convert_ids_to_tokens(snake_case__				): i for i in range(self.vocab_size				)}
         vocab.update(self.added_tokens_encoder				)
         return vocab



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			Tuple[str]:



         """simple docstring"""





         if not os.path.isdir(snake_case__				):
                logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
                return
         UpperCAmelCase	    =			os.path.join(
             snake_case__ ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

         if os.path.abspath(self.vocab_file				) != os.path.abspath(snake_case__				) and os.path.isfile(self.vocab_file				):
                copyfile(self.vocab_file ,		snake_case__				)
         elif not os.path.isfile(self.vocab_file				):
                with open(snake_case__ ,		"""wb"""				) as fi:
                       UpperCAmelCase	    =			self.sp_model.serialized_model_proto()
                       fi.write(snake_case__				)

         return (out_vocab_file,)



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = False				)			->			Union[List[int], List[List[int]], "torch.Tensor"]:



         """simple docstring"""





         if isinstance(snake_case__ ,		snake_case__				):
                UpperCAmelCase	    =			self.preprocess_text(snake_case__				)
                UpperCAmelCase	    =			self.sp_model.encode(snake_case__				)
         else:
                UpperCAmelCase	    =			[self.preprocess_text(snake_case__				) for t in text]
                UpperCAmelCase	    =			self.sp_model.encode(snake_case__				)

         if return_tensors is True or return_tensors == "pt":
                UpperCAmelCase	    =			torch.tensor(snake_case__				)

         return token_ids



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			str:



         """simple docstring"""





         return self.sp_model.decode(snake_case__				)



  def 		UpperCamelCase_				(      self ,		snake_case__				)			->			List[int]:



         """simple docstring"""





         UpperCAmelCase	    =			[f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
         UpperCAmelCase	    =			(
             f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(snake_case__				) + f'''{self.bos_token}Bot:'''
         )
         return self.encode(text=snake_case__				)








"""simple docstring"""







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_	:    Tuple            =      logging.get_logger(__name__)

lowerCAmelCase_	:    List[str]            =      {
    '''google/bigbird-roberta-base''': '''https://huggingface.co/google/bigbird-roberta-base/resolve/main/config.json''',
    '''google/bigbird-roberta-large''': '''https://huggingface.co/google/bigbird-roberta-large/resolve/main/config.json''',
    '''google/bigbird-base-trivia-itc''': '''https://huggingface.co/google/bigbird-base-trivia-itc/resolve/main/config.json''',
    # See all BigBird models at https://huggingface.co/models?filter=big_bird
}





class   UpperCamelCase_ (       a_  ):
  _A       :					Any				=	'big_bird'



  def __init__(      self ,		snake_case__=5_03_58 ,		snake_case__=7_68 ,		snake_case__=12 ,		snake_case__=12 ,		snake_case__=30_72 ,		snake_case__="gelu_new" ,		snake_case__=0.1 ,		snake_case__=0.1 ,		snake_case__=40_96 ,		snake_case__=2 ,		snake_case__=0.02 ,		snake_case__=1e-12 ,		snake_case__=True ,		snake_case__=0 ,		snake_case__=1 ,		snake_case__=2 ,		snake_case__=66 ,		snake_case__="block_sparse" ,		snake_case__=True ,		snake_case__=False ,		snake_case__=64 ,		snake_case__=3 ,		snake_case__=None ,		**snake_case__ ,		)			->			List[str]:



         """simple docstring"""





         super().__init__(
             pad_token_id=snake_case__ ,		bos_token_id=snake_case__ ,		eos_token_id=snake_case__ ,		sep_token_id=snake_case__ ,		**snake_case__ ,		)

         UpperCAmelCase	    =			vocab_size
         UpperCAmelCase	    =			max_position_embeddings
         UpperCAmelCase	    =			hidden_size
         UpperCAmelCase	    =			num_hidden_layers
         UpperCAmelCase	    =			num_attention_heads
         UpperCAmelCase	    =			intermediate_size
         UpperCAmelCase	    =			hidden_act
         UpperCAmelCase	    =			hidden_dropout_prob
         UpperCAmelCase	    =			attention_probs_dropout_prob
         UpperCAmelCase	    =			initializer_range
         UpperCAmelCase	    =			type_vocab_size
         UpperCAmelCase	    =			layer_norm_eps
         UpperCAmelCase	    =			use_cache

         UpperCAmelCase	    =			rescale_embeddings
         UpperCAmelCase	    =			attention_type
         UpperCAmelCase	    =			use_bias
         UpperCAmelCase	    =			block_size
         UpperCAmelCase	    =			num_random_blocks
         UpperCAmelCase	    =			classifier_dropout





class   UpperCamelCase_ (       a_  ):



  @property
  def 		UpperCamelCase_				(      self				)			->			Mapping[str, Mapping[int, str]]:



         """simple docstring"""





         if self.task == "multiple-choice":
                UpperCAmelCase	    =			{0: """batch""", 1: """choice""", 2: """sequence"""}
         else:
                UpperCAmelCase	    =			{0: """batch""", 1: """sequence"""}
         return OrderedDict(
             [
                 ("""input_ids""", dynamic_axis),
                 ("""attention_mask""", dynamic_axis),
             ]				)
