




"""simple docstring"""

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


a         =      logging.get_logger(__name__)
a         =      {'''vocab_file''': '''spiece.model'''}

a         =      {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

a         =      {
    '''AI-Sweden/gpt-sw3-126m''': 2_048,
    '''AI-Sweden/gpt-sw3-350m''': 2_048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_048,
    '''AI-Sweden/gpt-sw3-20b''': 2_048,
}





class     lowercase_	(					A__							):



 '''simple docstring'''
 UpperCAmelCase   :					Tuple				  =     VOCAB_FILES_NAMES
 UpperCAmelCase   :					Dict				  =     PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase   :					Optional[Any]				  =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase   :					Union[str, Any]				  =     ['''input_ids''', '''attention_mask''']

 def __init__(			self	:	Tuple   ,							_UpperCAmelCase	:	Any   ,							_UpperCAmelCase	:	Optional[Any]=False   ,							_UpperCAmelCase	:	Tuple=False   ,							_UpperCAmelCase	:	Union[str, Any]=False   ,							_UpperCAmelCase	:	List[Any]=None   ,							_UpperCAmelCase	:	List[str]=None   ,							_UpperCAmelCase	:	Union[str, Any]=None   ,							_UpperCAmelCase	:	Tuple=None   ,							_UpperCAmelCase	:	Optional[Dict[str, Any]] = None   ,							**_UpperCAmelCase	:	Union[str, Any]   ,							):
       _A        =       {} if sp_model_kwargs is None else sp_model_kwargs

       _A        =       kwargs.get('name_or_path'		)
       if name_or_path is None:
             logger.warning(
                 'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                 ' you are testing the model, this can safely be ignored'		)
             _A        =       'None'

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       _A        =       '<|endoftext|>' if eos_token is None else eos_token
       _A        =       '<unk>' if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             _A        =       unk_token if pad_token is None else pad_token
             _A        =       eos_token if bos_token is None else bos_token
       else:
             _A        =       '<pad>' if pad_token is None else pad_token
             _A        =       '<s>' if bos_token is None else bos_token

       super().__init__(
           do_lower_case=__snake_case   ,							remove_space=__snake_case   ,							keep_accents=__snake_case   ,							bos_token=__snake_case   ,							eos_token=__snake_case   ,							unk_token=__snake_case   ,							pad_token=__snake_case   ,							sp_model_kwargs=self.sp_model_kwargs   ,							**__snake_case   ,							)

       _A        =       do_lower_case
       _A        =       remove_space
       _A        =       keep_accents
       _A        =       vocab_file

       _A        =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(__snake_case		)

       # Used for whitespace normalization in input texts
       # fmt : off
       _A        =       {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       _A        =       re.compile(
           F'''[{"".join(map(__snake_case   ,							list(range(0   ,							9		)		) + list(range(11   ,							32		)		) + list(range(127   ,							160		)		) + [160, 173, 8_203]		)		)}]'''		)

 def __getstate__(			self	:	Union[str, Any]		):
       _A        =       self.__dict__.copy()
       _A        =       None
       return state

 def __setstate__(			self	:	List[Any]   ,							_UpperCAmelCase	:	int		):
       _A        =       d

       # for backward compatibility
       if not hasattr(self   ,							'sp_model_kwargs'		):
             _A        =       {}

       _A        =       spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(self.vocab_file		)

 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def       lowerCAmelCase_							(			self	:	str		):
       return len(self.sp_model		)

 def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	str		):
       _A        =       self.non_printing_characters_re.sub(''   ,							__snake_case		)

       # Normalize whitespaces
       _A        =       ''.join([char if char not in self.whitespaces else ' ' for char in text]		)

       # NFC Unicode normalization
       _A        =       unicodedata.normalize('NFC'   ,							__snake_case		)
       return text

 def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	str   ,							**_UpperCAmelCase	:	Dict		):
       _A        =       self.preprocess_text(__snake_case		)
       return self.sp_model.encode(__snake_case   ,							out_type=__snake_case		)

 def       lowerCAmelCase_							(			self	:	int   ,							_UpperCAmelCase	:	str		):
       return self.sp_model.PieceToId(__snake_case		)

 def       lowerCAmelCase_							(			self	:	Optional[Any]   ,							_UpperCAmelCase	:	int		):
       return self.sp_model.IdToPiece(__snake_case		)

 @staticmethod
 def       lowerCAmelCase_							(			_UpperCAmelCase	:	str		):
       return out_string

 def       lowerCAmelCase_							(			self	:	Union[str, Any]   ,							_UpperCAmelCase	:	List[str]		):
       _A        =       []
       _A        =       ''
       _A        =       False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(__snake_case		) + token
                   _A        =       True
                   _A        =       []
             else:
                   current_sub_tokens.append(__snake_case		)
                   _A        =       False
       out_string += self.sp_model.decode(__snake_case		)

       return out_string

 def       lowerCAmelCase_							(			self	:	Dict		):
       _A        =       {self.convert_ids_to_tokens(__snake_case		): i for i in range(self.vocab_size		)}
       vocab.update(self.added_tokens_encoder		)
       return vocab

 def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	Optional[str] = None		):
       if not os.path.isdir(__snake_case		):
             logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''		)
             return
       _A        =       os.path.join(
           __snake_case   ,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)

       if os.path.abspath(self.vocab_file		) != os.path.abspath(__snake_case		) and os.path.isfile(self.vocab_file		):
             copyfile(self.vocab_file   ,							__snake_case		)
       elif not os.path.isfile(self.vocab_file		):
             with open(__snake_case   ,							'wb'		) as fi:
                   _A        =       self.sp_model.serialized_model_proto()
                   fi.write(__snake_case		)

       return (out_vocab_file,)

 def       lowerCAmelCase_							(			self	:	Any   ,							_UpperCAmelCase	:	Union[str, List[str]]   ,							_UpperCAmelCase	:	Union[str, bool] = False		):

       if isinstance(__snake_case   ,							__snake_case		):
             _A        =       self.preprocess_text(__snake_case		)
             _A        =       self.sp_model.encode(__snake_case		)
       else:
             _A        =       [self.preprocess_text(__snake_case		) for t in text]
             _A        =       self.sp_model.encode(__snake_case		)

       if return_tensors is True or return_tensors == "pt":
             _A        =       torch.tensor(__snake_case		)

       return token_ids

 def       lowerCAmelCase_							(			self	:	Union[str, Any]   ,							_UpperCAmelCase	:	Union[int, List[int]]		):
       return self.sp_model.decode(__snake_case		)



 def       lowerCAmelCase_							(			self	:	Tuple   ,							_UpperCAmelCase	:	"Conversation"		):
       _A        =       [F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
       _A        =       (
           F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(__snake_case		) + F'''{self.bos_token}Bot:'''
       )
       return self.encode(text=__snake_case		)







"""simple docstring"""

import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
       import tensorflow as tf
       from tensorflow.python.framework.errors_impl import ResourceExhaustedError

       from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
       import pyanvml.pyanvml as nvml

a         =      logging.get_logger(__name__)






def 						_snake_case					( _snake_case							: bool    ,  _snake_case							: bool							)   -> Tuple:






      '''simple docstring'''


      def run_func(_snake_case							: Any							):
            @wraps(_snake_case							)
            def run_in_eager_mode(*_snake_case							: List[str]    ,  **_snake_case							: Tuple							):
                  return func(*_snake_case    ,  **_snake_case							)

            @wraps(_snake_case							)
            @tf.function(experimental_compile=_snake_case							)
            def run_in_graph_mode(*_snake_case							: Dict    ,  **_snake_case							: Tuple							):
                  return func(*_snake_case    ,  **_snake_case							)

            if do_eager_mode is True:
                  if use_xla is not False:
                        raise ValueError(
                            'Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.'							)
                  return run_in_eager_mode
            else:
                  return run_in_graph_mode

      return run_func






def 						_snake_case					( _snake_case							: int    ,  _snake_case							: int    ,  _snake_case							: int							)   -> ["tf.Tensor"]:






      '''simple docstring'''


      _A        =       random.Random()
      _A        =       [rng.randint(0    ,  vocab_size - 1							) for i in range(batch_size * sequence_length							)]
      return tf.constant(_snake_case    ,  shape=(batch_size, sequence_length)    ,  dtype=tf.intaa							)







class     lowercase_	(					__lowerCAmelCase							):



 '''simple docstring'''
 UpperCAmelCase   :					TensorFlowBenchmarkArguments
 UpperCAmelCase   :					PretrainedConfig
 UpperCAmelCase   :					str				  =     "TensorFlow"

 @property
 def       lowerCAmelCase_							(			self	:	str		):
       return tf.__version__

 def       lowerCAmelCase_							(			self	:	List[str]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       # initialize GPU on separate process
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_inference_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_speed(_inference		)

 def       lowerCAmelCase_							(			self	:	Optional[Any]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_train_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_speed(_train		)

 def       lowerCAmelCase_							(			self	:	Dict   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       # initialize GPU on separate process
       if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]   ,							_UpperCAmelCase		)
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)
       _A        =       self._prepare_inference_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_memory(_inference		)

 def       lowerCAmelCase_							(			self	:	int   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       if self.args.is_gpu:
             tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]   ,							_UpperCAmelCase		)
       _A        =       self.args.strategy
       if strategy is None:
             raise ValueError('A device strategy has to be initialized before using TensorFlow.'		)

       _A        =       self._prepare_train_func(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)
       return self._measure_memory(_train		)

 def       lowerCAmelCase_							(			self	:	List[Any]   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.config_dict[model_name]

       if self.args.fpaa:
             raise NotImplementedError('Mixed precision is currently not supported.'		)

       _A        =       (
           hasattr(_UpperCAmelCase   ,							'architectures'		)
           and isinstance(config.architectures   ,							_UpperCAmelCase		)
           and len(config.architectures		) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   _A        =       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _A        =       __import__('transformers'   ,							fromlist=[model_class]		)
                   _A        =       getattr(_UpperCAmelCase   ,							_UpperCAmelCase		)
                   _A        =       model_cls(_UpperCAmelCase		)
             except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
       else:
             _A        =       TF_MODEL_MAPPING[config.__class__](_UpperCAmelCase		)

       # encoder-decoder has vocab size saved differently
       _A        =       config.vocab_size if hasattr(_UpperCAmelCase   ,							'vocab_size'		) else config.encoder.vocab_size
       _A        =       random_input_ids(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_decoder_forward():
             return model(_UpperCAmelCase   ,							decoder_input_ids=_UpperCAmelCase   ,							training=_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_forward():
             return model(_UpperCAmelCase   ,							training=_UpperCAmelCase		)

       _A        =       encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

       return _inference

 def       lowerCAmelCase_							(			self	:	str   ,							_UpperCAmelCase	:	str   ,							_UpperCAmelCase	:	int   ,							_UpperCAmelCase	:	int		):
       _A        =       self.config_dict[model_name]

       if self.args.eager_mode is not False:
             raise ValueError('Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`.'		)

       if self.args.fpaa:
             raise NotImplementedError('Mixed precision is currently not supported.'		)

       _A        =       (
           hasattr(_UpperCAmelCase   ,							'architectures'		)
           and isinstance(config.architectures   ,							_UpperCAmelCase		)
           and len(config.architectures		) > 0
       )
       if not self.args.only_pretrain_model and has_model_class_in_config:
             try:
                   _A        =       'TF' + config.architectures[0]  # prepend 'TF' for tensorflow model
                   _A        =       __import__('transformers'   ,							fromlist=[model_class]		)
                   _A        =       getattr(_UpperCAmelCase   ,							_UpperCAmelCase		)
                   _A        =       model_cls(_UpperCAmelCase		)
             except ImportError:
                   raise ImportError(
                       F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
                       ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'		)
       else:
             _A        =       TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](_UpperCAmelCase		)

       # encoder-decoder has vocab size saved differently
       _A        =       config.vocab_size if hasattr(_UpperCAmelCase   ,							'vocab_size'		) else config.encoder.vocab_size
       _A        =       random_input_ids(_UpperCAmelCase   ,							_UpperCAmelCase   ,							_UpperCAmelCase		)

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_decoder_train():
             _A        =       model(_UpperCAmelCase   ,							decoder_input_ids=_UpperCAmelCase   ,							labels=_UpperCAmelCase   ,							training=_UpperCAmelCase		)[0]
             _A        =       tf.gradients(_UpperCAmelCase   ,							model.trainable_variables		)
             return gradients

       @run_with_tf_optimizations(self.args.eager_mode   ,							self.args.use_xla		)
       def encoder_train():
             _A        =       model(_UpperCAmelCase   ,							labels=_UpperCAmelCase   ,							training=_UpperCAmelCase		)[0]
             _A        =       tf.gradients(_UpperCAmelCase   ,							model.trainable_variables		)
             return gradients

       _A        =       encoder_decoder_train if config.is_encoder_decoder else encoder_train

       return _train

 def       lowerCAmelCase_							(			self	:	Tuple   ,							_UpperCAmelCase	:	int		):
       with self.args.strategy.scope():
             try:
                   if self.args.is_tpu or self.args.use_xla:
                         # run additional 10 times to stabilize compilation for tpu
                         logger.info('Do inference on TPU. Running model 5 times to stabilize compilation'		)
                         timeit.repeat(_UpperCAmelCase   ,							repeat=1   ,							number=5		)

                   # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
                   _A        =       timeit.repeat(
                       _UpperCAmelCase   ,							repeat=self.args.repeat   ,							number=10   ,							)

                   return min(_UpperCAmelCase		) / 10.0
             except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)



 def       lowerCAmelCase_							(			self	:	Optional[int]   ,							_UpperCAmelCase	:	Callable[[], None]		):
       logger.info(
           'Note that TensorFlow allocates more memory than '
           'it might need to speed up computation. '
           'The memory reported here corresponds to the memory '
           'reported by `nvidia-smi`, which can vary depending '
           'on total available memory on the GPU that is used.'		)
       with self.args.strategy.scope():
             try:
                   if self.args.trace_memory_line_by_line:
                         if not self.args.eager_mode:
                               raise ValueError(
                                   '`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory'
                                   ' consumption line by line.'		)
                         _A        =       start_memory_tracing('transformers'		)

                   if self.args.is_tpu:
                         # tpu
                         raise NotImplementedError(
                             'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking'
                             ' with `args.memory=False`'		)
                   elif self.args.is_gpu:
                         # gpu
                         if not is_pyanvml_available():
                               logger.warning(
                                   'py3nvml not installed, we won\'t log GPU memory usage. '
                                   'Install py3nvml (pip install py3nvml) to log information about GPU.'		)
                               _A        =       'N/A'
                         else:
                               logger.info(
                                   'Measuring total GPU usage on GPU device. Make sure to not have additional processes'
                                   ' running on the same GPU.'		)
                               # init nvml
                               nvml.nvmlInit()
                               func()
                               _A        =       nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx		)
                               _A        =       nvml.nvmlDeviceGetMemoryInfo(_UpperCAmelCase		)
                               _A        =       meminfo.used
                               _A        =       Memory(_UpperCAmelCase		)
                               # shutdown nvml
                               nvml.nvmlShutdown()
                   else:
                         # cpu
                         if self.args.trace_memory_line_by_line:
                               logger.info(
                                   'When enabling line by line tracing, the max peak memory for CPU is inaccurate in'
                                   ' TensorFlow.'		)
                               _A        =       None
                         else:
                               _A        =       measure_peak_memory_cpu(_UpperCAmelCase		)
                               _A        =       Memory(_UpperCAmelCase		) if isinstance(_UpperCAmelCase   ,							_UpperCAmelCase		) else memory_bytes
                   if self.args.trace_memory_line_by_line:
                         _A        =       stop_memory_tracing(_UpperCAmelCase		)
                         if memory is None:
                               _A        =       summary.total
                   else:
                         _A        =       None

                   return memory, summary
             except ResourceExhaustedError as e:
                   self.print_fn(F'''Doesn\'t fit on GPU. {e}'''		)
                   return "N/A", None

