


import os
import re
import warnings
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer


if TYPE_CHECKING:
    from ...tokenization_utils_base import TextInput
from ...utils import logging


A      = logging.get_logger(__name__)

A      = {"vocab_file": "spiece.model"}

A      = {
    "vocab_file": {
        "t5-small": "https://huggingface.co/t5-small/resolve/main/spiece.model",
        "t5-base": "https://huggingface.co/t5-base/resolve/main/spiece.model",
        "t5-large": "https://huggingface.co/t5-large/resolve/main/spiece.model",
        "t5-3b": "https://huggingface.co/t5-3b/resolve/main/spiece.model",
        "t5-11b": "https://huggingface.co/t5-11b/resolve/main/spiece.model",
    }
}


# TODO(PVP) - this should be removed in Transformers v5
A      = {
    "t5-small": 512,
    "t5-base": 512,
    "t5-large": 512,
    "t5-3b": 512,
    "t5-11b": 512,
}

A      = "▁"


class     lowercase__  (					__SCREAMING_SNAKE_CASE		):
  A__=  VOCAB_FILES_NAMES
  A__=  PRETRAINED_VOCAB_FILES_MAP
  A__=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A__=  ['input_ids', 'attention_mask']






  def __init__(					self      :					Tuple		,						_lowercase      :					int		,						_lowercase      :					str="</s>"		,						_lowercase      :					Optional[Any]="<unk>"		,						_lowercase      :					Dict="<pad>"		,						_lowercase      :					Tuple=1_00		,						_lowercase      :					str=None		,						_lowercase      :					Optional[Dict[str, Any]] = None		,						_lowercase      :					str=True		,						**_lowercase      :					Optional[Any]		,						):




         """simple docstring"""







         if extra_ids > 0 and additional_special_tokens is None:
                UpperCAmelCase__										=     [F"""<extra_id_{i}>""" for i in range(lowercase_ )]
         elif extra_ids > 0 and additional_special_tokens is not None:
                # Check that we have the right number of extra_id special tokens
                UpperCAmelCase__										=     len(set(filter(lambda _lowercase : bool("extra_id" in str(lowercase_ ) )		,						lowercase_ ) ) )
                if extra_tokens != extra_ids:
                       raise ValueError(
                           F"""Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are"""
                           " provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"
                           " tokens" )
         if legacy:
                logger.warning_once(
                    F"""You are using the legacy behaviour of the {self.__class__}. This means that tokens that come after special tokens will not be properly handled. We recommend you to"""
                    " read the related pull request available at https://github.com/huggingface/transformers/pull/24565" )

         UpperCAmelCase__										=     legacy
         UpperCAmelCase__										=     {} if sp_model_kwargs is None else sp_model_kwargs

         super().__init__(
             eos_token=lowercase_		,						unk_token=lowercase_		,						pad_token=lowercase_		,						extra_ids=lowercase_		,						additional_special_tokens=lowercase_		,						sp_model_kwargs=self.sp_model_kwargs		,						legacy=lowercase_		,						**lowercase_		,						)

         UpperCAmelCase__										=     vocab_file
         UpperCAmelCase__										=     extra_ids

         UpperCAmelCase__										=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
         self.sp_model.Load(lowercase_ )






  @staticmethod
  def 			_UpperCAmelCase  (					_lowercase      :					Optional[Any]		,						_lowercase      :					Dict		,						_lowercase      :					str ):




         """simple docstring"""







         if pretrained_model_name_or_path in TaTokenizer.max_model_input_sizes:
                UpperCAmelCase__										=     TaTokenizer.max_model_input_sizes[pretrained_model_name_or_path]
                if init_max_model_length is not None and init_max_model_length != max_model_length:
                       return init_max_model_length
                elif init_max_model_length is None:
                       warnings.warn(
                           "This tokenizer was incorrectly instantiated with a model max length of"
                           F""" {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this"""
                           " behavior is kept to avoid breaking backwards compatibility when padding/encoding with"
                           " `truncation is True`.\n- Be aware that you SHOULD NOT rely on"
                           F""" {pretrained_model_name_or_path} automatically truncating your input to"""
                           F""" {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences"""
                           F""" longer than {deprecated_max_model_length} you can either instantiate this tokenizer with"""
                           " `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"
                           " instantiate this tokenizer with `model_max_length` set to your preferred value."		,						lowercase_		,						)

         return max_model_length






  @property
  def 			_UpperCAmelCase  (					self      :					Dict ):




         """simple docstring"""







         return self.sp_model.get_piece_size() + self._extra_ids






  def 			_UpperCAmelCase  (					self      :					Dict ):




         """simple docstring"""







         UpperCAmelCase__										=     {self.convert_ids_to_tokens(lowercase_ ): i for i in range(self.vocab_size )}
         vocab.update(self.added_tokens_encoder )
         return vocab






  def 			_UpperCAmelCase  (					self      :					Dict		,						_lowercase      :					List[int]		,						_lowercase      :					Optional[List[int]] = None		,						_lowercase      :					bool = False ):




         """simple docstring"""







         if already_has_special_tokens:
                return super().get_special_tokens_mask(
                    token_ids_a=lowercase_		,						token_ids_a=lowercase_		,						already_has_special_tokens=lowercase_ )

         # normal case: some special tokens
         if token_ids_a is None:
                return ([0] * len(lowercase_ )) + [1]
         return ([0] * len(lowercase_ )) + [1] + ([0] * len(lowercase_ )) + [1]






  def 			_UpperCAmelCase  (					self      :					str ):




         """simple docstring"""







         return list(
             set(filter(lambda _lowercase : bool(re.search(r"<extra_id_\d+>"		,						lowercase_ ) ) is not None		,						self.additional_special_tokens ) ) )






  def 			_UpperCAmelCase  (					self      :					List[Any] ):




         """simple docstring"""







         return [self._convert_token_to_id(lowercase_ ) for token in self.get_sentinel_tokens()]






  def 			_UpperCAmelCase  (					self      :					Optional[Any]		,						_lowercase      :					List[int] ):




         """simple docstring"""







         if len(lowercase_ ) > 0 and token_ids[-1] == self.eos_token_id:
                warnings.warn(
                    F"""This sequence already has {self.eos_token}. In future versions this behavior may lead to duplicated"""
                    " eos tokens being added." )
                return token_ids
         else:
                return token_ids + [self.eos_token_id]






  def 			_UpperCAmelCase  (					self      :					List[str]		,						_lowercase      :					List[int]		,						_lowercase      :					Optional[List[int]] = None ):




         """simple docstring"""







         UpperCAmelCase__										=     [self.eos_token_id]

         if token_ids_a is None:
                return len(token_ids_a + eos ) * [0]
         return len(token_ids_a + eos + token_ids_a + eos ) * [0]






  def 			_UpperCAmelCase  (					self      :					Optional[int]		,						_lowercase      :					List[int]		,						_lowercase      :					Optional[List[int]] = None ):




         """simple docstring"""







         UpperCAmelCase__										=     self._add_eos_if_not_present(lowercase_ )
         if token_ids_a is None:
                return token_ids_a
         else:
                UpperCAmelCase__										=     self._add_eos_if_not_present(lowercase_ )
                return token_ids_a + token_ids_a






  def __getstate__(					self      :					Tuple ):




         """simple docstring"""







         UpperCAmelCase__										=     self.__dict__.copy()
         UpperCAmelCase__										=     None
         return state






  def __setstate__(					self      :					Optional[Any]		,						_lowercase      :					Any ):




         """simple docstring"""







         UpperCAmelCase__										=     d

         # for backward compatibility
         if not hasattr(self		,						"sp_model_kwargs" ):
                UpperCAmelCase__										=     {}

         UpperCAmelCase__										=     spm.SentencePieceProcessor(**self.sp_model_kwargs )
         self.sp_model.Load(self.vocab_file )






  def 			_UpperCAmelCase  (					self      :					int		,						_lowercase      :					"TextInput"		,						**_lowercase      :					Optional[int] ):




         """simple docstring"""







         if not self.legacy:
                UpperCAmelCase__										=     SPIECE_UNDERLINE + text.replace(lowercase_		,						" " )
         return super().tokenize(lowercase_		,						**lowercase_ )






  def 			_UpperCAmelCase  (					self      :					Union[str, Any]		,						_lowercase      :					int		,						**_lowercase      :					Optional[int] ):




         """simple docstring"""







         if not self.legacy:
                UpperCAmelCase__										=     text.startswith(lowercase_ )
                if is_first:
                       UpperCAmelCase__										=     text[1:]

         UpperCAmelCase__										=     self.sp_model.encode(lowercase_		,						out_type=lowercase_ )

         if not self.legacy and not is_first and not text.startswith(" " ) and tokens[0].startswith(lowercase_ ):
                UpperCAmelCase__										=     ([tokens[0][1:]] if len(tokens[0] ) > 1 else []) + tokens[1:]
         return tokens






  def 			_UpperCAmelCase  (					self      :					Optional[Any]		,						_lowercase      :					Optional[Any] ):




         """simple docstring"""







         if token.startswith("<extra_id_" ):
                UpperCAmelCase__										=     re.match(r"<extra_id_(\d+)>"		,						lowercase_ )
                UpperCAmelCase__										=     int(match.group(1 ) )
                return self.vocab_size - num - 1
         return self.sp_model.piece_to_id(lowercase_ )






  def 			_UpperCAmelCase  (					self      :					List[Any]		,						_lowercase      :					Any ):




         """simple docstring"""







         if index < self.sp_model.get_piece_size():
                UpperCAmelCase__										=     self.sp_model.IdToPiece(lowercase_ )
         else:
                UpperCAmelCase__										=     F"""<extra_id_{self.vocab_size - 1 - index}>"""
         return token






  def 			_UpperCAmelCase  (					self      :					Dict		,						_lowercase      :					Optional[int] ):




         """simple docstring"""







         UpperCAmelCase__										=     []
         UpperCAmelCase__										=     ""
         UpperCAmelCase__										=     False
         for token in tokens:
                # make sure that special tokens are not decoded using sentencepiece model
                if token in self.all_special_tokens:
                       if not prev_is_special:
                              out_string += " "
                       out_string += self.sp_model.decode(lowercase_ ) + token
                       UpperCAmelCase__										=     True
                       UpperCAmelCase__										=     []
                else:
                       current_sub_tokens.append(lowercase_ )
                       UpperCAmelCase__										=     False
         out_string += self.sp_model.decode(lowercase_ )
         return out_string.strip()






  def 			_UpperCAmelCase  (					self      :					List[str]		,						_lowercase      :					str		,						_lowercase      :					Optional[str] = None ):




         """simple docstring"""







         if not os.path.isdir(lowercase_ ):
                logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
                return
         UpperCAmelCase__										=     os.path.join(
             lowercase_		,						(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"] )

         if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase_ ) and os.path.isfile(self.vocab_file ):
                copyfile(self.vocab_file		,						lowercase_ )
         elif not os.path.isfile(self.vocab_file ):
                with open(lowercase_		,						"wb" ) as fi:
                       UpperCAmelCase__										=     self.sp_model.serialized_model_proto()
                       fi.write(lowercase_ )

         return (out_vocab_file,)









from math import atan, cos, radians, sin, tan

from .haversine_distance import haversine_distance

A      = 6378137.0
A      = 6356752.314245
A      = 637_8137





def __UpperCAmelCase       (		__A ,						__A ,						__A ,						__A			)						->					float:



       '''simple docstring'''





       UpperCAmelCase__										=     (AXIS_A - AXIS_B) / AXIS_A
       # Parametric latitudes
       # https://en.wikipedia.org/wiki/Latitude#Parametric_(or_reduced)_latitude
       UpperCAmelCase__										=     atan((1 - flattening) * tan(radians(__A			)			)			)
       UpperCAmelCase__										=     atan((1 - flattening) * tan(radians(__A			)			)			)

       # Compute central angle between two points
       # using haversine theta. sigma =  haversine_distance / equatorial radius
       UpperCAmelCase__										=     haversine_distance(__A ,						__A ,						__A ,						__A			) / EQUATORIAL_RADIUS

       # Intermediate P and Q values
       UpperCAmelCase__										=     (b_lata + b_lata) / 2
       UpperCAmelCase__										=     (b_lata - b_lata) / 2

       # Intermediate X value
       # X = (sigma - sin(sigma)) * sin^2Pcos^2Q / cos^2(sigma/2)
       UpperCAmelCase__										=     (sin(__A			) ** 2) * (cos(__A			) ** 2)
       UpperCAmelCase__										=     cos(sigma / 2			) ** 2
       UpperCAmelCase__										=     (sigma - sin(__A			)) * (x_numerator / x_demonimator)

       # Intermediate Y value
       # Y = (sigma + sin(sigma)) * cos^2Psin^2Q / sin^2(sigma/2)
       UpperCAmelCase__										=     (cos(__A			) ** 2) * (sin(__A			) ** 2)
       UpperCAmelCase__										=     sin(sigma / 2			) ** 2
       UpperCAmelCase__										=     (sigma + sin(__A			)) * (y_numerator / y_denominator)

       return EQUATORIAL_RADIUS * (sigma - ((flattening / 2) * (x_value + y_value)))


if __name__ == "__main__":
    import doctest

    doctest.testmod()





