


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


A      = get_tests_dir("fixtures/test_sentencepiece.model")


@require_sentencepiece
@require_tokenizers
class     lowercase__  (					__SCREAMING_SNAKE_CASE					,							unittest.TestCase		):
  A__=  ReformerTokenizer
  A__=  ReformerTokenizerFast
  A__=  True
  A__=  False
  A__=  True






  def 			_UpperCAmelCase  (					self      :					Optional[int] ):




         """simple docstring"""







         super().setUp()

         UpperCAmelCase__										=     ReformerTokenizer(_lowercase		,						keep_accents=_lowercase )
         tokenizer.save_pretrained(self.tmpdirname )






  def 			_UpperCAmelCase  (					self      :					Dict ):




         """simple docstring"""







         UpperCAmelCase__										=     "<s>"
         UpperCAmelCase__										=     1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowercase )		,						_lowercase )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowercase )		,						_lowercase )






  def 			_UpperCAmelCase  (					self      :					Optional[int] ):




         """simple docstring"""







         UpperCAmelCase__										=     list(self.get_tokenizer().get_vocab().keys() )

         self.assertEqual(vocab_keys[0]		,						"<unk>" )
         self.assertEqual(vocab_keys[1]		,						"<s>" )
         self.assertEqual(vocab_keys[-1]		,						"j" )
         self.assertEqual(len(_lowercase )		,						10_00 )






  def 			_UpperCAmelCase  (					self      :					Dict ):




         """simple docstring"""







         self.assertEqual(self.get_tokenizer().vocab_size		,						10_00 )






  def 			_UpperCAmelCase  (					self      :					Tuple ):




         """simple docstring"""







         if not self.test_rust_tokenizer:
                return

         UpperCAmelCase__										=     self.get_tokenizer()
         UpperCAmelCase__										=     self.get_rust_tokenizer()

         UpperCAmelCase__										=     "I was born in 92000, and this is falsé."

         UpperCAmelCase__										=     tokenizer.tokenize(_lowercase )
         UpperCAmelCase__										=     rust_tokenizer.tokenize(_lowercase )
         self.assertListEqual(_lowercase		,						_lowercase )

         UpperCAmelCase__										=     tokenizer.encode(_lowercase		,						add_special_tokens=_lowercase )
         UpperCAmelCase__										=     rust_tokenizer.encode(_lowercase		,						add_special_tokens=_lowercase )
         self.assertListEqual(_lowercase		,						_lowercase )

         UpperCAmelCase__										=     self.get_rust_tokenizer()
         UpperCAmelCase__										=     tokenizer.encode(_lowercase )
         UpperCAmelCase__										=     rust_tokenizer.encode(_lowercase )
         self.assertListEqual(_lowercase		,						_lowercase )






  def 			_UpperCAmelCase  (					self      :					Dict		,						_lowercase      :					str=15 ):




         """simple docstring"""







         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})""" ):
                       UpperCAmelCase__										=     self.rust_tokenizer_class.from_pretrained(_lowercase		,						**_lowercase )

                       # Simple input
                       UpperCAmelCase__										=     "This is a simple input"
                       UpperCAmelCase__										=     ["This is a simple input 1", "This is a simple input 2"]
                       UpperCAmelCase__										=     ("This is a simple input", "This is a pair")
                       UpperCAmelCase__										=     [
                           ("This is a simple input 1", "This is a simple input 2"),
                           ("This is a simple pair 1", "This is a simple pair 2"),
                       ]

                       # Simple input tests
                       self.assertRaises(_lowercase		,						tokenizer_r.encode		,						_lowercase		,						max_length=_lowercase		,						padding="max_length" )

                       # Simple input
                       self.assertRaises(_lowercase		,						tokenizer_r.encode_plus		,						_lowercase		,						max_length=_lowercase		,						padding="max_length" )

                       # Simple input
                       self.assertRaises(
                           _lowercase		,						tokenizer_r.batch_encode_plus		,						_lowercase		,						max_length=_lowercase		,						padding="max_length"		,						)

                       # Pair input
                       self.assertRaises(_lowercase		,						tokenizer_r.encode		,						_lowercase		,						max_length=_lowercase		,						padding="max_length" )

                       # Pair input
                       self.assertRaises(_lowercase		,						tokenizer_r.encode_plus		,						_lowercase		,						max_length=_lowercase		,						padding="max_length" )

                       # Pair input
                       self.assertRaises(
                           _lowercase		,						tokenizer_r.batch_encode_plus		,						_lowercase		,						max_length=_lowercase		,						padding="max_length"		,						)






  def 			_UpperCAmelCase  (					self      :					str ):




         """simple docstring"""







         pass






  def 			_UpperCAmelCase  (					self      :					int ):




         """simple docstring"""







         UpperCAmelCase__										=     ReformerTokenizer(_lowercase		,						keep_accents=_lowercase )

         UpperCAmelCase__										=     tokenizer.tokenize("This is a test" )
         self.assertListEqual(_lowercase		,						["▁This", "▁is", "▁a", "▁t", "est"] )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(_lowercase )		,						[2_85, 46, 10, 1_70, 3_82]		,						)

         UpperCAmelCase__										=     tokenizer.tokenize("I was born in 92000, and this is falsé." )
         self.assertListEqual(
             _lowercase		,						[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]		,						)
         UpperCAmelCase__										=     tokenizer.convert_tokens_to_ids(_lowercase )
         self.assertListEqual(
             _lowercase		,						[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]		,						)

         UpperCAmelCase__										=     tokenizer.convert_ids_to_tokens(_lowercase )
         self.assertListEqual(
             _lowercase		,						[
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]		,						)






  @cached_property
  def 			_UpperCAmelCase  (					self      :					Any ):




         """simple docstring"""







         return ReformerTokenizer.from_pretrained("google/reformer-crime-and-punishment" )






  @slow
  def 			_UpperCAmelCase  (					self      :					Union[str, Any] ):




         """simple docstring"""







         UpperCAmelCase__										=     "Hello World!"
         UpperCAmelCase__										=     [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

         self.assertListEqual(_lowercase		,						self.big_tokenizer.encode(_lowercase ) )






  @slow
  def 			_UpperCAmelCase  (					self      :					List[Any] ):




         """simple docstring"""







         UpperCAmelCase__										=     (
             "This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"
             " add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"
         )
         UpperCAmelCase__										=     [
             1_08,
             2_65,
             24,
             1_11,
             4,
             2_58,
             1_56,
             35,
             28,
             2_75,
             3,
             2_59,
             2_97,
             2_60,
             84,
             4,
             35,
             1_10,
             44,
             8,
             2_59,
             91,
             2_68,
             21,
             11,
             2_09,
             2_74,
             1_09,
             2_66,
             2_77,
             1_17,
             86,
             93,
             3_15,
             2_58,
             2_78,
             2_58,
             2_77,
             2_58,
             0,
             2_58,
             2_88,
             2_58,
             3_19,
             2_58,
             0,
             2_58,
             0,
             2_58,
             0,
             2_58,
             0,
             2_58,
             2_87,
             2_58,
             3_15,
             2_58,
             2_89,
             2_58,
             2_78,
             99,
             2_69,
             2_66,
             2_62,
             8,
             2_59,
             2_41,
             4,
             2_17,
             2_30,
             2_68,
             2_66,
             55,
             1_68,
             1_06,
             75,
             1_93,
             2_66,
             2_23,
             27,
             49,
             26,
             2_82,
             25,
             2_64,
             2_99,
             19,
             26,
             0,
             2_58,
             2_77,
             1_17,
             86,
             93,
             1_76,
             1_83,
             2_70,
             11,
             2_62,
             42,
             61,
             2_65,
         ]

         self.assertListEqual(_lowercase		,						self.big_tokenizer.encode(_lowercase ) )






  @require_torch
  @slow
  def 			_UpperCAmelCase  (					self      :					Optional[int] ):




         """simple docstring"""







         import torch

         from transformers import ReformerConfig, ReformerModel

         # Build sequence
         UpperCAmelCase__										=     list(self.big_tokenizer.get_vocab().keys() )[:10]
         UpperCAmelCase__										=     " ".join(_lowercase )
         UpperCAmelCase__										=     self.big_tokenizer.encode_plus(_lowercase		,						return_tensors="pt" )
         UpperCAmelCase__										=     self.big_tokenizer.batch_encode_plus([sequence, sequence]		,						return_tensors="pt" )

         UpperCAmelCase__										=     ReformerConfig()
         # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
         UpperCAmelCase__										=     encoded_sequence["input_ids"].shape
         UpperCAmelCase__										=     ReformerModel(_lowercase )

         # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
         assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

         with torch.no_grad():
                model(**_lowercase )
                model(**_lowercase )






  @slow
  def 			_UpperCAmelCase  (					self      :					Tuple ):




         """simple docstring"""







         UpperCAmelCase__										=     {"input_ids": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         # This tokenizer does not know some characters like ")".
         # That is the reason why we use very simple texts here.
         # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
         UpperCAmelCase__										=     [
             "This is a very simple sentence.",
             "The quick brown fox jumps over the lazy dog.",
         ]

         self.tokenizer_integration_test_util(
             expected_encoding=_lowercase		,						model_name="google/reformer-crime-and-punishment"		,						revision="0e6c3decb8211d49bf881013425dc8b0448b3f5a"		,						padding=_lowercase		,						sequences=_lowercase		,						)









from math import atan, cos, radians, sin, tan

from .haversine_distance import haversine_distance

A      = 6378137.0
A      = 6356752.314245
A      = 637_8137





def __UpperCAmelCase       (		__A ,						__A ,						__A ,						__A			)						->					float:



       '''simple docstring'''





       UpperCAmelCase__										=     (AXIS_A - AXIS_B) / AXIS_A
       # Parametric latitudes
       # https://en.wikipedia.org/wiki/Latitude#Parametric_(or_reduced)_latitude
       UpperCAmelCase__										=     atan((1 - flattening) * tan(radians(__A			)			)			)
       UpperCAmelCase__										=     atan((1 - flattening) * tan(radians(__A			)			)			)

       # Compute central angle between two points
       # using haversine theta. sigma =  haversine_distance / equatorial radius
       UpperCAmelCase__										=     haversine_distance(__A ,						__A ,						__A ,						__A			) / EQUATORIAL_RADIUS

       # Intermediate P and Q values
       UpperCAmelCase__										=     (b_lata + b_lata) / 2
       UpperCAmelCase__										=     (b_lata - b_lata) / 2

       # Intermediate X value
       # X = (sigma - sin(sigma)) * sin^2Pcos^2Q / cos^2(sigma/2)
       UpperCAmelCase__										=     (sin(__A			) ** 2) * (cos(__A			) ** 2)
       UpperCAmelCase__										=     cos(sigma / 2			) ** 2
       UpperCAmelCase__										=     (sigma - sin(__A			)) * (x_numerator / x_demonimator)

       # Intermediate Y value
       # Y = (sigma + sin(sigma)) * cos^2Psin^2Q / sin^2(sigma/2)
       UpperCAmelCase__										=     (cos(__A			) ** 2) * (sin(__A			) ** 2)
       UpperCAmelCase__										=     sin(sigma / 2			) ** 2
       UpperCAmelCase__										=     (sigma + sin(__A			)) * (y_numerator / y_denominator)

       return EQUATORIAL_RADIUS * (sigma - ((flattening / 2) * (x_value + y_value)))


if __name__ == "__main__":
    import doctest

    doctest.testmod()





