





'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					str						=			mock.Mock()
          __A		:					List[Any]						=			500
          __A		:					Optional[int]						=			{}
          __A		:					str						=			HTTPError
          __A		:					Any						=			{}

          # Download this model to make sure it's in the cache.
          __A		:					Tuple						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('requests.Session.request'		,      return_value=_UpperCAmelCase) as mock_head:
               __A		:					Optional[Any]						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
               # This check we did call the fake head request
               mock_head.assert_called()


     @require_tokenizers
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			mock.Mock()
          __A		:					Union[str, Any]						=			500
          __A		:					Any						=			{}
          __A		:					Tuple						=			HTTPError
          __A		:					Tuple						=			{}

          # Download this model to make sure it's in the cache.
          __A		:					Optional[int]						=			GPTaTokenizerFast.from_pretrained('gpt2')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('requests.Session.request'		,      return_value=_UpperCAmelCase) as mock_head:
               __A		:					str						=			GPTaTokenizerFast.from_pretrained('gpt2')
               # This check we did call the fake head request
               mock_head.assert_called()


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          try:
               __A		:					Any						=			tempfile.mktemp()
               with open(_UpperCAmelCase		,      'wb') as f:
                    http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'		,      _UpperCAmelCase)

               __A		:					Tuple						=			AlbertTokenizer.from_pretrained(_UpperCAmelCase)
          finally:
               os.remove(_UpperCAmelCase)

          # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
          # the current folder and have the right name.
          if os.path.isfile('tokenizer.json'):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
          try:
               with open('tokenizer.json'		,      'wb') as f:
                    http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'		,      _UpperCAmelCase)
               __A		:					Any						=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size		,      1000)
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

          finally:
               os.remove('tokenizer.json')


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')




@is_staging_test
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):
     lowerCAmelCase						     =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']


     @classmethod
     def 				SCREAMING_SNAKE_CASE    (		cls):

          '''simple docstring'''






          __A		:					List[Any]						=			TOKEN
          HfFolder.save_token(_UpperCAmelCase)


     @classmethod
     def 				SCREAMING_SNAKE_CASE    (		cls):

          '''simple docstring'''






          try:
               delete_repo(token=cls._token		,      repo_id='test-tokenizer')
          except HTTPError:
               pass

          try:
               delete_repo(token=cls._token		,      repo_id='valid_org/test-tokenizer-org')
          except HTTPError:
               pass

          try:
               delete_repo(token=cls._token		,      repo_id='test-dynamic-tokenizer')
          except HTTPError:
               pass


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					Dict						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					Optional[Any]						=			BertTokenizer(_UpperCAmelCase)

          tokenizer.push_to_hub('test-tokenizer'		,      use_auth_token=self._token)
          __A		:					Dict						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token		,      repo_id='test-tokenizer')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase		,      repo_id='test-tokenizer'		,      push_to_hub=_UpperCAmelCase		,      use_auth_token=self._token)

          __A		:					Any						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					Optional[int]						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					List[Any]						=			BertTokenizer(_UpperCAmelCase)

          tokenizer.push_to_hub('valid_org/test-tokenizer-org'		,      use_auth_token=self._token)
          __A		:					List[Any]						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token		,      repo_id='valid_org/test-tokenizer-org')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   _UpperCAmelCase		,      repo_id='valid_org/test-tokenizer-org'		,      push_to_hub=_UpperCAmelCase		,      use_auth_token=self._token)

          __A		:					str						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)


     @require_tokenizers
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          CustomTokenizer.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					str						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					Any						=			CustomTokenizer(_UpperCAmelCase)

          # No fast custom tokenizer
          tokenizer.push_to_hub('test-dynamic-tokenizer'		,      use_auth_token=self._token)

          __A		:					List[Any]						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizer')

          # Fast and slow custom tokenizer
          CustomTokenizerFast.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					List[str]						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

               __A		:					Tuple						=			BertTokenizerFast.from_pretrained(_UpperCAmelCase)
               bert_tokenizer.save_pretrained(_UpperCAmelCase)
               __A		:					Any						=			CustomTokenizerFast.from_pretrained(_UpperCAmelCase)

          tokenizer.push_to_hub('test-dynamic-tokenizer'		,      use_auth_token=self._token)

          __A		:					Tuple						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizerFast')
          __A		:					Union[str, Any]						=			AutoTokenizer.from_pretrained(
              F'{USER}/test-dynamic-tokenizer'		,      use_fast=_UpperCAmelCase		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizer')




class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					str						=			Trie()
          trie.add('Hello 友達')
          self.assertEqual(trie.data		,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
          trie.add('Hello')
          trie.data
          self.assertEqual(trie.data		,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Any						=			Trie()
          self.assertEqual(trie.split('[CLS] This is a extra_id_100')		,      ['[CLS] This is a extra_id_100'])
          trie.add('[CLS]')
          trie.add('extra_id_1')
          trie.add('extra_id_100')
          self.assertEqual(trie.split('[CLS] This is a extra_id_100')		,      ['[CLS]', ' This is a ', 'extra_id_100'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Union[str, Any]						=			Trie()
          trie.add('A')
          self.assertEqual(trie.split('ABC')		,      ['A', 'BC'])
          self.assertEqual(trie.split('BCA')		,      ['BC', 'A'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			Trie()
          trie.add('TOKEN]')
          trie.add('[SPECIAL_TOKEN]')
          self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')		,      ['This is something ', '[SPECIAL_TOKEN]'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			Trie()
          trie.add('A')
          trie.add('P')
          trie.add('[SPECIAL_TOKEN]')
          self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')		,      ['This is something ', '[SPECIAL_TOKEN]'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			Trie()
          trie.add('AB')
          trie.add('B')
          trie.add('C')
          self.assertEqual(trie.split('ABC')		,      ['AB', 'C'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[Any]						=			Trie()
          trie.add('ABC')
          trie.add('B')
          trie.add('CD')
          self.assertEqual(trie.split('ABCD')		,      ['ABC', 'D'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			Trie()
          __A		:					Optional[int]						=			trie.cut_text('ABC'		,      [0, 0, 2, 1, 2, 3])
          self.assertEqual(_UpperCAmelCase		,      ['AB', 'C'])






from math import pi, sqrt, tan

def      UpperCAmelCase_    (    _UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if side_length < 0:
          raise ValueError('''surface_area_cube() only accepts non-negative values'''				)
     return 6 * side_length**2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if length < 0 or breadth < 0 or height < 0:
          raise ValueError('''surface_area_cuboid() only accepts non-negative values'''				)
     return 2 * ((length * breadth) + (breadth * height) + (length * height))

def      UpperCAmelCase_    (    _UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius < 0:
          raise ValueError('''surface_area_sphere() only accepts non-negative values'''				)
     return 4 * pi * radius**2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius < 0:
          raise ValueError('''surface_area_hemisphere() only accepts non-negative values'''				)
     return 3 * pi * radius**2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius < 0 or height < 0:
          raise ValueError('''surface_area_cone() only accepts non-negative values'''				)
     return pi * radius * (radius + (height**2 + radius**2) ** 0.5)

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius_a < 0 or radius_a < 0 or height < 0:
          raise ValueError(
              '''surface_area_conical_frustum() only accepts non-negative values'''				)
     A_          =       (height**2 + (radius_a - radius_a) ** 2) ** 0.5
     return pi * ((slant_height * (radius_a + radius_a)) + radius_a**2 + radius_a**2)

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius < 0 or height < 0:
          raise ValueError('''surface_area_cylinder() only accepts non-negative values'''				)
     return 2 * pi * radius * (height + radius)

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if torus_radius < 0 or tube_radius < 0:
          raise ValueError('''surface_area_torus() only accepts non-negative values'''				)
     if torus_radius < tube_radius:
          raise ValueError(
              '''surface_area_torus() does not support spindle or self intersecting tori'''				)
     return 4 * pow(_UpperCAmelCase  ,						2				) * torus_radius * tube_radius

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if length < 0 or width < 0:
          raise ValueError('''area_rectangle() only accepts non-negative values'''				)
     return length * width

def      UpperCAmelCase_    (    _UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if side_length < 0:
          raise ValueError('''area_square() only accepts non-negative values'''				)
     return side_length**2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if base < 0 or height < 0:
          raise ValueError('''area_triangle() only accepts non-negative values'''				)
     return (base * height) / 2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if sidea < 0 or sidea < 0 or sidea < 0:
          raise ValueError('''area_triangle_three_sides() only accepts non-negative values'''				)
     elif sidea + sidea < sidea or sidea + sidea < sidea or sidea + sidea < sidea:
          raise ValueError('''Given three sides do not form a triangle'''				)
     A_          =       (sidea + sidea + sidea) / 2
     A_          =       sqrt(
         semi_perimeter
         * (semi_perimeter - sidea)
         * (semi_perimeter - sidea)
         * (semi_perimeter - sidea)				)
     return area

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if base < 0 or height < 0:
          raise ValueError('''area_parallelogram() only accepts non-negative values'''				)
     return base * height

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if basea < 0 or basea < 0 or height < 0:
          raise ValueError('''area_trapezium() only accepts non-negative values'''				)
     return 1 / 2 * (basea + basea) * height

def      UpperCAmelCase_    (    _UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius < 0:
          raise ValueError('''area_circle() only accepts non-negative values'''				)
     return pi * radius**2

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if radius_x < 0 or radius_y < 0:
          raise ValueError('''area_ellipse() only accepts non-negative values'''				)
     return pi * radius_x * radius_y

def      UpperCAmelCase_    (    _UpperCAmelCase							:float  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if diagonal_a < 0 or diagonal_a < 0:
          raise ValueError('''area_rhombus() only accepts non-negative values'''				)
     return 1 / 2 * diagonal_a * diagonal_a

def      UpperCAmelCase_    (    _UpperCAmelCase							:int  ,						_UpperCAmelCase							:float				)	->				float:
     '''simple docstring'''


     if not isinstance(_UpperCAmelCase  ,						_UpperCAmelCase				) or sides < 3:
          raise ValueError(
              '''area_reg_polygon() only accepts integers greater than or \
equal to three as number of sides'''				)
     elif length < 0:
          raise ValueError(
              '''area_reg_polygon() only accepts non-negative values as \
length of a side'''				)
     return (sides * length**2) / (4 * tan(pi / sides				))
     return (sides * length**2) / (4 * tan(pi / sides				))


if __name__ == "__main__":
      import doctest

      doctest.testmod(verbose=True)  # verbose so we can see methods missing tests

      print('[DEMO] Areas of various geometric shapes: \n')
      print(f'''Rectangle: {area_rectangle(10, 20) = }''')
      print(f'''Square: {area_square(10) = }''')
      print(f'''Triangle: {area_triangle(10, 10) = }''')
      print(f'''Triangle: {area_triangle_three_sides(5, 12, 13) = }''')
      print(f'''Parallelogram: {area_parallelogram(10, 20) = }''')
      print(f'''Rhombus: {area_rhombus(10, 20) = }''')
      print(f'''Trapezium: {area_trapezium(10, 20, 30) = }''')
      print(f'''Circle: {area_circle(20) = }''')
      print(f'''Ellipse: {area_ellipse(10, 20) = }''')
      print('\nSurface Areas of various geometric shapes: \n')
      print(f'''Cube: {surface_area_cube(20) = }''')
      print(f'''Cuboid: {surface_area_cuboid(10, 20, 30) = }''')
      print(f'''Sphere: {surface_area_sphere(20) = }''')
      print(f'''Hemisphere: {surface_area_hemisphere(20) = }''')
      print(f'''Cone: {surface_area_cone(10, 20) = }''')
      print(f'''Conical Frustum: {surface_area_conical_frustum(10, 20, 30) = }''')
      print(f'''Cylinder: {surface_area_cylinder(10, 20) = }''')
      print(f'''Torus: {surface_area_torus(20, 10) = }''')
      print(f'''Equilateral Triangle: {area_reg_polygon(3, 10) = }''')
      print(f'''Square: {area_reg_polygon(4, 10) = }''')
      print(f'''Reqular Pentagon: {area_reg_polygon(5, 10) = }''')



