


'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_ :     str								=      get_tests_dir('fixtures/test_sentencepiece.model')

@require_sentencepiece
@require_tokenizers
class 						_lowerCamelCase					(    UpperCamelCase_    ,		unittest.TestCase				):




 '''simple docstring'''
 __lowercase				:		Any    							=						XLNetTokenizer
 __lowercase				:		List[str]    							=						XLNetTokenizerFast
 __lowercase				:		Union[str, Any]    							=						True
 __lowercase				:		str    							=						True


 def  snake_case__   (      self						):

       """simple docstring"""







       super().setUp()

       # We have a SentencePiece fixture for testing
       __A  :		int       =			XLNetTokenizer(__a      ,      keep_accents=__a						)
       tokenizer.sanitize_special_tokens()
       tokenizer.save_pretrained(self.tmpdirname						)


 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		Optional[int]       =			'<s>'
       __A  :		Tuple       =			1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a						)      ,      __a						)
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a						)      ,      __a						)


 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		Tuple       =			list(self.get_tokenizer().get_vocab().keys()						)

       self.assertEqual(vocab_keys[0]      ,      '<unk>'						)
       self.assertEqual(vocab_keys[1]      ,      '<s>'						)
       self.assertEqual(vocab_keys[-1]      ,      '<eod>'						)
       self.assertEqual(len(__a						)      ,      1_006						)


 def  snake_case__   (      self						):

       """simple docstring"""







       self.assertEqual(self.get_tokenizer().vocab_size      ,      1_000						)


 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		Dict       =			XLNetTokenizer(__a      ,      keep_accents=__a						)

       __A  :		List[Any]       =			tokenizer.tokenize('This is a test'						)
       self.assertListEqual(__a      ,      ['▁This', '▁is', '▁a', '▁t', 'est']						)

       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a						)      ,      [285, 46, 10, 170, 382]						)

       __A  :		List[Any]       =			tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       self.assertListEqual(
           __a      ,      [
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               'é',
               '.',
           ]      ,      )
       __A  :		Tuple       =			tokenizer.convert_tokens_to_ids(__a						)
       self.assertListEqual(__a      ,      [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]						)

       __A  :		Union[str, Any]       =			tokenizer.convert_ids_to_tokens(__a						)
       self.assertListEqual(
           __a      ,      [
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '<unk>',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               's',
               '<unk>',
               '.',
           ]      ,      )


 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		List[Any]       =			XLNetTokenizer(__a      ,      do_lower_case=__a						)
       __A  :		Optional[int]       =			tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       self.assertListEqual(
           __a      ,      [
               SPIECE_UNDERLINE + '',
               'i',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               'se',
               '.',
           ]      ,      )
       self.assertListEqual(tokenizer.tokenize('H\u00E9llo'						)      ,      ['▁he', 'll', 'o']						)


 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		int       =			XLNetTokenizer(__a      ,      do_lower_case=__a						)
       __A  :		Tuple       =			tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
       self.assertListEqual(
           __a      ,      [
               SPIECE_UNDERLINE + 'I',
               SPIECE_UNDERLINE + 'was',
               SPIECE_UNDERLINE + 'b',
               'or',
               'n',
               SPIECE_UNDERLINE + 'in',
               SPIECE_UNDERLINE + '',
               '9',
               '2',
               '0',
               '0',
               '0',
               ',',
               SPIECE_UNDERLINE + 'and',
               SPIECE_UNDERLINE + 'this',
               SPIECE_UNDERLINE + 'is',
               SPIECE_UNDERLINE + 'f',
               'al',
               'se',
               '.',
           ]      ,      )


 @slow
 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		List[Any]       =			XLNetTokenizer.from_pretrained('xlnet-base-cased'						)

       __A  :		List[str]       =			tokenizer.encode('sequence builders'      ,      add_special_tokens=__a						)
       __A  :		List[Any]       =			tokenizer.encode('multi-sequence build'      ,      add_special_tokens=__a						)

       __A  :		Dict       =			tokenizer.build_inputs_with_special_tokens(__a						)
       __A  :		Dict       =			tokenizer.build_inputs_with_special_tokens(__a      ,      __a						)

       assert encoded_sentence == text + [4, 3]
       assert encoded_pair == text + [4] + text_a + [4, 3]







 @slow
 def  snake_case__   (      self						):

       """simple docstring"""







       __A  :		List[str]       =			{'input_ids': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
       # fmt: on

       self.tokenizer_integration_test_util(
           expected_encoding=__a      ,      model_name='xlnet-base-cased'      ,      revision='c841166438c31ec7ca9a106dee7bb312b73ae511'      ,      )










'''simple docstring'''
from math import pi, sqrt, tan


def 			_lowercase       (				UpperCamelCase__       :					float			):
 if side_length < 0:
  raise ValueError('surface_area_cube() only accepts non-negative values'			)
 return 6 * side_length**2


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if length < 0 or breadth < 0 or height < 0:
  raise ValueError('surface_area_cuboid() only accepts non-negative values'			)
 return 2 * ((length * breadth) + (breadth * height) + (length * height))


def 			_lowercase       (				UpperCamelCase__       :					float			):
 if radius < 0:
  raise ValueError('surface_area_sphere() only accepts non-negative values'			)
 return 4 * pi * radius**2


def 			_lowercase       (				UpperCamelCase__       :					float			):
 if radius < 0:
  raise ValueError('surface_area_hemisphere() only accepts non-negative values'			)
 return 3 * pi * radius**2


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if radius < 0 or height < 0:
  raise ValueError('surface_area_cone() only accepts non-negative values'			)
 return pi * radius * (radius + (height**2 + radius**2) ** 0.5)


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if radius_a < 0 or radius_a < 0 or height < 0:
  raise ValueError(
      'surface_area_conical_frustum() only accepts non-negative values'			)
 __A  :		Optional[Any]       =			(height**2 + (radius_a - radius_a) ** 2) ** 0.5
 return pi * ((slant_height * (radius_a + radius_a)) + radius_a**2 + radius_a**2)


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if radius < 0 or height < 0:
  raise ValueError('surface_area_cylinder() only accepts non-negative values'			)
 return 2 * pi * radius * (height + radius)


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if torus_radius < 0 or tube_radius < 0:
  raise ValueError('surface_area_torus() only accepts non-negative values'			)
 if torus_radius < tube_radius:
  raise ValueError(
      'surface_area_torus() does not support spindle or self intersecting tori'			)
 return 4 * pow(UpperCamelCase__,				2			) * torus_radius * tube_radius


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if length < 0 or width < 0:
  raise ValueError('area_rectangle() only accepts non-negative values'			)
 return length * width


def 			_lowercase       (				UpperCamelCase__       :					float			):
 if side_length < 0:
  raise ValueError('area_square() only accepts non-negative values'			)
 return side_length**2


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if base < 0 or height < 0:
  raise ValueError('area_triangle() only accepts non-negative values'			)
 return (base * height) / 2


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if sidea < 0 or sidea < 0 or sidea < 0:
  raise ValueError('area_triangle_three_sides() only accepts non-negative values'			)
 elif sidea + sidea < sidea or sidea + sidea < sidea or sidea + sidea < sidea:
  raise ValueError('Given three sides do not form a triangle'			)
 __A  :		int       =			(sidea + sidea + sidea) / 2
 __A  :		List[Any]       =			sqrt(
     semi_perimeter
     * (semi_perimeter - sidea)
     * (semi_perimeter - sidea)
     * (semi_perimeter - sidea)			)
 return area


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if base < 0 or height < 0:
  raise ValueError('area_parallelogram() only accepts non-negative values'			)
 return base * height


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if basea < 0 or basea < 0 or height < 0:
  raise ValueError('area_trapezium() only accepts non-negative values'			)
 return 1 / 2 * (basea + basea) * height


def 			_lowercase       (				UpperCamelCase__       :					float			):
 if radius < 0:
  raise ValueError('area_circle() only accepts non-negative values'			)
 return pi * radius**2


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if radius_x < 0 or radius_y < 0:
  raise ValueError('area_ellipse() only accepts non-negative values'			)
 return pi * radius_x * radius_y


def 			_lowercase       (				UpperCamelCase__       :					float,				UpperCamelCase__       :					float			):
 if diagonal_a < 0 or diagonal_a < 0:
  raise ValueError('area_rhombus() only accepts non-negative values'			)
 return 1 / 2 * diagonal_a * diagonal_a


def 			_lowercase       (				UpperCamelCase__       :					int,				UpperCamelCase__       :					float			):
 if not isinstance(UpperCamelCase__,				UpperCamelCase__			) or sides < 3:
  raise ValueError(
      'area_reg_polygon() only accepts integers greater than or \
equal to three as number of sides'			)
 elif length < 0:
  raise ValueError(
      'area_reg_polygon() only accepts non-negative values as \
length of a side'			)
 return (sides * length**2) / (4 * tan(pi / sides			))
 return (sides * length**2) / (4 * tan(pi / sides			))


if __name__ == "__main__":
 import doctest

 doctest.testmod(verbose=True)  # verbose so we can see methods missing tests

 print('[DEMO] Areas of various geometric shapes: \n')
 print(f'''Rectangle: {area_rectangle(1_0, 2_0) = }''')
 print(f'''Square: {area_square(1_0) = }''')
 print(f'''Triangle: {area_triangle(1_0, 1_0) = }''')
 print(f'''Triangle: {area_triangle_three_sides(5, 1_2, 1_3) = }''')
 print(f'''Parallelogram: {area_parallelogram(1_0, 2_0) = }''')
 print(f'''Rhombus: {area_rhombus(1_0, 2_0) = }''')
 print(f'''Trapezium: {area_trapezium(1_0, 2_0, 3_0) = }''')
 print(f'''Circle: {area_circle(2_0) = }''')
 print(f'''Ellipse: {area_ellipse(1_0, 2_0) = }''')
 print('\nSurface Areas of various geometric shapes: \n')
 print(f'''Cube: {surface_area_cube(2_0) = }''')
 print(f'''Cuboid: {surface_area_cuboid(1_0, 2_0, 3_0) = }''')
 print(f'''Sphere: {surface_area_sphere(2_0) = }''')
 print(f'''Hemisphere: {surface_area_hemisphere(2_0) = }''')
 print(f'''Cone: {surface_area_cone(1_0, 2_0) = }''')
 print(f'''Conical Frustum: {surface_area_conical_frustum(1_0, 2_0, 3_0) = }''')
 print(f'''Cylinder: {surface_area_cylinder(1_0, 2_0) = }''')
 print(f'''Torus: {surface_area_torus(2_0, 1_0) = }''')
 print(f'''Equilateral Triangle: {area_reg_polygon(3, 1_0) = }''')
 print(f'''Square: {area_reg_polygon(4, 1_0) = }''')
 print(f'''Reqular Pentagon: {area_reg_polygon(5, 1_0) = }''')






