




'''simple docstring'''
import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
  from tokenizers import Tokenizer
  from tokenizers.models import WordPiece






class        SCREAMING_SNAKE_CASE(				unittest.TestCase     ):

       def     snake_case__			(    self					,		lowerCamelCase__     )    ->					Union[str, Any]:

           """simple docstring"""



           __lowercase							=						list(tokenizer_class.max_model_input_sizes.keys()     )
           for model_name in sa_models[:1]:
               __lowercase							=						tokenizer_class.from_pretrained(_a     )
               self.assertIsNotNone(_a     )
               self.assertIsInstance(_a					,		_a     )
               self.assertIsInstance(_a					,		_a     )

               for special_tok in tokenizer.all_special_tokens:
                   self.assertIsInstance(_a					,		_a     )
                   __lowercase							=						tokenizer.convert_tokens_to_ids(_a     )
                   self.assertIsInstance(_a					,		_a     )

       def     snake_case__			(    self					,		lowerCamelCase__					,		lowerCamelCase__ = None     )    ->					Union[str, Any]:

           """simple docstring"""



           __lowercase							=						pickle.dumps(_a     )
           self.assertIsNotNone(_a     )

           __lowercase							=						pickle.loads(_a     )

           # Ensure is_fast is correctly restored
           self.assertEqual(be_restored.is_fast					,		be_original.is_fast     )

           # Ensure encodings are potentially correctly restored
           if be_original.is_fast:
               self.assertIsNotNone(be_restored.encodings     )
           else:
               self.assertIsNone(be_restored.encodings     )

           # Ensure the keys are the same
           for original_v, restored_v in zip(be_original.values()					,		be_restored.values()     ):
               if equal_op:
                   self.assertTrue(equal_op(_a					,		_a     )     )
               else:
                   self.assertEqual(_a					,		_a     )

       @slow
       def     snake_case__			(    self     )    ->					int:

           """simple docstring"""



           self.check_tokenizer_from_pretrained(_a     )

       def     snake_case__			(    self     )    ->					List[str]:

           """simple docstring"""



           self.assertEqual(TensorType("""tf"""     )					,		TensorType.TENSORFLOW     )
           self.assertEqual(TensorType("""pt"""     )					,		TensorType.PYTORCH     )
           self.assertEqual(TensorType("""np"""     )					,		TensorType.NUMPY     )

       @require_tokenizers
       def     snake_case__			(    self     )    ->					Any:

           """simple docstring"""



           import numpy as np

           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )
           __lowercase							=						BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

           # Python no tensor
           with self.subTest("""BatchEncoding (Python, return_tensors=None)"""     ):
               self.assert_dump_and_restore(tokenizer_p("""Small example to encode"""     )     )

           with self.subTest("""BatchEncoding (Python, return_tensors=NUMPY)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,		return_tensors=TensorType.NUMPY     )					,		np.array_equal     )

           with self.subTest("""BatchEncoding (Rust, return_tensors=None)"""     ):
               self.assert_dump_and_restore(tokenizer_r("""Small example to encode"""     )     )

           with self.subTest("""BatchEncoding (Rust, return_tensors=NUMPY)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,		return_tensors=TensorType.NUMPY     )					,		np.array_equal     )

       @require_tf
       @require_tokenizers
       def     snake_case__			(    self     )    ->					List[str]:

           """simple docstring"""



           import tensorflow as tf

           def tf_array_equals(lowerCamelCase__					,		lowerCamelCase__     ):
               return tf.reduce_all(tf.equal(_a					,		_a     )     )

           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )
           __lowercase							=						BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

           with self.subTest("""BatchEncoding (Python, return_tensors=TENSORFLOW)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,		return_tensors=TensorType.TENSORFLOW     )					,		_a     )

           with self.subTest("""BatchEncoding (Rust, return_tensors=TENSORFLOW)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,		return_tensors=TensorType.TENSORFLOW     )					,		_a     )

       @require_torch
       @require_tokenizers
       def     snake_case__			(    self     )    ->					List[Any]:

           """simple docstring"""



           import torch

           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )
           __lowercase							=						BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

           with self.subTest("""BatchEncoding (Python, return_tensors=PYTORCH)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_p("""Small example to encode"""					,		return_tensors=TensorType.PYTORCH     )					,		torch.equal     )

           with self.subTest("""BatchEncoding (Rust, return_tensors=PYTORCH)"""     ):
               self.assert_dump_and_restore(
                   tokenizer_r("""Small example to encode"""					,		return_tensors=TensorType.PYTORCH     )					,		torch.equal     )

       @require_tokenizers
       def     snake_case__			(    self     )    ->					List[str]:

           """simple docstring"""



           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )
           __lowercase							=						BertTokenizerFast.from_pretrained("""bert-base-cased"""     )

           with self.subTest("""Python Tokenizer"""     ):
               self.assertFalse(tokenizer_p("""Small example to_encode"""     ).is_fast     )

           with self.subTest("""Rust Tokenizer"""     ):
               self.assertTrue(tokenizer_r("""Small example to_encode"""     ).is_fast     )

       @require_tokenizers
       def     snake_case__			(    self     )    ->					Dict:

           """simple docstring"""



           __lowercase							=						BertTokenizerFast.from_pretrained("""bert-base-cased"""     )
           __lowercase							=						tokenizer_r(["""Test""", """\xad""", """test"""]					,		is_split_into_words=_a     )

           self.assertEqual(encoded.word_to_tokens(0     )					,		TokenSpan(start=1					,		end=2     )     )
           self.assertEqual(encoded.word_to_tokens(1     )					,		_a     )
           self.assertEqual(encoded.word_to_tokens(2     )					,		TokenSpan(start=2					,		end=3     )     )

       def     snake_case__			(    self     )    ->					Any:

           """simple docstring"""



           __lowercase							=						BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""np"""     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(2, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
               __lowercase							=						batch.convert_to_tensors(tensor_type="""np"""     )
           self.assertFalse(len(cs.err     )					,		msg=F'should have no warning, but got {cs.err}'     )

           __lowercase							=						BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""np"""					,		prepend_batch_axis=_a     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(1, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(1,)     )

       @require_torch
       def     snake_case__			(    self     )    ->					Dict:

           """simple docstring"""



           __lowercase							=						BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""pt"""     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(2, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
               __lowercase							=						batch.convert_to_tensors(tensor_type="""pt"""     )
           self.assertFalse(len(cs.err     )					,		msg=F'should have no warning, but got {cs.err}'     )

           __lowercase							=						BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""pt"""					,		prepend_batch_axis=_a     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(1, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(1,)     )

       @require_tf
       def     snake_case__			(    self     )    ->					Optional[int]:

           """simple docstring"""



           __lowercase							=						BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""tf"""     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(2, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
               __lowercase							=						batch.convert_to_tensors(tensor_type="""tf"""     )
           self.assertFalse(len(cs.err     )					,		msg=F'should have no warning, but got {cs.err}'     )

           __lowercase							=						BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""tf"""					,		prepend_batch_axis=_a     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(1, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(1,)     )

       @require_flax
       def     snake_case__			(    self     )    ->					Any:

           """simple docstring"""



           __lowercase							=						BatchEncoding({"""inputs""": [[1, 2, 3], [4, 5, 6]], """labels""": [0, 1]}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""jax"""     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(2, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(2,)     )
           # test converting the converted
           with CaptureStderr() as cs:
               __lowercase							=						batch.convert_to_tensors(tensor_type="""jax"""     )
           self.assertFalse(len(cs.err     )					,		msg=F'should have no warning, but got {cs.err}'     )

           __lowercase							=						BatchEncoding({"""inputs""": [1, 2, 3], """labels""": 0}     )
           __lowercase							=						batch.convert_to_tensors(tensor_type="""jax"""					,		prepend_batch_axis=_a     )
           self.assertEqual(tensor_batch["""inputs"""].shape					,		(1, 3)     )
           self.assertEqual(tensor_batch["""labels"""].shape					,		(1,)     )

       def     snake_case__			(    self     )    ->					List[Any]:

           """simple docstring"""



           __lowercase							=						[{"""input_ids""": np.array([0, 1, 2]     )}, {"""input_ids""": np.array([0, 1, 2, 3]     )}]
           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )

           __lowercase							=						tokenizer.pad(_a					,		padding=_a     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		np.ndarray     )     )
           self.assertEqual(batch["""input_ids"""].tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __lowercase							=						tokenizer.pad(_a					,		padding=_a					,		return_tensors="""np"""     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		np.ndarray     )     )
           self.assertEqual(batch["""input_ids"""].tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

       @require_torch
       def     snake_case__			(    self     )    ->					Optional[Any]:

           """simple docstring"""



           import torch

           __lowercase							=						[{"""input_ids""": torch.tensor([0, 1, 2]     )}, {"""input_ids""": torch.tensor([0, 1, 2, 3]     )}]
           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )

           __lowercase							=						tokenizer.pad(_a					,		padding=_a     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		torch.Tensor     )     )
           self.assertEqual(batch["""input_ids"""].tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __lowercase							=						tokenizer.pad(_a					,		padding=_a					,		return_tensors="""pt"""     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		torch.Tensor     )     )
           self.assertEqual(batch["""input_ids"""].tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

       @require_tf
       def     snake_case__			(    self     )    ->					Any:

           """simple docstring"""



           import tensorflow as tf

           __lowercase							=						[{"""input_ids""": tf.constant([0, 1, 2]     )}, {"""input_ids""": tf.constant([0, 1, 2, 3]     )}]
           __lowercase							=						BertTokenizer.from_pretrained("""bert-base-cased"""     )

           __lowercase							=						tokenizer.pad(_a					,		padding=_a     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		tf.Tensor     )     )
           self.assertEqual(batch["""input_ids"""].numpy().tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )
           __lowercase							=						tokenizer.pad(_a					,		padding=_a					,		return_tensors="""tf"""     )
           self.assertTrue(isinstance(batch["""input_ids"""]					,		tf.Tensor     )     )
           self.assertEqual(batch["""input_ids"""].numpy().tolist()					,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]     )

       @require_tokenizers
       def     snake_case__			(    self     )    ->					str:

           """simple docstring"""



           __lowercase							=						Tokenizer(WordPiece(unk_token="""[UNK]"""     )     )
           PreTrainedTokenizerFast(tokenizer_object=_a     )




       @require_tokenizers
       def     snake_case__			(    self     )    ->					Dict:

           """simple docstring"""



           __lowercase							=						Tokenizer(WordPiece(unk_token="""[UNK]"""     )     )
           with tempfile.TemporaryDirectory() as tmpdirname:
               bert_tokenizer.save(os.path.join(_a					,		"""tokenizer.json"""     )     )
               PreTrainedTokenizerFast(tokenizer_file=os.path.join(_a					,		"""tokenizer.json"""     )     )








from __future__ import annotations

from collections.abc import Callable







def   lowerCamelCase(      a__		,a__		,a__		,a__ = 100		,):
				_SCREAMING_SNAKE_CASE	     =x_start
				_SCREAMING_SNAKE_CASE	     =fnc(a__)
				_SCREAMING_SNAKE_CASE	     =0.0

				for _ in range(a__):
								# Approximates small segments of curve as linear and solve
								# for trapezoidal area
								_SCREAMING_SNAKE_CASE	     =(x_end - x_start) / steps + xa
								_SCREAMING_SNAKE_CASE	     =fnc(a__)
								area += abs(fxa + fxa) * (xa - xa) / 2

								# Increment step
								_SCREAMING_SNAKE_CASE	     =xa
								_SCREAMING_SNAKE_CASE	     =fxa
				return area


if __name__ == "__main__":







							def   lowerCamelCase(      a__):
											return x**3

							print('''f(x) = x^3''')
							print('''The area between the curve, x = -10, x = 10 and the x axis is:''')
							snake_case_  :	Any							  =							10
							while i <= 10_00_00:
														snake_case_  :	Tuple							  =							trapezoidal_area(f, -5, 5, i)
														print(f"""with {i} steps: {area}""")
														i *= 10