
import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
			import PIL.Image

			from .features import FeatureType


_A     =  None
_A     =  "<" if sys.byteorder == "little" else ">"
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
_A     =  [
    np.dtype("|b1"),
    np.dtype("|u1"),
    np.dtype("<u2"),
    np.dtype(">u2"),
    np.dtype("<i2"),
    np.dtype(">i2"),
    np.dtype("<u4"),
    np.dtype(">u4"),
    np.dtype("<i4"),
    np.dtype(">i4"),
    np.dtype("<f4"),
    np.dtype(">f4"),
    np.dtype("<f8"),
    np.dtype(">f8"),
]





@dataclass
class 			_lowerCAmelCase      :
				_lowercase    	=True
				_lowercase    	=None
				# Automatically constructed
				_lowercase    	="PIL.Image.Image"
				_lowercase    	=pa.struct({'''bytes''': pa.binary(), '''path''': pa.string()} )
				_lowercase    	=field(default='''Image'''      ,   init=__a      ,   repr=__a )




				def __call__(      self  )       ->   Any:
							return self.pa_type




				def   __a		(      self       ,       _UpperCamelCase  )       ->   dict:
							if config.PIL_AVAILABLE:
										import PIL.Image
							else:
										raise ImportError("To support encoding images, please install 'Pillow'."  )

							if isinstance(_UpperCamelCase       ,       _UpperCamelCase  ):
										lowerCAmelCase_      			=			np.array(_UpperCamelCase  )

							if isinstance(_UpperCamelCase       ,       _UpperCamelCase  ):
										return {"path": value, "bytes": None}
							elif isinstance(_UpperCamelCase       ,       _UpperCamelCase  ):
										return {"path": None, "bytes": value}
							elif isinstance(_UpperCamelCase       ,       np.ndarray  ):
										# convert the image array to PNG/TIFF bytes
										return encode_np_array(_UpperCamelCase  )
							elif isinstance(_UpperCamelCase       ,       PIL.Image.Image  ):
										# convert the PIL image to bytes (default format is PNG/TIFF)
										return encode_pil_image(_UpperCamelCase  )
							elif value.get("path"  ) is not None and os.path.isfile(value["path"]  ):
										# we set "bytes": None to not duplicate the data if they're already available locally
										return {"bytes": None, "path": value.get("path"  )}
							elif value.get("bytes"  ) is not None or value.get("path"  ) is not None:
										# store the image bytes, and path is used to infer the image format using the file extension
										return {"bytes": value.get("bytes"  ), "path": value.get("path"  )}
							else:
										raise ValueError(
										    f"""An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""  )




				def   __a		(      self       ,       _UpperCamelCase       ,       _UpperCamelCase=None  )       ->   "PIL.Image.Image":
							if not self.decode:
										raise RuntimeError("Decoding is disabled for this feature. Please use Image(decode=True) instead."  )

							if config.PIL_AVAILABLE:
										import PIL.Image
							else:
										raise ImportError("To support decoding images, please install 'Pillow'."  )

							if token_per_repo_id is None:
										lowerCAmelCase_      			=			{}

							lowerCAmelCase_			,						lowerCAmelCase_      			=			value["path"], value["bytes"]
							if bytes_ is None:
										if path is None:
													raise ValueError(f"""An image should have one of 'path' or 'bytes' but both are None in {value}."""  )
										else:
													if is_local_path(_UpperCamelCase  ):
																lowerCAmelCase_      			=			PIL.Image.open(_UpperCamelCase  )
													else:
																lowerCAmelCase_      			=			path.split("::"  )[-1]
																try:
																			lowerCAmelCase_      			=			string_to_dict(_UpperCamelCase       ,       config.HUB_DATASETS_URL  )["repo_id"]
																			lowerCAmelCase_      			=			token_per_repo_id.get(_UpperCamelCase  )
																except ValueError:
																			lowerCAmelCase_      			=			None
																with xopen(_UpperCamelCase       ,       "rb"       ,       use_auth_token=_UpperCamelCase  ) as f:
																			lowerCAmelCase_      			=			BytesIO(f.read()  )
																lowerCAmelCase_      			=			PIL.Image.open(bytes_  )
							else:
										lowerCAmelCase_      			=			PIL.Image.open(BytesIO(bytes_  )  )
							image.load()  # to avoid "Too many open files" errors
							return image




				def   __a		(      self  )       ->   Union["FeatureType", Dict[str, "FeatureType"]]:
							from .features import Value

							return (
							    self
							    if self.decode
							    else {
							        "bytes": Value("binary"  ),
							        "path": Value("string"  ),
							    }
							)




				def   __a		(      self       ,       _UpperCamelCase  )       ->   pa.StructArray:
							if pa.types.is_string(storage.type  ):
										lowerCAmelCase_      			=			pa.array([None] * len(_UpperCamelCase  )       ,       type=pa.binary()  )
										lowerCAmelCase_      			=			pa.StructArray.from_arrays([bytes_array, storage]       ,       ["bytes", "path"]       ,       mask=storage.is_null()  )
							elif pa.types.is_binary(storage.type  ):
										lowerCAmelCase_      			=			pa.array([None] * len(_UpperCamelCase  )       ,       type=pa.string()  )
										lowerCAmelCase_      			=			pa.StructArray.from_arrays([storage, path_array]       ,       ["bytes", "path"]       ,       mask=storage.is_null()  )
							elif pa.types.is_struct(storage.type  ):
										if storage.type.get_field_index("bytes"  ) >= 0:
													lowerCAmelCase_      			=			storage.field("bytes"  )
										else:
													lowerCAmelCase_      			=			pa.array([None] * len(_UpperCamelCase  )       ,       type=pa.binary()  )
										if storage.type.get_field_index("path"  ) >= 0:
													lowerCAmelCase_      			=			storage.field("path"  )
										else:
													lowerCAmelCase_      			=			pa.array([None] * len(_UpperCamelCase  )       ,       type=pa.string()  )
										lowerCAmelCase_      			=			pa.StructArray.from_arrays([bytes_array, path_array]       ,       ["bytes", "path"]       ,       mask=storage.is_null()  )
							elif pa.types.is_list(storage.type  ):
										lowerCAmelCase_      			=			pa.array(
										    [encode_np_array(np.array(_UpperCamelCase  )  )["bytes"] if arr is not None else None for arr in storage.to_pylist()]       ,       type=pa.binary()       ,       )
										lowerCAmelCase_      			=			pa.array([None] * len(_UpperCamelCase  )       ,       type=pa.string()  )
										lowerCAmelCase_      			=			pa.StructArray.from_arrays(
										    [bytes_array, path_array]       ,       ["bytes", "path"]       ,       mask=bytes_array.is_null()  )
							return array_cast(_UpperCamelCase       ,       self.pa_type  )




				def   __a		(      self       ,       _UpperCamelCase  )       ->   pa.StructArray:

							@no_op_if_value_is_null
							def path_to_bytes(_UpperCamelCase  ):
										with xopen(_UpperCamelCase       ,       "rb"  ) as f:
													lowerCAmelCase_      			=			f.read()
										return bytes_

							lowerCAmelCase_      			=			pa.array(
							    [
							        (path_to_bytes(x["path"]  ) if x["bytes"] is None else x["bytes"]) if x is not None else None
							        for x in storage.to_pylist()
							    ]       ,       type=pa.binary()       ,       )
							lowerCAmelCase_      			=			pa.array(
							    [os.path.basename(_UpperCamelCase  ) if path is not None else None for path in storage.field("path"  ).to_pylist()]       ,       type=pa.string()       ,       )
							lowerCAmelCase_      			=			pa.StructArray.from_arrays([bytes_array, path_array]       ,       ["bytes", "path"]       ,       mask=bytes_array.is_null()  )
							return array_cast(_UpperCamelCase       ,       self.pa_type  )






def      lowerCamelCase__       (       ):




			"""simple docstring"""


			if config.PIL_AVAILABLE:
						import PIL.Image
			else:
						raise ImportError("To support encoding images, please install 'Pillow'."       )

			global _IMAGE_COMPRESSION_FORMATS
			if _IMAGE_COMPRESSION_FORMATS is None:
						PIL.Image.init()
						lowerCAmelCase_      			=			list(set(PIL.Image.OPEN.keys()       ) & set(PIL.Image.SAVE.keys()       )       )
			return _IMAGE_COMPRESSION_FORMATS






def      lowerCamelCase__       (       __lowerCAmelCase						:						"PIL.Image.Image"       ):




			"""simple docstring"""


			lowerCAmelCase_      			=			BytesIO()
			if image.format in list_image_compression_formats():
						lowerCAmelCase_      			=			image.format
			else:
						lowerCAmelCase_      			=			"PNG" if image.mode in ["1", "L", "LA", "RGB", "RGBA"] else "TIFF"
			image.save(__lowerCAmelCase					,							format=__lowerCAmelCase       )
			return buffer.getvalue()






def      lowerCamelCase__       (       __lowerCAmelCase						:						"PIL.Image.Image"       ):




			"""simple docstring"""


			if hasattr(__lowerCAmelCase					,							"filename"       ) and image.filename != "":
						return {"path": image.filename, "bytes": None}
			else:
						return {"path": None, "bytes": image_to_bytes(__lowerCAmelCase       )}






def      lowerCamelCase__       (       __lowerCAmelCase						:						np.ndarray       ):




			"""simple docstring"""


			if config.PIL_AVAILABLE:
						import PIL.Image
			else:
						raise ImportError("To support encoding images, please install 'Pillow'."       )

			lowerCAmelCase_      			=			array.dtype
			lowerCAmelCase_      			=			dtype.byteorder if dtype.byteorder != "=" else _NATIVE_BYTEORDER
			lowerCAmelCase_      			=			dtype.kind
			lowerCAmelCase_      			=			dtype.itemsize

			lowerCAmelCase_      			=			None

			# Multi-channel array case (only np.dtype("|u1") is allowed)
			if array.shape[2:]:
						lowerCAmelCase_      			=			np.dtype("|u1"       )
						if dtype_kind not in ["u", "i"]:
									raise TypeError(
									    F"""Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."""       )
						if dtype is not dest_dtype:
									warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"""       )
    # Exact match
			elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
						lowerCAmelCase_      			=			dtype
			else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
						while dtype_itemsize >= 1:
									lowerCAmelCase_      			=			dtype_byteorder + dtype_kind + str(__lowerCAmelCase       )
									lowerCAmelCase_      			=			np.dtype(__lowerCAmelCase       )
									if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
												warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"""       )
												break
									else:
												dtype_itemsize //= 2

			if dest_dtype is None:
						raise TypeError(
						    F"""Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"""       )
			lowerCAmelCase_      			=			PIL.Image.fromarray(array.astype(__lowerCAmelCase       )       )
			return {"path": None, "bytes": image_to_bytes(__lowerCAmelCase       )}






def      lowerCamelCase__       (       __lowerCAmelCase						:						Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]]       ):




			"""simple docstring"""


			if config.PIL_AVAILABLE:
						import PIL.Image
			else:
						raise ImportError("To support encoding images, please install 'Pillow'."       )

			if objs:
						lowerCAmelCase_			,						lowerCAmelCase_      			=			first_non_null_value(__lowerCAmelCase       )
						if isinstance(__lowerCAmelCase					,							__lowerCAmelCase       ):
									return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
						if isinstance(__lowerCAmelCase					,							np.ndarray       ):
									lowerCAmelCase_      			=			no_op_if_value_is_null(__lowerCAmelCase       )
									return [obj_to_image_dict_func(__lowerCAmelCase       ) for obj in objs]
						elif isinstance(__lowerCAmelCase					,							PIL.Image.Image       ):
									lowerCAmelCase_      			=			no_op_if_value_is_null(__lowerCAmelCase       )
									return [obj_to_image_dict_func(__lowerCAmelCase       ) for obj in objs]
						else:
									return objs
			else:
						return objs





import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
			from test_module.custom_tokenization_fast import CustomTokenizerFast





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   str:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			BertTokenizer.from_pretrained("hf-internal-testing/tiny-random-bert"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				@require_tokenizers
				def   __a		(      self  )       ->   int:
							# A mock response for an HTTP head request to emulate server down
							lowerCAmelCase_      			=			mock.Mock()
							lowerCAmelCase_      			=			500
							lowerCAmelCase_      			=			{}
							lowerCAmelCase_      			=			HTTPError
							lowerCAmelCase_      			=			{}

							# Download this model to make sure it's in the cache.
							lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )

							# Under the mock environment we get a 500 error when trying to reach the tokenizer.
							with mock.patch("requests.Session.request"       ,       return_value=_UpperCamelCase  ) as mock_head:
										lowerCAmelCase_      			=			GPTaTokenizerFast.from_pretrained("gpt2"  )
										# This check we did call the fake head request
										mock_head.assert_called()




				def   __a		(      self  )       ->   List[Any]:
							# This test is for deprecated behavior and can be removed in v5
							try:
										lowerCAmelCase_      			=			tempfile.mktemp()
										with open(_UpperCamelCase       ,       "wb"  ) as f:
													http_get("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"       ,       _UpperCamelCase  )

										lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained(_UpperCamelCase  )
							finally:
										os.remove(_UpperCamelCase  )

							# Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
							# the current folder and have the right name.
							if os.path.isfile("tokenizer.json"  ):
										# We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
										return
							try:
										with open("tokenizer.json"       ,       "wb"  ) as f:
													http_get("https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json"       ,       _UpperCamelCase  )
										lowerCAmelCase_      			=			AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-gpt2"  )
										# The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
										self.assertEqual(tokenizer.vocab_size       ,       1_000  )
										# Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

							finally:
										os.remove("tokenizer.json"  )




				def   __a		(      self  )       ->   Union[str, Any]:
							# This test is for deprecated behavior and can be removed in v5
							lowerCAmelCase_      			=			AlbertTokenizer.from_pretrained("https://huggingface.co/albert-base-v1/resolve/main/spiece.model"  )





@is_staging_test
class 			_lowerCAmelCase      (			unittest.TestCase ):
				_lowercase    	=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							lowerCAmelCase_      			=			TOKEN
							HfFolder.save_token(_UpperCamelCase  )




				@classmethod
				def   __a		(      cls  )       ->   Tuple:
							try:
										delete_repo(token=cls._token       ,       repo_id="test-tokenizer"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="valid_org/test-tokenizer-org"  )
							except HTTPError:
										pass

							try:
										delete_repo(token=cls._token       ,       repo_id="test-dynamic-tokenizer"  )
							except HTTPError:
										pass




				def   __a		(      self  )       ->   int:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("test-tokenizer"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="test-tokenizer"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(_UpperCamelCase       ,       repo_id="test-tokenizer"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained(f"""{USER}/test-tokenizer"""  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				def   __a		(      self  )       ->   Optional[int]:
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			BertTokenizer(_UpperCamelCase  )

							tokenizer.push_to_hub("valid_org/test-tokenizer-org"       ,       use_auth_token=self._token  )
							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )

							# Reset repo
							delete_repo(token=self._token       ,       repo_id="valid_org/test-tokenizer-org"  )

							# Push to hub via save_pretrained
							with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(
										    _UpperCamelCase       ,       repo_id="valid_org/test-tokenizer-org"       ,       push_to_hub=_UpperCamelCase       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			BertTokenizer.from_pretrained("valid_org/test-tokenizer-org"  )
							self.assertDictEqual(new_tokenizer.vocab       ,       tokenizer.vocab  )




				@require_tokenizers
				def   __a		(      self  )       ->   Optional[Any]:
							CustomTokenizer.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )
										lowerCAmelCase_      			=			CustomTokenizer(_UpperCamelCase  )

							# No fast custom tokenizer
							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )

							# Fast and slow custom tokenizer
							CustomTokenizerFast.register_for_auto_class()
							with tempfile.TemporaryDirectory() as tmp_dir:
										lowerCAmelCase_      			=			os.path.join(_UpperCamelCase       ,       "vocab.txt"  )
										with open(_UpperCamelCase       ,       "w"       ,       encoding="utf-8"  ) as vocab_writer:
													vocab_writer.write("".join([x + "\n" for x in self.vocab_tokens]  )  )

										lowerCAmelCase_      			=			BertTokenizerFast.from_pretrained(_UpperCamelCase  )
										bert_tokenizer.save_pretrained(_UpperCamelCase  )
										lowerCAmelCase_      			=			CustomTokenizerFast.from_pretrained(_UpperCamelCase  )

							tokenizer.push_to_hub("test-dynamic-tokenizer"       ,       use_auth_token=self._token  )

							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(f"""{USER}/test-dynamic-tokenizer"""       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizerFast"  )
							lowerCAmelCase_      			=			AutoTokenizer.from_pretrained(
							    f"""{USER}/test-dynamic-tokenizer"""       ,       use_fast=_UpperCamelCase       ,       trust_remote_code=_UpperCamelCase  )
							# Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
							self.assertEqual(tokenizer.__class__.__name__       ,       "CustomTokenizer"  )





class 			_lowerCAmelCase      (			unittest.TestCase ):




				def   __a		(      self  )       ->   Any:
							lowerCAmelCase_      			=			Trie()
							trie.add("Hello 友達"  )
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {" ": {"友": {"達": {"": 1}}}}}}}}}  )
							trie.add("Hello"  )
							trie.data
							self.assertEqual(trie.data       ,       {"H": {"e": {"l": {"l": {"o": {"": 1, " ": {"友": {"達": {"": 1}}}}}}}}}  )




				def   __a		(      self  )       ->   Optional[int]:
							lowerCAmelCase_      			=			Trie()
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS] This is a extra_id_100"]  )
							trie.add("[CLS]"  )
							trie.add("extra_id_1"  )
							trie.add("extra_id_100"  )
							self.assertEqual(trie.split("[CLS] This is a extra_id_100"  )       ,       ["[CLS]", " This is a ", "extra_id_100"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["A", "BC"]  )
							self.assertEqual(trie.split("BCA"  )       ,       ["BC", "A"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("TOKEN]"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   int:
							lowerCAmelCase_      			=			Trie()
							trie.add("A"  )
							trie.add("P"  )
							trie.add("[SPECIAL_TOKEN]"  )
							self.assertEqual(trie.split("This is something [SPECIAL_TOKEN]"  )       ,       ["This is something ", "[SPECIAL_TOKEN]"]  )




				def   __a		(      self  )       ->   str:
							lowerCAmelCase_      			=			Trie()
							trie.add("AB"  )
							trie.add("B"  )
							trie.add("C"  )
							self.assertEqual(trie.split("ABC"  )       ,       ["AB", "C"]  )




				def   __a		(      self  )       ->   List[Any]:
							lowerCAmelCase_      			=			Trie()
							trie.add("ABC"  )
							trie.add("B"  )
							trie.add("CD"  )
							self.assertEqual(trie.split("ABCD"  )       ,       ["ABC", "D"]  )




				def   __a		(      self  )       ->   List[Any]:
							# Even if the offsets are wrong, we necessarily output correct string
							# parts.
							lowerCAmelCase_      			=			Trie()
							lowerCAmelCase_      			=			trie.cut_text("ABC"       ,       [0, 0, 2, 1, 2, 3]  )
							self.assertEqual(_UpperCamelCase       ,       ["AB", "C"]  )



