

"""simple docstring"""







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_albert import AlbertTokenizer
else:
		a__       :							str        =  None

a__       :							List[str]        =  logging.get_logger(__name__)
a__       :							Any        =  {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

a__       :							Tuple        =  {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json""",
    },
}

a__       :							Optional[Any]        =  {
    """albert-base-v1""": 5_1_2,
    """albert-large-v1""": 5_1_2,
    """albert-xlarge-v1""": 5_1_2,
    """albert-xxlarge-v1""": 5_1_2,
    """albert-base-v2""": 5_1_2,
    """albert-large-v2""": 5_1_2,
    """albert-xlarge-v2""": 5_1_2,
    """albert-xxlarge-v2""": 5_1_2,
}

a__       :							Dict        =  """▁"""




class 				UpperCamelCase_			(				_snake_case):

	"""simple docstring"""





	snake_case__       :						Dict					      =    VOCAB_FILES_NAMES
	snake_case__       :						int					      =    PRETRAINED_VOCAB_FILES_MAP
	snake_case__       :						List[Any]					      =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case__       :						Tuple					      =    AlbertTokenizer




	def __init__(				self    :       Tuple    ,    UpperCAmelCase__    :       Tuple=None    ,    UpperCAmelCase__    :       Union[str, Any]=None    ,    UpperCAmelCase__    :       List[Any]=True    ,    UpperCAmelCase__    :       Any=True    ,    UpperCAmelCase__    :       List[Any]=False    ,    UpperCAmelCase__    :       List[str]="[CLS]"    ,    UpperCAmelCase__    :       Tuple="[SEP]"    ,    UpperCAmelCase__    :       List[str]="<unk>"    ,    UpperCAmelCase__    :       Any="[SEP]"    ,    UpperCAmelCase__    :       Any="<pad>"    ,    UpperCAmelCase__    :       str="[CLS]"    ,    UpperCAmelCase__    :       List[Any]="[MASK]"    ,    **UpperCAmelCase__    :       str    ,    ) ->			Any:
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						__SCREAMING_SNAKE_CASE		     =       (
						    AddedToken(snake_case_    ,    lstrip=snake_case_    ,    rstrip=snake_case_    ,    normalized=snake_case_     )
						    if isinstance(snake_case_    ,    snake_case_     )
						    else mask_token
						)

						super().__init__(
						    snake_case_    ,    tokenizer_file=snake_case_    ,    do_lower_case=snake_case_    ,    remove_space=snake_case_    ,    keep_accents=snake_case_    ,    bos_token=snake_case_    ,    eos_token=snake_case_    ,    unk_token=snake_case_    ,    sep_token=snake_case_    ,    pad_token=snake_case_    ,    cls_token=snake_case_    ,    mask_token=snake_case_    ,    **snake_case_    ,    )

						__SCREAMING_SNAKE_CASE		     =       do_lower_case
						__SCREAMING_SNAKE_CASE		     =       remove_space
						__SCREAMING_SNAKE_CASE		     =       keep_accents
						__SCREAMING_SNAKE_CASE		     =       vocab_file
						__SCREAMING_SNAKE_CASE		     =       False if not self.vocab_file else True




	def 						UpperCAmelCase_   (				self    :       Union[str, Any]    ,    UpperCAmelCase__    :       Any    ,    UpperCAmelCase__    :       Dict = None     ) ->			Tuple:
						__SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
						__SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]
						if token_ids_a is None:
											return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep




	def 						UpperCAmelCase_   (				self    :       str    ,    UpperCAmelCase__    :       Union[str, Any]    ,    UpperCAmelCase__    :       str = None     ) ->			List[str]:
						__SCREAMING_SNAKE_CASE		     =       [self.sep_token_id]
						__SCREAMING_SNAKE_CASE		     =       [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep     ) * [0]
						return len(cls + token_ids_a + sep     ) * [0] + len(token_ids_a + sep     ) * [1]




	def 						UpperCAmelCase_   (				self    :       int    ,    UpperCAmelCase__    :       int    ,    UpperCAmelCase__    :       Optional[int] = None     ) ->			Union[str, Any]:
						if not self.can_save_slow_tokenizer:
											raise ValueError(
											    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
											    "tokenizer."     )

						if not os.path.isdir(snake_case_     ):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""     )
											return
						__SCREAMING_SNAKE_CASE		     =       os.path.join(
						    snake_case_    ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )

						if os.path.abspath(self.vocab_file     ) != os.path.abspath(snake_case_     ):
											copyfile(self.vocab_file    ,    snake_case_     )

						return (out_vocab_file,)




"""simple docstring"""







import numpy as np
from matplotlib import pyplot as plt
from sklearn.datasets import load_iris
from sklearn.metrics import ConfusionMatrixDisplay
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier




def      UpperCAmelCase__ (lowerCAmelCase_     ):




					'''simple docstring'''


					return (data["data"], data["target"])




def      UpperCAmelCase__ (lowerCAmelCase_						,		lowerCAmelCase_     ):




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       XGBClassifier()
					classifier.fit(lowerCAmelCase_						,		lowerCAmelCase_     )
					return classifier




def      UpperCAmelCase__ ():




					'''simple docstring'''


					__SCREAMING_SNAKE_CASE		     =       load_iris()
					__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE		     =       data_handling(lowerCAmelCase_     )
					__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE     ,						__SCREAMING_SNAKE_CASE		     =       train_test_split(
					    lowerCAmelCase_						,		lowerCAmelCase_						,		test_size=0.25     )

					__SCREAMING_SNAKE_CASE		     =       iris["target_names"]

					# Create an XGBoost Classifier from the training data
					__SCREAMING_SNAKE_CASE		     =       xgboost(lowerCAmelCase_						,		lowerCAmelCase_     )

					# Display the confusion matrix of the classifier with both training and test sets
					ConfusionMatrixDisplay.from_estimator(
					    lowerCAmelCase_						,		lowerCAmelCase_						,		lowerCAmelCase_						,		display_labels=lowerCAmelCase_						,		cmap="Blues"						,		normalize="true"						,		)
					plt.title("Normalized Confusion Matrix - IRIS Dataset"     )
					plt.show()


if __name__ == "__main__":
		import doctest

		doctest.testmod(verbose=True)
		main()
