"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_UpperCamelCase								=		logging.get_logger(__name__)


_UpperCamelCase								=		{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all BART models at https://huggingface.co/models?filter=bart
_UpperCamelCase								=		{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json",
    },
}

_UpperCamelCase								=		{
    "facebook/bart-base": 10_24,
    "facebook/bart-large": 10_24,
    "facebook/bart-large-mnli": 10_24,
    "facebook/bart-large-cnn": 10_24,
    "facebook/bart-large-xsum": 10_24,
    "yjernite/bart_eli5": 10_24,
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Dict					     =				VOCAB_FILES_NAMES
			_UpperCamelCase  :						Optional[int]					     =				PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase  :						List[str]					     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase  :						Optional[Any]					     =				['input_ids', 'attention_mask']
			_UpperCamelCase  :						str					     =				BartTokenizer





			def __init__(			self   ,      snake_case_=None   ,      snake_case_=None   ,      snake_case_=None   ,      snake_case_="replace"   ,      snake_case_="<s>"   ,      snake_case_="</s>"   ,      snake_case_="</s>"   ,      snake_case_="<s>"   ,      snake_case_="<unk>"   ,      snake_case_="<pad>"   ,      snake_case_="<mask>"   ,      snake_case_=False   ,      snake_case_=True   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(
										    snake_case_   ,      snake_case_   ,      tokenizer_file=snake_case_   ,      errors=snake_case_   ,      bos_token=snake_case_   ,      eos_token=snake_case_   ,      sep_token=snake_case_   ,      cls_token=snake_case_   ,      unk_token=snake_case_   ,      pad_token=snake_case_   ,      mask_token=snake_case_   ,      add_prefix_space=snake_case_   ,      trim_offsets=snake_case_   ,      **snake_case_   ,      )

										A__       :     Optional[int]								=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
										if pre_tok_state.get("""add_prefix_space"""   ,      snake_case_							) != add_prefix_space:
																	A__       :     List[Any]								=			getattr(snake_case_   ,      pre_tok_state.pop("""type"""							)							)
																	A__       :     Optional[int]								=			add_prefix_space
																	A__       :     Tuple								=			pre_tok_class(**snake_case_							)

										A__       :     Dict								=			add_prefix_space

										# the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
										A__       :     Union[str, Any]								=			"""post_processor"""
										A__       :     Union[str, Any]								=			getattr(self.backend_tokenizer   ,      snake_case_   ,      snake_case_							)
										if tokenizer_component_instance:
																	A__       :     Dict								=			json.loads(tokenizer_component_instance.__getstate__()							)

																	# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
																	if "sep" in state:
																								A__       :     str								=			tuple(state["""sep"""]							)
																	if "cls" in state:
																								A__       :     Union[str, Any]								=			tuple(state["""cls"""]							)

																	A__       :     Optional[Any]								=			False

																	if state.get("""add_prefix_space"""   ,      snake_case_							) != add_prefix_space:
																								A__       :     List[Any]								=			add_prefix_space
																								A__       :     Optional[int]								=			True

																	if state.get("""trim_offsets"""   ,      snake_case_							) != trim_offsets:
																								A__       :     Optional[Any]								=			trim_offsets
																								A__       :     Any								=			True

																	if changes_to_apply:
																								A__       :     Union[str, Any]								=			getattr(snake_case_   ,      state.pop("""type"""							)							)
																								A__       :     Union[str, Any]								=			component_class(**snake_case_							)
																								setattr(self.backend_tokenizer   ,      snake_case_   ,      snake_case_							)





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										if self._mask_token is None:
																	if self.verbose:
																								logger.error("""Using mask_token, but it is not set yet."""							)
																	return None
										return str(self._mask_token							)





			@mask_token.setter
			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			AddedToken(snake_case_   ,      lstrip=snake_case_   ,      rstrip=snake_case_							) if isinstance(snake_case_   ,      snake_case_							) else value
										A__       :     List[str]								=			value





			def 		lowerCamelCase					(			self   ,      *snake_case_   ,      **snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			kwargs.get("""is_split_into_words"""   ,      snake_case_							)

										if is_split_into_words and not self.add_prefix_space:
																	raise ValueError(
																	    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
																	    """to use it with pretokenized inputs."""							)

										return super()._batch_encode_plus(*snake_case_   ,      **snake_case_							)





			def 		lowerCamelCase					(			self   ,      *snake_case_   ,      **snake_case_							):







										'''simple docstring'''


										A__       :     Dict								=			kwargs.get("""is_split_into_words"""   ,      snake_case_							)

										if is_split_into_words and not self.add_prefix_space:
																	raise ValueError(
																	    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
																	    """to use it with pretokenized inputs."""							)

										return super()._encode_plus(*snake_case_   ,      **snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     Union[str, Any]								=			self._tokenizer.model.save(snake_case_   ,      name=snake_case_							)
										return tuple(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_=None							):







										'''simple docstring'''


										A__       :     Dict								=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
																	return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     Any								=			[self.sep_token_id]
										A__       :     Optional[Any]								=			[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep							) * [0]






"""simple docstring"""



import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin
class 						__UpperCAmelCase    (unittest.TestCase							,   __A     ):







			'''simple docstring'''





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     List[Any]								=			load_tool("""text-classification"""							)
										self.tool.setup()
										A__       :     Any								=			load_tool("""text-classification"""   ,      remote=snake_case_							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     int								=			self.tool("""That's quite cool"""   ,      ["""positive""", """negative"""]							)
										self.assertEqual(snake_case_   ,      """positive"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			self.remote_tool("""That's quite cool"""   ,      ["""positive""", """negative"""]							)
										self.assertEqual(snake_case_   ,      """positive"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     str								=			self.tool(text="""That's quite cool"""   ,      labels=["""positive""", """negative"""]							)
										self.assertEqual(snake_case_   ,      """positive"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     int								=			self.remote_tool(text="""That's quite cool"""   ,      labels=["""positive""", """negative"""]							)
										self.assertEqual(snake_case_   ,      """positive"""							)





