




'''simple docstring'''


import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case   :   str     							=	logging.get_logger(__name__)

_snake_case   :   Any     							=	'▁'

_snake_case   :   List[str]     							=	{'vocab_file': 'sentencepiece.bpe.model'}

_snake_case   :   Tuple     							=	{
    'vocab_file': {
        'facebook/xglm-564M': 'https://huggingface.co/facebook/xglm-564M/resolve/main/sentencepiece.bpe.model',
    }
}

_snake_case   :   Any     							=	{
    'facebook/xglm-564M': 2048,
}

class     A					(	_a					):
							lowercase_           =       VOCAB_FILES_NAMES
							lowercase_           =       PRETRAINED_VOCAB_FILES_MAP
							lowercase_           =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_           =       ['input_ids', 'attention_mask']







							def __init__(						self :						int							,    lowerCAmelCase_ :						Dict							,    lowerCAmelCase_ :						Dict="<s>"							,    lowerCAmelCase_ :						Optional[int]="</s>"							,    lowerCAmelCase_ :						List[Any]="</s>"							,    lowerCAmelCase_ :						str="<s>"							,    lowerCAmelCase_ :						Tuple="<unk>"							,    lowerCAmelCase_ :						int="<pad>"							,    lowerCAmelCase_ :						Optional[Dict[str, Any]] = None							,    **lowerCAmelCase_ :						Tuple							,    )    ->							None:


														"""simple docstring"""


														_a			  =       {} if sp_model_kwargs is None else sp_model_kwargs

														# Compatibility with the original tokenizer
														_a			  =       7
														_a			  =       [F'<madeupword{i}>' for i in range(self.num_madeup_words      )]

														_a			  =       kwargs.get('''additional_special_tokens'''							,    []      )
														kwargs["additional_special_tokens"] += [
														    word for word in madeup_words if word not in kwargs["additional_special_tokens"]
														]

														super().__init__(
														    bos_token=lowerCAmelCase_							,    eos_token=lowerCAmelCase_							,    unk_token=lowerCAmelCase_							,    sep_token=lowerCAmelCase_							,    cls_token=lowerCAmelCase_							,    pad_token=lowerCAmelCase_							,    sp_model_kwargs=self.sp_model_kwargs							,    **lowerCAmelCase_							,    )

														_a			  =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
														self.sp_model.Load(str(lowerCAmelCase_      )      )
														_a			  =       vocab_file

														# Original fairseq vocab and spm vocab must be "aligned":
														# Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
														# -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
														# fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
														# spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

														# The first "real" token "," has position 4 in the original fairseq vocab and position 3 in the spm vocab
														_a			  =       1

														# Mimic fairseq token-to-id alignment for the first 4 token
														_a			  =       {'''<s>''': 0, '''<pad>''': 1, '''</s>''': 2, '''<unk>''': 3}

														_a			  =       len(self.sp_model      )
														_a			  =       {F'<madeupword{i}>': sp_size + i + self.fairseq_offset for i in range(self.num_madeup_words      )}
														self.fairseq_tokens_to_ids.update(lowerCAmelCase_      )

														_a			  =       {v: k for k, v in self.fairseq_tokens_to_ids.items()}







							def __getstate__(						self :						Any      )    ->							Dict:


														"""simple docstring"""


														_a			  =       self.__dict__.copy()
														_a			  =       None
														_a			  =       self.sp_model.serialized_model_proto()
														return state







							def __setstate__(						self :						Any							,    lowerCAmelCase_ :						Optional[int]      )    ->							Optional[Any]:


														"""simple docstring"""


														_a			  =       d

														# for backward compatibility
														if not hasattr(self							,    '''sp_model_kwargs'''      ):
																					_a			  =       {}

														_a			  =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
														self.sp_model.LoadFromSerializedProto(self.sp_model_proto      )







							def      __lowerCAmelCase    (						self :						str							,    lowerCAmelCase_ :						List[int]							,    lowerCAmelCase_ :						Optional[List[int]] = None      )    ->							List[int]:


														"""simple docstring"""


														if token_ids_a is None:
																					return [self.sep_token_id] + token_ids_a
														_a			  =       [self.sep_token_id]
														return sep + token_ids_a + sep + sep + token_ids_a







							def      __lowerCAmelCase    (						self :						Union[str, Any]							,    lowerCAmelCase_ :						List[int]							,    lowerCAmelCase_ :						Optional[List[int]] = None							,    lowerCAmelCase_ :						bool = False      )    ->							List[int]:


														"""simple docstring"""


														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=lowerCAmelCase_							,    token_ids_a=lowerCAmelCase_							,    already_has_special_tokens=lowerCAmelCase_      )

														if token_ids_a is None:
																					return [1] + ([0] * len(lowerCAmelCase_      ))
														return [1] + ([0] * len(lowerCAmelCase_      )) + [1, 1] + ([0] * len(lowerCAmelCase_      ))







							def      __lowerCAmelCase    (						self :						List[Any]							,    lowerCAmelCase_ :						List[int]							,    lowerCAmelCase_ :						Optional[List[int]] = None      )    ->							List[int]:


														"""simple docstring"""


														_a			  =       [self.sep_token_id]

														if token_ids_a is None:
																					return len(sep + token_ids_a      ) * [0]
														return len(sep + token_ids_a + sep + sep + token_ids_a      ) * [0]







							@property
							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							str:


														"""simple docstring"""


														return len(self.sp_model      ) + self.fairseq_offset + self.num_madeup_words







							def      __lowerCAmelCase    (						self :						Dict      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       {self.convert_ids_to_tokens(lowerCAmelCase_      ): i for i in range(self.vocab_size      )}
														vocab.update(self.added_tokens_encoder      )
														return vocab







							def      __lowerCAmelCase    (						self :						Dict							,    lowerCAmelCase_ :						str      )    ->							List[str]:


														"""simple docstring"""


														return self.sp_model.encode(lowerCAmelCase_							,    out_type=lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						int							,    lowerCAmelCase_ :						Tuple      )    ->							List[Any]:


														"""simple docstring"""


														if token in self.fairseq_tokens_to_ids:
																					return self.fairseq_tokens_to_ids[token]
														_a			  =       self.sp_model.PieceToId(lowerCAmelCase_      )

														# Need to return unknown token if the SP model returned 0
														return spm_id + self.fairseq_offset if spm_id else self.unk_token_id







							def      __lowerCAmelCase    (						self :						Optional[int]							,    lowerCAmelCase_ :						int      )    ->							Tuple:


														"""simple docstring"""


														if index in self.fairseq_ids_to_tokens:
																					return self.fairseq_ids_to_tokens[index]
														return self.sp_model.IdToPiece(index - self.fairseq_offset      )







							def      __lowerCAmelCase    (						self :						Optional[int]							,    lowerCAmelCase_ :						str      )    ->							Tuple:


														"""simple docstring"""


														_a			  =       ''''''.join(lowerCAmelCase_      ).replace(lowerCAmelCase_							,    ''' '''      ).strip()
														return out_string







							def      __lowerCAmelCase    (						self :						Optional[Any]							,    lowerCAmelCase_ :						str							,    lowerCAmelCase_ :						Optional[str] = None      )    ->							Tuple[str]:


														"""simple docstring"""


														if not os.path.isdir(lowerCAmelCase_      ):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory'      )
																					return
														_a			  =       os.path.join(
														    lowerCAmelCase_							,    (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

														if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowerCAmelCase_      ) and os.path.isfile(self.vocab_file      ):
																					copyfile(self.vocab_file							,    lowerCAmelCase_      )
														elif not os.path.isfile(self.vocab_file      ):
																					with open(lowerCAmelCase_							,    '''wb'''      ) as fi:
																												_a			  =       self.sp_model.serialized_model_proto()
																												fi.write(lowerCAmelCase_      )

														return (out_vocab_file,)







'''simple docstring'''


import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


_snake_case   :   str     							=	'\nHugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.\n\nIn March 2021, Hugging Face raised $40 million in a Series B funding round.[3]\n\nOn April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]\n'

class     A					(	unittest.TestCase  ,_a					):







							def      __lowerCAmelCase    (						self :						Any      )    ->							Dict:


														"""simple docstring"""


														_a			  =       load_tool('''text-question-answering'''      )
														self.tool.setup()
														_a			  =       load_tool('''text-question-answering'''							,    remote=lowerCAmelCase_      )







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							Optional[int]:


														"""simple docstring"""


														_a			  =       self.tool(lowerCAmelCase_							,    '''What did Hugging Face do in April 2021?'''      )
														self.assertEqual(lowerCAmelCase_							,    '''launched the BigScience Research Workshop'''      )







							def      __lowerCAmelCase    (						self :						Union[str, Any]      )    ->							Dict:


														"""simple docstring"""


														_a			  =       self.remote_tool(lowerCAmelCase_							,    '''What did Hugging Face do in April 2021?'''      )
														self.assertEqual(lowerCAmelCase_							,    '''launched the BigScience Research Workshop'''      )







							def      __lowerCAmelCase    (						self :						List[Any]      )    ->							List[str]:


														"""simple docstring"""


														_a			  =       self.tool(text=lowerCAmelCase_							,    question='''What did Hugging Face do in April 2021?'''      )
														self.assertEqual(lowerCAmelCase_							,    '''launched the BigScience Research Workshop'''      )







							def      __lowerCAmelCase    (						self :						int      )    ->							List[Any]:


														"""simple docstring"""


														_a			  =       self.remote_tool(text=lowerCAmelCase_							,    question='''What did Hugging Face do in April 2021?'''      )
														self.assertEqual(lowerCAmelCase_							,    '''launched the BigScience Research Workshop'''      )

