






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__							=     logging.get_logger(__name__)

lowerCamelCase__							=     {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCamelCase__							=     {
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCamelCase__							=     {
    """ctrl""": 256,
}

lowerCamelCase__							=     {
    """Pregnancy""": 16_8629,
    """Christianity""": 7675,
    """Explain""": 10_6423,
    """Fitness""": 6_3440,
    """Saving""": 6_3163,
    """Ask""": 2_7171,
    """Ass""": 9_5985,
    """Joke""": 16_3509,
    """Questions""": 4_5622,
    """Thoughts""": 4_9605,
    """Retail""": 5_2342,
    """Feminism""": 16_4338,
    """Writing""": 1_1992,
    """Atheism""": 19_2263,
    """Netflix""": 4_8616,
    """Computing""": 3_9639,
    """Opinion""": 4_3213,
    """Alone""": 4_4967,
    """Funny""": 5_8917,
    """Gaming""": 4_0358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 7_7138,
    """Diet""": 3_6206,
    """Legal""": 1_1859,
    """Norman""": 4939,
    """Tip""": 7_2689,
    """Weight""": 5_2343,
    """Movies""": 4_6273,
    """Running""": 2_3425,
    """Science""": 2090,
    """Horror""": 3_7793,
    """Confession""": 6_0572,
    """Finance""": 1_2250,
    """Politics""": 1_6360,
    """Scary""": 19_1985,
    """Support""": 1_2654,
    """Technologies""": 3_2516,
    """Teenage""": 6_6160,
    """Event""": 3_2769,
    """Learned""": 6_7460,
    """Notion""": 18_2770,
    """Wikipedia""": 3_7583,
    """Books""": 6665,
    """Extract""": 7_6050,
    """Confessions""": 10_2701,
    """Conspiracy""": 7_5932,
    """Links""": 6_3674,
    """Narcissus""": 15_0425,
    """Relationship""": 5_4766,
    """Relationships""": 13_4796,
    """Reviews""": 4_1671,
    """News""": 4256,
    """Translation""": 2_6820,
    """multilingual""": 12_8406,
}






def   UpperCamelCase  (       snake_case__     :   Optional[Any]  ):




     '''simple docstring'''






     __snake_case			:Optional[Any]          =						set()
     __snake_case			:int          =						word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)  )
          __snake_case			:int          =						char

     __snake_case			:int          =						set(UpperCamelCase__  )
     return pairs




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= VOCAB_FILES_NAMES
       lowerCamelCase							:				Tuple 				= PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase							:				Union[str, Any] 				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCamelCase							:				Tuple 				= CONTROL_CODES



       def __init__(							self			,						a__			,						a__			,						a__="<unk>"			,						**a__    ) ->      Any:






            '''simple docstring'''





            super().__init__(unk_token=a__			,						**a__    )

            with open(a__			,						encoding="""utf-8"""    ) as vocab_handle:
                 __snake_case			:str          =						json.load(a__    )
            __snake_case			:Dict          =						{v: k for k, v in self.encoder.items()}
            with open(a__			,						encoding="""utf-8"""    ) as merges_handle:
                 __snake_case			:str          =						merges_handle.read().split("""\n"""    )[1:-1]
            __snake_case			:Dict          =						[tuple(merge.split()    ) for merge in merges]
            __snake_case			:Tuple          =						dict(zip(a__			,						range(len(a__    )    )    )    )
            __snake_case			:Any          =						{}



       @property
       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            return len(self.encoder    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            return dict(self.encoder			,						**self.added_tokens_encoder    )



       def 					__lowercase						(							self			,						a__    ) ->      Tuple:






            '''simple docstring'''





            if token in self.cache:
                 return self.cache[token]
            __snake_case			:str          =						tuple(a__    )
            __snake_case			:List[Any]          =						tuple(list(word[:-1]    ) + [word[-1] + """</w>"""]    )
            __snake_case			:Union[str, Any]          =						get_pairs(a__    )

            if not pairs:
                 return token

            while True:
                 __snake_case			:str          =						min(a__			,						key=lambda a__    : self.bpe_ranks.get(a__			,						float("""inf"""    )    )    )
                 if bigram not in self.bpe_ranks:
                      break
                 __snake_case       , __snake_case			:Optional[Any]          =						bigram
                 __snake_case			:List[Any]          =						[]
                 __snake_case			:Dict          =						0
                 while i < len(a__    ):
                      try:
                           __snake_case			:Optional[int]          =						word.index(a__			,						a__    )
                      except ValueError:
                           new_word.extend(word[i:]    )
                           break
                      else:
                           new_word.extend(word[i:j]    )
                           __snake_case			:List[str]          =						j

                      if word[i] == first and i < len(a__    ) - 1 and word[i + 1] == second:
                           new_word.append(first + second    )
                           i += 2
                      else:
                           new_word.append(word[i]    )
                           i += 1
                 __snake_case			:int          =						tuple(a__    )
                 __snake_case			:Union[str, Any]          =						new_word
                 if len(a__    ) == 1:
                      break
                 else:
                      __snake_case			:Dict          =						get_pairs(a__    )
            __snake_case			:Any          =						"""@@ """.join(a__    )
            __snake_case			:List[Any]          =						word[:-4]
            __snake_case			:Optional[int]          =						word
            return word



       def 					__lowercase						(							self			,						a__    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Tuple          =						[]

            __snake_case			:Optional[Any]          =						re.findall(R"""\S+\n?"""			,						a__    )

            for token in words:
                 split_tokens.extend(list(self.bpe(a__    ).split(""" """    )    )    )
            return split_tokens



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            return self.encoder.get(a__			,						self.encoder.get(self.unk_token    )    )



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            return self.decoder.get(a__			,						self.unk_token    )



       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:List[str]          =						""" """.join(a__    ).replace("""@@ """			,						""""""    ).strip()
            return out_string



       def 					__lowercase						(							self			,						a__			,						a__ = None    ) ->      Union[str, Any]:






            '''simple docstring'''





            if not os.path.isdir(a__    ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
                 return
            __snake_case			:List[Any]          =						os.path.join(
                a__			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )
            __snake_case			:str          =						os.path.join(
                a__			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]    )

            with open(a__			,						"""w"""			,						encoding="""utf-8"""    ) as f:
                 f.write(json.dumps(self.encoder			,						indent=2			,						sort_keys=a__			,						ensure_ascii=a__    ) + """\n"""    )

            __snake_case			:str          =						0
            with open(a__			,						"""w"""			,						encoding="""utf-8"""    ) as writer:
                 writer.write("""#version: 0.2\n"""    )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,						key=lambda a__    : kv[1]    ):
                      if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               """ Please check that the tokenizer is not corrupted!"""    )
                           __snake_case			:Dict          =						token_index
                      writer.write(""" """.join(a__    ) + """\n"""    )
                      index += 1

            return vocab_file, merge_file

       # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
       #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
       #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
       #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
       #     return ''.join(tokens_generated_so_far)










import unittest

from transformers import load_tool

from .test_tools_common import ToolTesterMixin


lowerCamelCase__							=     """
Hugging Face was founded in 2016 by French entrepreneurs Clément Delangue, Julien Chaumond, and Thomas Wolf originally as a company that developed a chatbot app targeted at teenagers.[2] After open-sourcing the model behind the chatbot, the company pivoted to focus on being a platform for machine learning.

In March 2021, Hugging Face raised $40 million in a Series B funding round.[3]

On April 28, 2021, the company launched the BigScience Research Workshop in collaboration with several other research groups to release an open large language model.[4] In 2022, the workshop concluded with the announcement of BLOOM, a multilingual large language model with 176 billion parameters.[5]
"""




class 			snake_case__   (							unittest.TestCase	,    lowercase_):

       '''simple docstring'''



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:int          =						load_tool("""text-question-answering"""    )
            self.tool.setup()
            __snake_case			:Tuple          =						load_tool("""text-question-answering"""			,						remote=a__    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tool(a__			,						"""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.remote_tool(a__			,						"""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Dict:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.tool(text=a__			,						question="""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.remote_tool(text=a__			,						question="""What did Hugging Face do in April 2021?"""    )
            self.assertEqual(a__			,						"""launched the BigScience Research Workshop"""    )


