






import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__       		=  logging.get_logger(__name__)

lowerCamelCase__       		=  {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

lowerCamelCase__       		=  {
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

lowerCamelCase__       		=  {
    """ctrl""": 256,
}

lowerCamelCase__       		=  {
    """Pregnancy""": 16_8629,
    """Christianity""": 7675,
    """Explain""": 10_6423,
    """Fitness""": 6_3440,
    """Saving""": 6_3163,
    """Ask""": 2_7171,
    """Ass""": 9_5985,
    """Joke""": 16_3509,
    """Questions""": 4_5622,
    """Thoughts""": 4_9605,
    """Retail""": 5_2342,
    """Feminism""": 16_4338,
    """Writing""": 1_1992,
    """Atheism""": 19_2263,
    """Netflix""": 4_8616,
    """Computing""": 3_9639,
    """Opinion""": 4_3213,
    """Alone""": 4_4967,
    """Funny""": 5_8917,
    """Gaming""": 4_0358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 7_7138,
    """Diet""": 3_6206,
    """Legal""": 1_1859,
    """Norman""": 4939,
    """Tip""": 7_2689,
    """Weight""": 5_2343,
    """Movies""": 4_6273,
    """Running""": 2_3425,
    """Science""": 2090,
    """Horror""": 3_7793,
    """Confession""": 6_0572,
    """Finance""": 1_2250,
    """Politics""": 1_6360,
    """Scary""": 19_1985,
    """Support""": 1_2654,
    """Technologies""": 3_2516,
    """Teenage""": 6_6160,
    """Event""": 3_2769,
    """Learned""": 6_7460,
    """Notion""": 18_2770,
    """Wikipedia""": 3_7583,
    """Books""": 6665,
    """Extract""": 7_6050,
    """Confessions""": 10_2701,
    """Conspiracy""": 7_5932,
    """Links""": 6_3674,
    """Narcissus""": 15_0425,
    """Relationship""": 5_4766,
    """Relationships""": 13_4796,
    """Reviews""": 4_1671,
    """News""": 4256,
    """Translation""": 2_6820,
    """multilingual""": 12_8406,
}






def 	UpperCamelCase	(							snake_case__  : Any	):




     '''simple docstring'''






     __snake_case      :List[Any] 							=    set()
     __snake_case      :Optional[Any] 							=    word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)	)
          __snake_case      :Tuple 							=    char

     __snake_case      :Optional[Any] 							=    set(snake_case__	)
     return pairs




class       snake_case__ (				lowercase_):

       '''simple docstring'''
       lowerCamelCase				:					int								=			VOCAB_FILES_NAMES
       lowerCamelCase				:					Optional[int]								=			PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase				:					List[Any]								=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCamelCase				:					Union[str, Any]								=			CONTROL_CODES



       def __init__(				self      ,    a__      ,    a__      ,    a__="<unk>"      ,    **a__     )			->   Optional[int]:






            '''simple docstring'''





            super().__init__(unk_token=a__      ,    **a__     )

            with open(a__      ,    encoding="""utf-8"""     ) as vocab_handle:
                 __snake_case      :Tuple 							=    json.load(a__     )
            __snake_case      :Any 							=    {v: k for k, v in self.encoder.items()}
            with open(a__      ,    encoding="""utf-8"""     ) as merges_handle:
                 __snake_case      :List[str] 							=    merges_handle.read().split("""\n"""     )[1:-1]
            __snake_case      :Union[str, Any] 							=    [tuple(merge.split()     ) for merge in merges]
            __snake_case      :str 							=    dict(zip(a__      ,    range(len(a__     )     )     )     )
            __snake_case      :Optional[int] 							=    {}



       @property
       def        __lowercase    (				self     )			->   int:






            '''simple docstring'''





            return len(self.encoder     )



       def        __lowercase    (				self     )			->   Tuple:






            '''simple docstring'''





            return dict(self.encoder      ,    **self.added_tokens_encoder     )



       def        __lowercase    (				self      ,    a__     )			->   Dict:






            '''simple docstring'''





            if token in self.cache:
                 return self.cache[token]
            __snake_case      :Tuple 							=    tuple(a__     )
            __snake_case      :Optional[Any] 							=    tuple(list(word[:-1]     ) + [word[-1] + """</w>"""]     )
            __snake_case      :Union[str, Any] 							=    get_pairs(a__     )

            if not pairs:
                 return token

            while True:
                 __snake_case      :List[Any] 							=    min(a__      ,    key=lambda a__     : self.bpe_ranks.get(a__      ,    float("""inf"""     )     )     )
                 if bigram not in self.bpe_ranks:
                      break
                 __snake_case							,			__snake_case      :Optional[Any] 							=    bigram
                 __snake_case      :int 							=    []
                 __snake_case      :List[str] 							=    0
                 while i < len(a__     ):
                      try:
                           __snake_case      :Tuple 							=    word.index(a__      ,    a__     )
                      except ValueError:
                           new_word.extend(word[i:]     )
                           break
                      else:
                           new_word.extend(word[i:j]     )
                           __snake_case      :List[Any] 							=    j

                      if word[i] == first and i < len(a__     ) - 1 and word[i + 1] == second:
                           new_word.append(first + second     )
                           i += 2
                      else:
                           new_word.append(word[i]     )
                           i += 1
                 __snake_case      :Optional[Any] 							=    tuple(a__     )
                 __snake_case      :Dict 							=    new_word
                 if len(a__     ) == 1:
                      break
                 else:
                      __snake_case      :int 							=    get_pairs(a__     )
            __snake_case      :Optional[Any] 							=    """@@ """.join(a__     )
            __snake_case      :List[str] 							=    word[:-4]
            __snake_case      :Any 							=    word
            return word



       def        __lowercase    (				self      ,    a__     )			->   Optional[Any]:






            '''simple docstring'''





            __snake_case      :Dict 							=    []

            __snake_case      :Optional[int] 							=    re.findall(R"""\S+\n?"""      ,    a__     )

            for token in words:
                 split_tokens.extend(list(self.bpe(a__     ).split(""" """     )     )     )
            return split_tokens



       def        __lowercase    (				self      ,    a__     )			->   Tuple:






            '''simple docstring'''





            return self.encoder.get(a__      ,    self.encoder.get(self.unk_token     )     )



       def        __lowercase    (				self      ,    a__     )			->   List[Any]:






            '''simple docstring'''





            return self.decoder.get(a__      ,    self.unk_token     )



       def        __lowercase    (				self      ,    a__     )			->   Any:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    """ """.join(a__     ).replace("""@@ """      ,    """"""     ).strip()
            return out_string



       def        __lowercase    (				self      ,    a__      ,    a__ = None     )			->   Tuple[str]:






            '''simple docstring'''





            if not os.path.isdir(a__     ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''     )
                 return
            __snake_case      :Tuple 							=    os.path.join(
                a__      ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )
            __snake_case      :Tuple 							=    os.path.join(
                a__      ,    (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]     )

            with open(a__      ,    """w"""      ,    encoding="""utf-8"""     ) as f:
                 f.write(json.dumps(self.encoder      ,    indent=2      ,    sort_keys=a__      ,    ensure_ascii=a__     ) + """\n"""     )

            __snake_case      :List[str] 							=    0
            with open(a__      ,    """w"""      ,    encoding="""utf-8"""     ) as writer:
                 writer.write("""#version: 0.2\n"""     )
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()      ,    key=lambda a__     : kv[1]     ):
                      if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               """ Please check that the tokenizer is not corrupted!"""     )
                           __snake_case      :Optional[Any] 							=    token_index
                      writer.write(""" """.join(a__     ) + """\n"""     )
                      index += 1

            return vocab_file, merge_file

       # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
       #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
       #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
       #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
       #     return ''.join(tokens_generated_so_far)










from collections.abc import Callable




class       snake_case__ :

       '''simple docstring'''



       def __init__(				self      ,    a__ = None     )			->   None:






            '''simple docstring'''





            __snake_case      :list 							=    []
            # Stores indexes of each item for supporting updates and deletion.
            __snake_case      :dict 							=    {}
            # Stores current size of heap.
            __snake_case      :List[Any] 							=    0
            # Stores function used to evaluate the score of an item on which basis ordering
            # will be done.
            __snake_case      :Optional[Any] 							=    key or (lambda a__     : x)



       def        __lowercase    (				self      ,    a__     )			->   int | None:






            '''simple docstring'''





            return int((i - 1) / 2     ) if i > 0 else None



       def        __lowercase    (				self      ,    a__     )			->   int | None:






            '''simple docstring'''





            __snake_case      :int 							=    int(2 * i + 1     )
            return left if 0 < left < self.size else None



       def        __lowercase    (				self      ,    a__     )			->   int | None:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    int(2 * i + 2     )
            return right if 0 < right < self.size else None



       def        __lowercase    (				self      ,    a__      ,    a__     )			->   None:






            '''simple docstring'''





            __snake_case							,			__snake_case      :Dict 							=    (
                self.pos_map[self.arr[j][0]],
                self.pos_map[self.arr[i][0]],
            )
            # Then swap the items in the list.
            __snake_case							,			__snake_case      :Tuple 							=    self.arr[j], self.arr[i]



       def        __lowercase    (				self      ,    a__      ,    a__     )			->   bool:






            '''simple docstring'''





            return self.arr[i][1] < self.arr[j][1]



       def        __lowercase    (				self      ,    a__     )			->   int:






            '''simple docstring'''





            __snake_case      :int 							=    self._left(a__     )
            __snake_case      :Optional[int] 							=    self._right(a__     )
            __snake_case      :List[Any] 							=    i

            if left is not None and not self._cmp(a__      ,    a__     ):
                 __snake_case      :int 							=    left
            if right is not None and not self._cmp(a__      ,    a__     ):
                 __snake_case      :Tuple 							=    right

            return valid_parent



       def        __lowercase    (				self      ,    a__     )			->   None:






            '''simple docstring'''





            __snake_case      :int 							=    self._parent(a__     )
            while parent is not None and not self._cmp(a__      ,    a__     ):
                 self._swap(a__      ,    a__     )
                 __snake_case							,			__snake_case      :str 							=    parent, self._parent(a__     )



       def        __lowercase    (				self      ,    a__     )			->   None:






            '''simple docstring'''





            __snake_case      :List[Any] 							=    self._get_valid_parent(a__     )
            while valid_parent != index:
                 self._swap(a__      ,    a__     )
                 __snake_case							,			__snake_case      :Optional[int] 							=    valid_parent, self._get_valid_parent(a__     )



       def        __lowercase    (				self      ,    a__      ,    a__     )			->   None:






            '''simple docstring'''





            if item not in self.pos_map:
                 return
            __snake_case      :List[Any] 							=    self.pos_map[item]
            __snake_case      :Optional[Any] 							=    [item, self.key(a__     )]
            # Make sure heap is right in both up and down direction.
            # Ideally only one of them will make any change.
            self._heapify_up(a__     )
            self._heapify_down(a__     )



       def        __lowercase    (				self      ,    a__     )			->   None:






            '''simple docstring'''





            if item not in self.pos_map:
                 return
            __snake_case      :Dict 							=    self.pos_map[item]
            del self.pos_map[item]
            __snake_case      :Dict 							=    self.arr[self.size - 1]
            __snake_case      :List[str] 							=    index
            self.size -= 1
            # Make sure heap is right in both up and down direction. Ideally only one
            # of them will make any change- so no performance loss in calling both.
            if self.size > index:
                 self._heapify_up(a__     )
                 self._heapify_down(a__     )



       def        __lowercase    (				self      ,    a__      ,    a__     )			->   None:






            '''simple docstring'''





            __snake_case      :Optional[int] 							=    len(self.arr     )
            if arr_len == self.size:
                 self.arr.append([item, self.key(a__     )]     )
            else:
                 __snake_case      :Optional[Any] 							=    [item, self.key(a__     )]
            __snake_case      :str 							=    self.size
            self.size += 1
            self._heapify_up(self.size - 1     )



       def        __lowercase    (				self     )			->   tuple | None:






            '''simple docstring'''





            return self.arr[0] if self.size else None



       def        __lowercase    (				self     )			->   tuple | None:






            '''simple docstring'''





            __snake_case      :Optional[Any] 							=    self.get_top()
            if top_item_tuple:
                 self.delete_item(top_item_tuple[0]     )
            return top_item_tuple






def 	UpperCamelCase	(							):




     '''simple docstring'''








if __name__ == "__main__":
     import doctest

     doctest.testmod()


