


import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_snake_case			 =	logging.get_logger(__name__)

_snake_case			 =	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_snake_case			 =	{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

_snake_case			 =	{
    '''ctrl''': 2_56,
}

_snake_case			 =	{
    '''Pregnancy''': 16_86_29,
    '''Christianity''': 76_75,
    '''Explain''': 10_64_23,
    '''Fitness''': 6_34_40,
    '''Saving''': 6_31_63,
    '''Ask''': 2_71_71,
    '''Ass''': 9_59_85,
    '''Joke''': 16_35_09,
    '''Questions''': 4_56_22,
    '''Thoughts''': 4_96_05,
    '''Retail''': 5_23_42,
    '''Feminism''': 16_43_38,
    '''Writing''': 1_19_92,
    '''Atheism''': 19_22_63,
    '''Netflix''': 4_86_16,
    '''Computing''': 3_96_39,
    '''Opinion''': 4_32_13,
    '''Alone''': 4_49_67,
    '''Funny''': 5_89_17,
    '''Gaming''': 4_03_58,
    '''Human''': 40_88,
    '''India''': 13_31,
    '''Joker''': 7_71_38,
    '''Diet''': 3_62_06,
    '''Legal''': 1_18_59,
    '''Norman''': 49_39,
    '''Tip''': 7_26_89,
    '''Weight''': 5_23_43,
    '''Movies''': 4_62_73,
    '''Running''': 2_34_25,
    '''Science''': 20_90,
    '''Horror''': 3_77_93,
    '''Confession''': 6_05_72,
    '''Finance''': 1_22_50,
    '''Politics''': 1_63_60,
    '''Scary''': 19_19_85,
    '''Support''': 1_26_54,
    '''Technologies''': 3_25_16,
    '''Teenage''': 6_61_60,
    '''Event''': 3_27_69,
    '''Learned''': 6_74_60,
    '''Notion''': 18_27_70,
    '''Wikipedia''': 3_75_83,
    '''Books''': 66_65,
    '''Extract''': 7_60_50,
    '''Confessions''': 10_27_01,
    '''Conspiracy''': 7_59_32,
    '''Links''': 6_36_74,
    '''Narcissus''': 15_04_25,
    '''Relationship''': 5_47_66,
    '''Relationships''': 13_47_96,
    '''Reviews''': 4_16_71,
    '''News''': 42_56,
    '''Translation''': 2_68_20,
    '''multilingual''': 12_84_06,
}






def 						lowercase_(    SCREAMING_SNAKE_CASE_      ):


  '''simple docstring'''

  lowerCamelCase					:   Tuple       =							set()
  lowerCamelCase					:   int       =							word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)      )
    lowerCamelCase					:   Any       =							char

  lowerCamelCase					:   Union[str, Any]       =							set(UpperCamelCase__      )
  return pairs







class       UpperCAmelCase_   (   UpperCAmelCase__       ):

   '''simple docstring'''




   __A					: str							=						VOCAB_FILES_NAMES
   __A					: Optional[int]							=						PRETRAINED_VOCAB_FILES_MAP
   __A					: Tuple							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __A					: Union[str, Any]							=						CONTROL_CODES






   def __init__(						self		,							__A		,							__A		,							__A="<unk>"		,							**__A					):






     """simple docstring"""





     super().__init__(unk_token=__A		,							**__A					)

     with open(__A		,							encoding="utf-8"					) as vocab_handle:
       lowerCamelCase					:   Tuple       =							json.load(__A					)
     lowerCamelCase					:   Union[str, Any]       =							{v: k for k, v in self.encoder.items()}
     with open(__A		,							encoding="utf-8"					) as merges_handle:
       lowerCamelCase					:   List[str]       =							merges_handle.read().split("\n"					)[1:-1]
     lowerCamelCase					:   List[str]       =							[tuple(merge.split()					) for merge in merges]
     lowerCamelCase					:   int       =							dict(zip(__A		,							range(len(__A					)					)					)					)
     lowerCamelCase					:   List[Any]       =							{}






   @property
   def _snake_case       (						self					):






     """simple docstring"""





     return len(self.encoder					)






   def _snake_case       (						self					):






     """simple docstring"""





     return dict(self.encoder		,							**self.added_tokens_encoder					)






   def _snake_case       (						self		,							__A					):






     """simple docstring"""





     if token in self.cache:
       return self.cache[token]
     lowerCamelCase					:   str       =							tuple(__A					)
     lowerCamelCase					:   Optional[int]       =							tuple(list(word[:-1]					) + [word[-1] + "</w>"]					)
     lowerCamelCase					:   List[str]       =							get_pairs(__A					)

     if not pairs:
       return token

     while True:
       lowerCamelCase					:   Optional[Any]       =							min(__A		,							key=lambda __A					: self.bpe_ranks.get(__A		,							float("inf"					)					)					)
       if bigram not in self.bpe_ranks:
         break
       lowerCamelCase       ,      lowerCamelCase					:   List[Any]       =							bigram
       lowerCamelCase					:   Any       =							[]
       lowerCamelCase					:   List[str]       =							0
       while i < len(__A					):
         try:
           lowerCamelCase					:   str       =							word.index(__A		,							__A					)
         except ValueError:
           new_word.extend(word[i:]					)
           break
         else:
           new_word.extend(word[i:j]					)
           lowerCamelCase					:   Union[str, Any]       =							j

         if word[i] == first and i < len(__A					) - 1 and word[i + 1] == second:
           new_word.append(first + second					)
           i += 2
         else:
           new_word.append(word[i]					)
           i += 1
       lowerCamelCase					:   Union[str, Any]       =							tuple(__A					)
       lowerCamelCase					:   Optional[Any]       =							new_word
       if len(__A					) == 1:
         break
       else:
         lowerCamelCase					:   Tuple       =							get_pairs(__A					)
     lowerCamelCase					:   List[Any]       =							"@@ ".join(__A					)
     lowerCamelCase					:   Dict       =							word[:-4]
     lowerCamelCase					:   int       =							word
     return word






   def _snake_case       (						self		,							__A					):






     """simple docstring"""





     lowerCamelCase					:   Union[str, Any]       =							[]

     lowerCamelCase					:   Optional[int]       =							re.findall(R"\S+\n?"		,							__A					)

     for token in words:
       split_tokens.extend(list(self.bpe(__A					).split(" "					)					)					)
     return split_tokens






   def _snake_case       (						self		,							__A					):






     """simple docstring"""





     return self.encoder.get(__A		,							self.encoder.get(self.unk_token					)					)






   def _snake_case       (						self		,							__A					):






     """simple docstring"""





     return self.decoder.get(__A		,							self.unk_token					)






   def _snake_case       (						self		,							__A					):






     """simple docstring"""





     lowerCamelCase					:   int       =							" ".join(__A					).replace("@@ "		,							""					).strip()
     return out_string






   def _snake_case       (						self		,							__A		,							__A = None					):






     """simple docstring"""





     if not os.path.isdir(__A					):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""					)
       return
     lowerCamelCase					:   List[Any]       =							os.path.join(
         __A		,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)
     lowerCamelCase					:   Any       =							os.path.join(
         __A		,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]					)

     with open(__A		,							"w"		,							encoding="utf-8"					) as f:
       f.write(json.dumps(self.encoder		,							indent=2		,							sort_keys=__A		,							ensure_ascii=__A					) + "\n"					)

     lowerCamelCase					:   List[str]       =							0
     with open(__A		,							"w"		,							encoding="utf-8"					) as writer:
       writer.write("#version: 0.2\n"					)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()		,							key=lambda __A					: kv[1]					):
         if index != token_index:
           logger.warning(
               F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
               " Please check that the tokenizer is not corrupted!"					)
           lowerCamelCase					:   int       =							token_index
         writer.write(" ".join(__A					) + "\n"					)
         index += 1

     return vocab_file, merge_file

   # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
   #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
   #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
   #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
   #     return ''.join(tokens_generated_so_far)




'''simple docstring'''




def __lowerCAmelCase					(				UpperCamelCase__							)      ->       bool:
    if number < 0:
        raise ValueError('''number must not be negative'''							)
    return number & (number - 1) == 0


if __name__ == "__main__":
  import doctest

  doctest.testmod()
