





"""simple docstring"""



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_			=      logging.get_logger(__name__)

a_			=      {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a_			=      {
    """vocab_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"""},
    """merges_file""": {"""ctrl""": """https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"""},
}

a_			=      {
    """ctrl""": 256,
}

a_			=      {
    """Pregnancy""": 168629,
    """Christianity""": 7675,
    """Explain""": 106423,
    """Fitness""": 63440,
    """Saving""": 63163,
    """Ask""": 27171,
    """Ass""": 95985,
    """Joke""": 163509,
    """Questions""": 45622,
    """Thoughts""": 49605,
    """Retail""": 52342,
    """Feminism""": 164338,
    """Writing""": 11992,
    """Atheism""": 192263,
    """Netflix""": 48616,
    """Computing""": 39639,
    """Opinion""": 43213,
    """Alone""": 44967,
    """Funny""": 58917,
    """Gaming""": 40358,
    """Human""": 4088,
    """India""": 1331,
    """Joker""": 77138,
    """Diet""": 36206,
    """Legal""": 11859,
    """Norman""": 4939,
    """Tip""": 72689,
    """Weight""": 52343,
    """Movies""": 46273,
    """Running""": 23425,
    """Science""": 2090,
    """Horror""": 37793,
    """Confession""": 60572,
    """Finance""": 12250,
    """Politics""": 16360,
    """Scary""": 191985,
    """Support""": 12654,
    """Technologies""": 32516,
    """Teenage""": 66160,
    """Event""": 32769,
    """Learned""": 67460,
    """Notion""": 182770,
    """Wikipedia""": 37583,
    """Books""": 6665,
    """Extract""": 76050,
    """Confessions""": 102701,
    """Conspiracy""": 75932,
    """Links""": 63674,
    """Narcissus""": 150425,
    """Relationship""": 54766,
    """Relationships""": 134796,
    """Reviews""": 41671,
    """News""": 4256,
    """Translation""": 26820,
    """multilingual""": 128406,
}







def 		__lowercase					(       snake_case_  :  List[Any]						)       ->List[Any]:



    '''simple docstring'''







    __A   :						List[Any]									=					set()
    __A   :						Union[str, Any]									=					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)						)
        __A   :						Union[str, Any]									=					char

    __A   :						int									=					set(snake_case_						)
    return pairs






class 		__snake_case (		SCREAMING_SNAKE_CASE__					):





 """simple docstring"""


 _lowerCamelCase				=					VOCAB_FILES_NAMES
 _lowerCamelCase				=					PRETRAINED_VOCAB_FILES_MAP
 _lowerCamelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCamelCase				=					CONTROL_CODES

 def __init__(   self ,						__lowerCamelCase ,						__lowerCamelCase ,						__lowerCamelCase="<unk>" ,						**__lowerCamelCase  ):



     '''simple docstring'''




     super().__init__(unk_token=__lowerCamelCase ,						**__lowerCamelCase  )

     with open(__lowerCamelCase ,						encoding='''utf-8'''  ) as vocab_handle:
         __A   :						List[str]									=					json.load(__lowerCamelCase  )
     __A   :						Optional[Any]									=					{v: k for k, v in self.encoder.items()}
     with open(__lowerCamelCase ,						encoding='''utf-8'''  ) as merges_handle:
         __A   :						Optional[int]									=					merges_handle.read().split('''\n'''  )[1:-1]
     __A   :						Any									=					[tuple(merge.split()  ) for merge in merges]
     __A   :						Tuple									=					dict(zip(__lowerCamelCase ,						range(len(__lowerCamelCase  )  )  )  )
     __A   :						Any									=					{}

 @property
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     return len(self.encoder  )

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     return dict(self.encoder ,						**self.added_tokens_encoder  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     if token in self.cache:
         return self.cache[token]
     __A   :						Union[str, Any]									=					tuple(__lowerCamelCase  )
     __A   :						int									=					tuple(list(word[:-1]  ) + [word[-1] + '''</w>''']  )
     __A   :						int									=					get_pairs(__lowerCamelCase  )

     if not pairs:
         return token

     while True:
         __A   :						List[str]									=					min(__lowerCamelCase ,						key=lambda __lowerCamelCase  : self.bpe_ranks.get(__lowerCamelCase ,						float('''inf'''  )  )  )
         if bigram not in self.bpe_ranks:
             break
         __A			,	__A   :						str									=					bigram
         __A   :						Union[str, Any]									=					[]
         __A   :						Any									=					0
         while i < len(__lowerCamelCase  ):
             try:
                 __A   :						Optional[Any]									=					word.index(__lowerCamelCase ,						__lowerCamelCase  )
             except ValueError:
                 new_word.extend(word[i:]  )
                 break
             else:
                 new_word.extend(word[i:j]  )
                 __A   :						Dict									=					j

             if word[i] == first and i < len(__lowerCamelCase  ) - 1 and word[i + 1] == second:
                 new_word.append(first + second  )
                 i += 2
             else:
                 new_word.append(word[i]  )
                 i += 1
         __A   :						List[Any]									=					tuple(__lowerCamelCase  )
         __A   :						Any									=					new_word
         if len(__lowerCamelCase  ) == 1:
             break
         else:
             __A   :						Dict									=					get_pairs(__lowerCamelCase  )
     __A   :						Dict									=					'''@@ '''.join(__lowerCamelCase  )
     __A   :						str									=					word[:-4]
     __A   :						Union[str, Any]									=					word
     return word

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						Dict									=					[]

     __A   :						Any									=					re.findall(r'''\S+\n?''' ,						__lowerCamelCase  )

     for token in words:
         split_tokens.extend(list(self.bpe(__lowerCamelCase  ).split(''' '''  )  )  )
     return split_tokens

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     return self.encoder.get(__lowerCamelCase ,						self.encoder.get(self.unk_token  )  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     return self.decoder.get(__lowerCamelCase ,						self.unk_token  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						List[Any]									=					''' '''.join(__lowerCamelCase  ).replace('''@@ ''' ,						''''''  ).strip()
     return out_string







 def  UpperCamelCase__(   self ,						__lowerCamelCase ,						__lowerCamelCase = None  ):



     '''simple docstring'''




     if not os.path.isdir(__lowerCamelCase  ):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
         return
     __A   :						str									=					os.path.join(
         __lowerCamelCase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
     __A   :						List[str]									=					os.path.join(
         __lowerCamelCase ,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

     with open(__lowerCamelCase ,						'''w''' ,						encoding='''utf-8'''  ) as f:
         f.write(json.dumps(self.encoder ,						indent=2 ,						sort_keys=__lowerCamelCase ,						ensure_ascii=__lowerCamelCase  ) + '''\n'''  )

     __A   :						List[str]									=					0
     with open(__lowerCamelCase ,						'''w''' ,						encoding='''utf-8'''  ) as writer:
         writer.write('''#version: 0.2\n'''  )
         for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,						key=lambda __lowerCamelCase  : kv[1]  ):
             if index != token_index:
                 logger.warning(
                     F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                     ''' Please check that the tokenizer is not corrupted!'''  )
                 __A   :						Any									=					token_index
             writer.write(''' '''.join(__lowerCamelCase  ) + '''\n'''  )
             index += 1

     return vocab_file, merge_file

 # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
 #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
 #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
 #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
 #     return ''.join(tokens_generated_so_far)











"""simple docstring"""



import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin






class 		__snake_case (		SCREAMING_SNAKE_CASE__							,							unittest.TestCase					):





 """simple docstring"""


 _lowerCamelCase				=					XLMTokenizer
 _lowerCamelCase				=					False

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     __A   :						Tuple									=					[
         '''l''',
         '''o''',
         '''w''',
         '''e''',
         '''r''',
         '''s''',
         '''t''',
         '''i''',
         '''d''',
         '''n''',
         '''w</w>''',
         '''r</w>''',
         '''t</w>''',
         '''lo''',
         '''low''',
         '''er</w>''',
         '''low</w>''',
         '''lowest</w>''',
         '''newer</w>''',
         '''wider</w>''',
         '''<unk>''',
     ]
     __A   :						Dict									=					dict(zip(__lowerCamelCase ,						range(len(__lowerCamelCase  )  )  )  )
     __A   :						Union[str, Any]									=					['''l o 123''', '''lo w 1456''', '''e r</w> 1789''', '''''']

     __A   :						Optional[Any]									=					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''vocab_file''']  )
     __A   :						Optional[Any]									=					os.path.join(self.tmpdirname ,						VOCAB_FILES_NAMES['''merges_file''']  )
     with open(self.vocab_file ,						'''w'''  ) as fp:
         fp.write(json.dumps(__lowerCamelCase  )  )
     with open(self.merges_file ,						'''w'''  ) as fp:
         fp.write('''\n'''.join(__lowerCamelCase  )  )

 def  UpperCamelCase__(   self ,						__lowerCamelCase  ):



     '''simple docstring'''




     __A   :						Optional[int]									=					'''lower newer'''
     __A   :						int									=					'''lower newer'''
     return input_text, output_text

 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Union[str, Any]									=					XLMTokenizer(self.vocab_file ,						self.merges_file  )

     __A   :						Optional[Any]									=					'''lower'''
     __A   :						Any									=					['''low''', '''er</w>''']
     __A   :						Tuple									=					tokenizer.tokenize(__lowerCamelCase  )
     self.assertListEqual(__lowerCamelCase ,						__lowerCamelCase  )

     __A   :						str									=					tokens + ['''<unk>''']
     __A   :						List[str]									=					[14, 15, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase  ) ,						__lowerCamelCase  )







 @slow
 def  UpperCamelCase__(   self  ):



     '''simple docstring'''




     __A   :						Optional[int]									=					XLMTokenizer.from_pretrained('''xlm-mlm-en-2048'''  )

     __A   :						Union[str, Any]									=					tokenizer.encode('''sequence builders''' ,						add_special_tokens=__lowerCamelCase  )
     __A   :						int									=					tokenizer.encode('''multi-sequence build''' ,						add_special_tokens=__lowerCamelCase  )

     __A   :						int									=					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase  )
     __A   :						List[Any]									=					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase ,						__lowerCamelCase  )

     assert encoded_sentence == [0] + text + [1]
     assert encoded_pair == [0] + text + [1] + text_a + [1]




