


'''simple docstring'''





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase												=						logging.get_logger(__name__)

__lowerCamelCase												=						{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__lowerCamelCase												=						{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

__lowerCamelCase												=						{
    '''ctrl''': 256,
}

__lowerCamelCase												=						{
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}


def        a__     (					UpperCamelCase_						:      List[Any]						):
   UpperCAmelCase__	:Union[str, Any]      =      set()
   UpperCAmelCase__	:Optional[Any]      =      word[0]
   for char in word[1:]:
      pairs.add((prev_char, char)						)
      UpperCAmelCase__	:Any      =      char

   UpperCAmelCase__	:int      =      set(UpperCamelCase_						)
   return pairs




class 				UpperCAmelCase			(			_snake_case						):
 UpperCAmelCase				=					VOCAB_FILES_NAMES
 UpperCAmelCase				=					PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase				=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 UpperCAmelCase				=					CONTROL_CODES


 def __init__(      self							:	Optional[Any]			,       __lowerCamelCase							:	Optional[Any]			,       __lowerCamelCase							:	str			,       __lowerCamelCase							:	Union[str, Any]="<unk>"			,       **__lowerCamelCase							:	Any ):
    super().__init__(unk_token=__lowerCamelCase			,       **__lowerCamelCase )

    with open(__lowerCamelCase			,       encoding='''utf-8''' ) as vocab_handle:
       UpperCAmelCase__	:str      =      json.load(__lowerCamelCase )
    UpperCAmelCase__	:Dict      =      {v: k for k, v in self.encoder.items()}
    with open(__lowerCamelCase			,       encoding='''utf-8''' ) as merges_handle:
       UpperCAmelCase__	:Optional[Any]      =      merges_handle.read().split('''\n''' )[1:-1]
    UpperCAmelCase__	:Any      =      [tuple(merge.split() ) for merge in merges]
    UpperCAmelCase__	:List[str]      =      dict(zip(__lowerCamelCase			,       range(len(__lowerCamelCase ) ) ) )
    UpperCAmelCase__	:List[str]      =      {}


 @property
 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str] ):
    return len(self.encoder )


 def        __SCREAMING_SNAKE_CASE					(      self							:	str ):
    return dict(self.encoder			,       **self.added_tokens_encoder )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any]			,       __lowerCamelCase							:	Tuple ):
    if token in self.cache:
       return self.cache[token]
    UpperCAmelCase__	:Optional[int]      =      tuple(__lowerCamelCase )
    UpperCAmelCase__	:Dict      =      tuple(list(word[:-1] ) + [word[-1] + '''</w>'''] )
    UpperCAmelCase__	:Dict      =      get_pairs(__lowerCamelCase )

    if not pairs:
       return token

    while True:
       UpperCAmelCase__	:Any      =      min(__lowerCamelCase			,       key=lambda __lowerCamelCase : self.bpe_ranks.get(__lowerCamelCase			,       float('''inf''' ) ) )
       if bigram not in self.bpe_ranks:
          break
       UpperCAmelCase__   ,						UpperCAmelCase__	:List[Any]      =      bigram
       UpperCAmelCase__	:Dict      =      []
       UpperCAmelCase__	:Dict      =      0
       while i < len(__lowerCamelCase ):
          try:
             UpperCAmelCase__	:Dict      =      word.index(__lowerCamelCase			,       __lowerCamelCase )
          except ValueError:
             new_word.extend(word[i:] )
             break
          else:
             new_word.extend(word[i:j] )
             UpperCAmelCase__	:List[Any]      =      j

          if word[i] == first and i < len(__lowerCamelCase ) - 1 and word[i + 1] == second:
             new_word.append(first + second )
             i += 2
          else:
             new_word.append(word[i] )
             i += 1
       UpperCAmelCase__	:List[str]      =      tuple(__lowerCamelCase )
       UpperCAmelCase__	:str      =      new_word
       if len(__lowerCamelCase ) == 1:
          break
       else:
          UpperCAmelCase__	:Optional[int]      =      get_pairs(__lowerCamelCase )
    UpperCAmelCase__	:str      =      '''@@ '''.join(__lowerCamelCase )
    UpperCAmelCase__	:List[str]      =      word[:-4]
    UpperCAmelCase__	:List[str]      =      word
    return word


 def        __SCREAMING_SNAKE_CASE					(      self							:	Tuple			,       __lowerCamelCase							:	Dict ):
    UpperCAmelCase__	:List[str]      =      []

    UpperCAmelCase__	:Union[str, Any]      =      re.findall(r'''\S+\n?'''			,       __lowerCamelCase )

    for token in words:
       split_tokens.extend(list(self.bpe(__lowerCamelCase ).split(''' ''' ) ) )
    return split_tokens


 def        __SCREAMING_SNAKE_CASE					(      self							:	Union[str, Any]			,       __lowerCamelCase							:	Optional[int] ):
    return self.encoder.get(__lowerCamelCase			,       self.encoder.get(self.unk_token ) )


 def        __SCREAMING_SNAKE_CASE					(      self							:	Optional[Any]			,       __lowerCamelCase							:	int ):
    return self.decoder.get(__lowerCamelCase			,       self.unk_token )


 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str]			,       __lowerCamelCase							:	Dict ):
    UpperCAmelCase__	:Optional[int]      =      ''' '''.join(__lowerCamelCase ).replace('''@@ '''			,       '''''' ).strip()
    return out_string





 def        __SCREAMING_SNAKE_CASE					(      self							:	List[str]			,       __lowerCamelCase							:	str			,       __lowerCamelCase							:	Optional[str] = None ):
    if not os.path.isdir(__lowerCamelCase ):
       logger.error(f'''Vocabulary path ({save_directory}) should be a directory''' )
       return
    UpperCAmelCase__	:str      =      os.path.join(
        __lowerCamelCase			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''] )
    UpperCAmelCase__	:Optional[Any]      =      os.path.join(
        __lowerCamelCase			,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''] )

    with open(__lowerCamelCase			,       '''w'''			,       encoding='''utf-8''' ) as f:
       f.write(json.dumps(self.encoder			,       indent=2			,       sort_keys=__lowerCamelCase			,       ensure_ascii=__lowerCamelCase ) + '''\n''' )

    UpperCAmelCase__	:Union[str, Any]      =      0
    with open(__lowerCamelCase			,       '''w'''			,       encoding='''utf-8''' ) as writer:
       writer.write('''#version: 0.2\n''' )
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,       key=lambda __lowerCamelCase : kv[1] ):
          if index != token_index:
             logger.warning(
                 f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                 ''' Please check that the tokenizer is not corrupted!''' )
             UpperCAmelCase__	:str      =      token_index
          writer.write(''' '''.join(__lowerCamelCase ) + '''\n''' )
          index += 1

    return vocab_file, merge_file

 # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
 #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
 #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
 #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
 #     return ''.join(tokens_generated_so_far)







'''simple docstring'''





import pytest

from datasets.splits import SplitDict, SplitInfo
from datasets.utils.py_utils import asdict


@pytest.mark.parametrize(
    '''split_dict''',   [
        SplitDict(),
        SplitDict({'''train''': SplitInfo(name='''train''',   num_bytes=1_337,   num_examples=42,   dataset_name='''my_dataset'''						)}						),
        SplitDict({'''train''': SplitInfo(name='''train''',   num_bytes=1_337,   num_examples=42						)}						),
        SplitDict({'''train''': SplitInfo()}						),
    ],   )
def        a__     (					UpperCamelCase_						:      SplitDict						):
   UpperCAmelCase__	:List[Any]      =      split_dict._to_yaml_list()
   assert len(UpperCamelCase_						) == len(UpperCamelCase_						)
   UpperCAmelCase__	:Tuple      =      SplitDict._from_yaml_list(UpperCamelCase_						)
   for split_name, split_info in split_dict.items():
      # dataset_name field is deprecated, and is therefore not part of the YAML dump
      UpperCAmelCase__	:Union[str, Any]      =      None
      # the split name of split_dict takes over the name of the split info object
      UpperCAmelCase__	:Union[str, Any]      =      split_name
   assert split_dict == reloaded


@pytest.mark.parametrize(
    '''split_info''',   [SplitInfo(), SplitInfo(dataset_name=UpperCamelCase_						), SplitInfo(dataset_name='''my_dataset'''						)]						)
def        a__     (					UpperCamelCase_						:      List[Any]						):
   # For backward compatibility, we need asdict(split_dict) to return split info dictrionaries with the "dataset_name"
   # field even if it's deprecated. This way old versionso of `datasets` can still reload dataset_infos.json files
   UpperCAmelCase__	:List[Any]      =      asdict(SplitDict({'''train''': split_info}						)						)
   assert "dataset_name" in split_dict_asdict["train"]
   assert split_dict_asdict["train"]["dataset_name"] == split_info.dataset_name



