"""simple docstring"""



import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

_UpperCamelCase								=		{
    "vocab_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json"},
    "merges_file": {"ctrl": "https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt"},
}

_UpperCamelCase								=		{
    "ctrl": 2_56,
}

_UpperCamelCase								=		{
    "Pregnancy": 16_86_29,
    "Christianity": 76_75,
    "Explain": 10_64_23,
    "Fitness": 6_34_40,
    "Saving": 6_31_63,
    "Ask": 2_71_71,
    "Ass": 9_59_85,
    "Joke": 16_35_09,
    "Questions": 4_56_22,
    "Thoughts": 4_96_05,
    "Retail": 5_23_42,
    "Feminism": 16_43_38,
    "Writing": 1_19_92,
    "Atheism": 19_22_63,
    "Netflix": 4_86_16,
    "Computing": 3_96_39,
    "Opinion": 4_32_13,
    "Alone": 4_49_67,
    "Funny": 5_89_17,
    "Gaming": 4_03_58,
    "Human": 40_88,
    "India": 13_31,
    "Joker": 7_71_38,
    "Diet": 3_62_06,
    "Legal": 1_18_59,
    "Norman": 49_39,
    "Tip": 7_26_89,
    "Weight": 5_23_43,
    "Movies": 4_62_73,
    "Running": 2_34_25,
    "Science": 20_90,
    "Horror": 3_77_93,
    "Confession": 6_05_72,
    "Finance": 1_22_50,
    "Politics": 1_63_60,
    "Scary": 19_19_85,
    "Support": 1_26_54,
    "Technologies": 3_25_16,
    "Teenage": 6_61_60,
    "Event": 3_27_69,
    "Learned": 6_74_60,
    "Notion": 18_27_70,
    "Wikipedia": 3_75_83,
    "Books": 66_65,
    "Extract": 7_60_50,
    "Confessions": 10_27_01,
    "Conspiracy": 7_59_32,
    "Links": 6_36_74,
    "Narcissus": 15_04_25,
    "Relationship": 5_47_66,
    "Relationships": 13_47_96,
    "Reviews": 4_16_71,
    "News": 42_56,
    "Translation": 2_68_20,
    "multilingual": 12_84_06,
}

def  _A(		lowerCAmelCase							):
							A__       :     Dict								=			set()
							A__       :     Union[str, Any]								=			word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)							)
														A__       :     List[Any]								=			char

							A__       :     List[str]								=			set(lowerCAmelCase							)
							return pairs



class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						int					     =				VOCAB_FILES_NAMES
			_UpperCamelCase  :						Optional[int]					     =				PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase  :						List[Any]					     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase  :						str					     =				CONTROL_CODES





			def __init__(			self   ,      snake_case_   ,      snake_case_   ,      snake_case_="<unk>"   ,      **snake_case_							):







										'''simple docstring'''


										super().__init__(unk_token=snake_case_   ,      **snake_case_							)

										with open(snake_case_   ,      encoding="""utf-8"""							) as vocab_handle:
																	A__       :     Union[str, Any]								=			json.load(snake_case_							)
										A__       :     Tuple								=			{v: k for k, v in self.encoder.items()}
										with open(snake_case_   ,      encoding="""utf-8"""							) as merges_handle:
																	A__       :     Tuple								=			merges_handle.read().split("""\n"""							)[1:-1]
										A__       :     str								=			[tuple(merge.split()							) for merge in merges]
										A__       :     List[str]								=			dict(zip(snake_case_   ,      range(len(snake_case_							)							)							)							)
										A__       :     Union[str, Any]								=			{}





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return len(self.encoder							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return dict(self.encoder   ,      **self.added_tokens_encoder							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										if token in self.cache:
																	return self.cache[token]
										A__       :     Optional[int]								=			tuple(snake_case_							)
										A__       :     Dict								=			tuple(list(word[:-1]							) + [word[-1] + """</w>"""]							)
										A__       :     Tuple								=			get_pairs(snake_case_							)

										if not pairs:
																	return token

										while True:
																	A__       :     Optional[int]								=			min(snake_case_   ,      key=lambda snake_case_							: self.bpe_ranks.get(snake_case_   ,      float("""inf"""							)							)							)
																	if bigram not in self.bpe_ranks:
																								break
																	A__				,							A__       :     str								=			bigram
																	A__       :     str								=			[]
																	A__       :     str								=			0
																	while i < len(snake_case_							):
																								try:
																															A__       :     List[str]								=			word.index(snake_case_   ,      snake_case_							)
																								except ValueError:
																															new_word.extend(word[i:]							)
																															break
																								else:
																															new_word.extend(word[i:j]							)
																															A__       :     int								=			j

																								if word[i] == first and i < len(snake_case_							) - 1 and word[i + 1] == second:
																															new_word.append(first + second							)
																															i += 2
																								else:
																															new_word.append(word[i]							)
																															i += 1
																	A__       :     Dict								=			tuple(snake_case_							)
																	A__       :     int								=			new_word
																	if len(snake_case_							) == 1:
																								break
																	else:
																								A__       :     Any								=			get_pairs(snake_case_							)
										A__       :     Any								=			"""@@ """.join(snake_case_							)
										A__       :     List[Any]								=			word[:-4]
										A__       :     Any								=			word
										return word





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Optional[int]								=			[]

										A__       :     List[str]								=			re.findall(r"""\S+\n?"""   ,      snake_case_							)

										for token in words:
																	split_tokens.extend(list(self.bpe(snake_case_							).split(""" """							)							)							)
										return split_tokens





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.encoder.get(snake_case_   ,      self.encoder.get(self.unk_token							)							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.decoder.get(snake_case_   ,      self.unk_token							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			""" """.join(snake_case_							).replace("""@@ """   ,      """"""							).strip()
										return out_string





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										if not os.path.isdir(snake_case_							):
																	logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
																	return
										A__       :     int								=			os.path.join(
										    snake_case_   ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)
										A__       :     Union[str, Any]								=			os.path.join(
										    snake_case_   ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]							)

										with open(snake_case_   ,      """w"""   ,      encoding="""utf-8"""							) as f:
																	f.write(json.dumps(self.encoder   ,      indent=2   ,      sort_keys=snake_case_   ,      ensure_ascii=snake_case_							) + """\n"""							)

										A__       :     str								=			0
										with open(snake_case_   ,      """w"""   ,      encoding="""utf-8"""							) as writer:
																	writer.write("""#version: 0.2\n"""							)
																	for bpe_tokens, token_index in sorted(self.bpe_ranks.items()   ,      key=lambda snake_case_							: kv[1]							):
																								if index != token_index:
																															logger.warning(
																															    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																															    """ Please check that the tokenizer is not corrupted!"""							)
																															A__       :     List[str]								=			token_index
																								writer.write(""" """.join(snake_case_							) + """\n"""							)
																								index += 1

										return vocab_file, merge_file

			# def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
			#     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
			#     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
			#     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
			#     return ''.join(tokens_generated_so_far)






"""simple docstring"""



import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{"vocab_file": "sentencepiece.model"}

_UpperCamelCase								=		{
    "vocab_file": {
        "google/rembert": "https://huggingface.co/google/rembert/resolve/main/sentencepiece.model",
    },
}

_UpperCamelCase								=		{
    "google/rembert": 2_56,
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Dict					     =				VOCAB_FILES_NAMES
			_UpperCamelCase  :						List[str]					     =				PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase  :						Any					     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





			def __init__(			self   ,      snake_case_   ,      snake_case_=False   ,      snake_case_=True   ,      snake_case_=True   ,      snake_case_="[CLS]"   ,      snake_case_="[SEP]"   ,      snake_case_="[UNK]"   ,      snake_case_="[SEP]"   ,      snake_case_="[PAD]"   ,      snake_case_="[CLS]"   ,      snake_case_="[MASK]"   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(
										    do_lower_case=snake_case_   ,      remove_space=snake_case_   ,      keep_accents=snake_case_   ,      bos_token=snake_case_   ,      eos_token=snake_case_   ,      unk_token=snake_case_   ,      sep_token=snake_case_   ,      pad_token=snake_case_   ,      cls_token=snake_case_   ,      mask_token=snake_case_   ,      **snake_case_   ,      )

										A__       :     List[Any]								=			do_lower_case
										A__       :     Dict								=			remove_space
										A__       :     Optional[int]								=			keep_accents
										A__       :     Tuple								=			vocab_file

										A__       :     List[str]								=			spm.SentencePieceProcessor()
										self.sp_model.Load(snake_case_							)





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return len(self.sp_model							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			{self.convert_ids_to_tokens(snake_case_							): i for i in range(self.vocab_size							)}
										vocab.update(self.added_tokens_encoder							)
										return vocab





			def __getstate__(			self							):







										'''simple docstring'''


										A__       :     str								=			self.__dict__.copy()
										A__       :     Tuple								=			None
										return state





			def __setstate__(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Dict								=			d
										A__       :     int								=			spm.SentencePieceProcessor()
										self.sp_model.Load(self.vocab_file							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_=False							):







										'''simple docstring'''


										A__       :     Tuple								=			self.sp_model.EncodeAsPieces(snake_case_							)
										return pieces





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.sp_model.PieceToId(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return self.sp_model.IdToPiece(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     str								=			self.sp_model.decode_pieces(snake_case_							)
										return out_string





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     List[Any]								=			[self.sep_token_id]
										A__       :     str								=			[self.cls_token_id]
										if token_ids_a is None:
																	return cls + token_ids_a + sep
										return cls + token_ids_a + sep + token_ids_a + sep





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None   ,      snake_case_ = False							):







										'''simple docstring'''


										if already_has_special_tokens:
																	if token_ids_a is not None:
																								raise ValueError(
																								    """You should not supply a second sequence if the provided sequence of """
																								    """ids is already formatted with special tokens for the model."""							)
																	return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

										if token_ids_a is not None:
																	return [1] + ([0] * len(snake_case_							)) + [1] + ([0] * len(snake_case_							)) + [1]
										return [1] + ([0] * len(snake_case_							)) + [1]





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     int								=			[self.sep_token_id]
										A__       :     Any								=			[self.cls_token_id]

										if token_ids_a is None:
																	return len(cls + token_ids_a + sep							) * [0]
										return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										if not os.path.isdir(snake_case_							):
																	logger.error("""Vocabulary path ({}) should be a directory""".format(snake_case_							)							)
																	return
										A__       :     Any								=			os.path.join(
										    snake_case_   ,      (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

										if os.path.abspath(self.vocab_file							) != os.path.abspath(snake_case_							):
																	copyfile(self.vocab_file   ,      snake_case_							)

										return (out_vocab_file,)





