


"""simple docstring"""


import os
import re
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A  :     int    					=    logging.get_logger(__name__)

A  :     Tuple    					=    {
    "vocab_file": "vocab.txt",
    "merges_file": "bpe.codes",
}

A  :     List[Any]    					=    {
    "vocab_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/vocab.txt",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/vocab.txt",
    },
    "merges_file": {
        "vinai/phobert-base": "https://huggingface.co/vinai/phobert-base/resolve/main/bpe.codes",
        "vinai/phobert-large": "https://huggingface.co/vinai/phobert-large/resolve/main/bpe.codes",
    },
}

A  :     Optional[int]    					=    {
    "vinai/phobert-base": 2_5_6,
    "vinai/phobert-large": 2_5_6,
}


def _lowerCamelCase			(	_UpperCamelCase   ):

			'''simple docstring'''




			__lowerCAmelCase       					=    set()
			__lowerCAmelCase       					=    word[0]
			for char in word[1:]:
						pairs.add((prev_char, char)   )
						__lowerCAmelCase       					=    char

			__lowerCAmelCase       					=    set(_UpperCamelCase   )
			return pairs


class 						_UpperCamelCase						(     lowerCAmelCase__	):
							'''simple docstring'''







							__UpperCAmelCase		:					Union[str, Any]          =VOCAB_FILES_NAMES
							__UpperCAmelCase		:					int          =PRETRAINED_VOCAB_FILES_MAP
							__UpperCAmelCase		:					List[str]          =PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

							def __init__(  self					,     __a					,     __a					,     __a="<s>"					,     __a="</s>"					,     __a="</s>"					,     __a="<s>"					,     __a="<unk>"					,     __a="<pad>"					,     __a="<mask>"					,     **__a					,     ):
										super().__init__(
										    bos_token=__a					,     eos_token=__a					,     unk_token=__a					,     sep_token=__a					,     cls_token=__a					,     pad_token=__a					,     mask_token=__a					,     **__a					,     )

										__lowerCAmelCase       					=    vocab_file
										__lowerCAmelCase       					=    merges_file

										__lowerCAmelCase       					=    {}
										__lowerCAmelCase       					=    0
										__lowerCAmelCase       					=    1
										__lowerCAmelCase       					=    2
										__lowerCAmelCase       					=    3

										self.add_from_file(__a			)

										__lowerCAmelCase       					=    {v: k for k, v in self.encoder.items()}

										with open(__a					,     encoding="utf-8"			) as merges_handle:
													__lowerCAmelCase       					=    merges_handle.read().split("\n"			)[:-1]
										__lowerCAmelCase       					=    [tuple(merge.split()[:-1]			) for merge in merges]
										__lowerCAmelCase       					=    dict(zip(__a					,     range(len(__a			)			)			)			)
										__lowerCAmelCase       					=    {}

							def     snake_case       (  self					,     __a					,     __a = None			):

										if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										__lowerCAmelCase       					=    [self.cls_token_id]
										__lowerCAmelCase       					=    [self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep

							def     snake_case       (  self					,     __a					,     __a = None					,     __a = False			):

										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=__a					,     token_ids_a=__a					,     already_has_special_tokens=__a			)

										if token_ids_a is None:
													return [1] + ([0] * len(__a			)) + [1]
										return [1] + ([0] * len(__a			)) + [1, 1] + ([0] * len(__a			)) + [1]

							def     snake_case       (  self					,     __a					,     __a = None			):
										__lowerCAmelCase       					=    [self.sep_token_id]
										__lowerCAmelCase       					=    [self.cls_token_id]

										if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]

							@property
							def     snake_case       (  self			):
										return len(self.encoder			)

							def     snake_case       (  self			):
										return dict(self.encoder					,     **self.added_tokens_encoder			)

							def     snake_case       (  self					,     __a			):
										if token in self.cache:
													return self.cache[token]
										__lowerCAmelCase       					=    tuple(__a			)
										__lowerCAmelCase       					=    tuple(list(word[:-1]			) + [word[-1] + "</w>"]			)
										__lowerCAmelCase       					=    get_pairs(__a			)

										if not pairs:
													return token

										while True:
													__lowerCAmelCase       					=    min(__a					,     key=lambda __a			: self.bpe_ranks.get(__a					,     float("inf"			)			)			)
													if bigram not in self.bpe_ranks:
																break
													__lowerCAmelCase					,			__lowerCAmelCase       					=    bigram
													__lowerCAmelCase       					=    []
													__lowerCAmelCase       					=    0
													while i < len(__a			):
																try:
																			__lowerCAmelCase       					=    word.index(__a					,     __a			)
																except ValueError:
																			new_word.extend(word[i:]			)
																			break
																else:
																			new_word.extend(word[i:j]			)
																			__lowerCAmelCase       					=    j

																if word[i] == first and i < len(__a			) - 1 and word[i + 1] == second:
																			new_word.append(first + second			)
																			i += 2
																else:
																			new_word.append(word[i]			)
																			i += 1
													__lowerCAmelCase       					=    tuple(__a			)
													__lowerCAmelCase       					=    new_word
													if len(__a			) == 1:
																break
													else:
																__lowerCAmelCase       					=    get_pairs(__a			)
										__lowerCAmelCase       					=    "@@ ".join(__a			)
										__lowerCAmelCase       					=    word[:-4]
										__lowerCAmelCase       					=    word
										return word

							def     snake_case       (  self					,     __a			):
										__lowerCAmelCase       					=    []

										__lowerCAmelCase       					=    re.findall(R"\S+\n?"					,     __a			)

										for token in words:
													split_tokens.extend(list(self.bpe(__a			).split(" "			)			)			)
										return split_tokens

							def     snake_case       (  self					,     __a			):
										return self.encoder.get(__a					,     self.encoder.get(self.unk_token			)			)

							def     snake_case       (  self					,     __a			):
										return self.decoder.get(__a					,     self.unk_token			)

							def     snake_case       (  self					,     __a			):
										__lowerCAmelCase       					=    " ".join(__a			).replace("@@ "					,     ""			).strip()
										return out_string

							def     snake_case       (  self					,     __a					,     __a = None			):
										if not os.path.isdir(__a			):
													logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
													return
										__lowerCAmelCase       					=    os.path.join(
										    __a					,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
										__lowerCAmelCase       					=    os.path.join(
										    __a					,     (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

										if os.path.abspath(self.vocab_file			) != os.path.abspath(__a			):
													copyfile(self.vocab_file					,     __a			)

										if os.path.abspath(self.merges_file			) != os.path.abspath(__a			):
													copyfile(self.merges_file					,     __a			)

										return out_vocab_file, out_merge_file



							def     snake_case       (  self					,     __a			):
										if isinstance(__a					,     __a			):
													try:
																with open(__a					,     "r"					,     encoding="utf-8"			) as fd:
																			self.add_from_file(__a			)
													except FileNotFoundError as fnfe:
																raise fnfe
													except UnicodeError:
																raise Exception(f"Incorrect encoding detected in {f}, please rebuild the dataset"			)
													return

										__lowerCAmelCase       					=    f.readlines()
										for lineTmp in lines:
													__lowerCAmelCase       					=    lineTmp.strip()
													__lowerCAmelCase       					=    line.rfind(" "			)
													if idx == -1:
																raise ValueError("Incorrect dictionary format, expected '<token> <cnt>'"			)
													__lowerCAmelCase       					=    line[:idx]
													__lowerCAmelCase       					=    len(self.encoder			)








"""simple docstring"""


A  :     List[Any]    					=    [4, 1, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A  :     Optional[int]    					=    [3, 7, 7, 4, 2, 6, 4, 1, 5, 3, 7, 5]
A  :     Optional[int]    					=    {
    0: "Sunday",
    1: "Monday",
    2: "Tuesday",
    3: "Wednesday",
    4: "Thursday",
    5: "Friday",
    6: "Saturday",
}


def _lowerCamelCase			(	_UpperCamelCase					,				_UpperCamelCase					,				_UpperCamelCase   ):

			'''simple docstring'''




			assert len(str(_UpperCamelCase   )   ) > 2, "year should be in YYYY format"
			assert 1 <= month <= 12, "month should be between 1 to 12"
			assert 1 <= day <= 31, "day should be between 1 to 31"

			# Doomsday algorithm:
			__lowerCAmelCase       					=    year // 100
			__lowerCAmelCase       					=    (5 * (century % 4) + 2) % 7
			__lowerCAmelCase       					=    year % 100
			__lowerCAmelCase       					=    centurian % 12
			__lowerCAmelCase       					=    (
			    (centurian // 12) + centurian_m + (centurian_m // 4) + century_anchor
			) % 7
			__lowerCAmelCase       					=    (
			    DOOMSDAY_NOT_LEAP[month - 1]
			    if (year % 4 != 0) or (centurian == 0 and (year % 400) == 0)
			    else DOOMSDAY_LEAP[month - 1]
			)
			__lowerCAmelCase       					=    (dooms_day + day - day_anchor) % 7
			return WEEK_DAY_NAMES[week_day]


if __name__ == "__main__":
	import doctest

	doctest.testmod()




