






"""simple docstring"""







import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
			if is_torch_available():
						import torch
			if is_tf_available():
						import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


a :				List[str]      					=		logging.get_logger(__name__)

a :				int      					=		{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

a :				int      					=		{
    """vocab_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """Salesforce/codegen-350M-mono""": """https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt""",
    },
}

a :				Dict      					=		{
    """Salesforce/codegen-350M-mono""": 2048,
}





@lru_cache()
def 						lowercase__()				->List[str]:




						"""simple docstring"""



						lowercase__				:      int=       (
						    list(range(ord("!"						)					,		ord("~"						) + 1						)						) + list(range(ord("¡"						)					,		ord("¬"						) + 1						)						) + list(range(ord("®"						)					,		ord("ÿ"						) + 1						)						)
						)
						lowercase__				:      Optional[Any]=       bs[:]
						lowercase__				:      int=       0
						for b in range(2**8						):
												if b not in bs:
																		bs.append(A						)
																		cs.append(2**8 + n						)
																		n += 1
						lowercase__				:      Any=       [chr(A						) for n in cs]
						return dict(zip(A					,		A						)						)





def 						lowercase__(A						)				->List[str]:




						"""simple docstring"""



						lowercase__				:      int=       set()
						lowercase__				:      Tuple=       word[0]
						for char in word[1:]:
												pairs.add((prev_char, char)						)
												lowercase__				:      Dict=       char
						return pairs







class     __UpperCAmelCase(   SCREAMING_SNAKE_CASE__							):




						"""simple docstring"""




						__lowerCamelCase				       =					VOCAB_FILES_NAMES
						__lowerCamelCase				       =					PRETRAINED_VOCAB_FILES_MAP
						__lowerCamelCase				       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						__lowerCamelCase				       =					["input_ids", "attention_mask"]




						def __init__(   self       ,  snake_case__       ,  snake_case__       ,  snake_case__="replace"       ,  snake_case__="<|endoftext|>"       ,  snake_case__="<|endoftext|>"       ,  snake_case__="<|endoftext|>"       ,  snake_case__=None       ,  snake_case__=False       ,  snake_case__=False       ,  **snake_case__       ,  ):
												'''simple docstring'''
												lowercase__				:      Union[str, Any]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else bos_token
												lowercase__				:      Optional[int]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else eos_token
												lowercase__				:      Optional[int]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else unk_token
												lowercase__				:      Union[str, Any]=       AddedToken(snake_case__       ,  lstrip=snake_case__       ,  rstrip=snake_case__			) if isinstance(snake_case__       ,  snake_case__			) else pad_token
												super().__init__(
												    errors=snake_case__       ,  unk_token=snake_case__       ,  bos_token=snake_case__       ,  eos_token=snake_case__       ,  pad_token=snake_case__       ,  add_prefix_space=snake_case__       ,  add_bos_token=snake_case__       ,  **snake_case__       ,  )
												lowercase__				:      Optional[Any]=       add_bos_token

												with open(snake_case__       ,  encoding="utf-8"			) as vocab_handle:
																		lowercase__				:      Optional[int]=       json.load(snake_case__			)
												lowercase__				:      str=       {v: k for k, v in self.encoder.items()}
												lowercase__				:      List[str]=       errors  # how to handle errors in decoding
												lowercase__				:      Union[str, Any]=       bytes_to_unicode()
												lowercase__				:      str=       {v: k for k, v in self.byte_encoder.items()}
												with open(snake_case__       ,  encoding="utf-8"			) as merges_handle:
																		lowercase__				:      Optional[int]=       merges_handle.read().split("\n"			)[1:-1]
												lowercase__				:      Union[str, Any]=       [tuple(merge.split()			) for merge in bpe_merges]
												lowercase__				:      Dict=       dict(zip(snake_case__       ,  range(len(snake_case__			)			)			)			)
												lowercase__				:      Tuple=       {}
												lowercase__				:      str=       add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												lowercase__				:      Tuple=       re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"			)




						@property
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												return len(self.encoder			)




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												return dict(self.encoder       ,  **self.added_tokens_encoder			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												if token in self.cache:
																		return self.cache[token]
												lowercase__				:      Any=       tuple(snake_case__			)
												lowercase__				:      Optional[int]=       get_pairs(snake_case__			)

												if not pairs:
																		return token

												while True:
																		lowercase__				:      Tuple=       min(snake_case__       ,  key=lambda snake_case__			: self.bpe_ranks.get(snake_case__       ,  float("inf"			)			)			)
																		if bigram not in self.bpe_ranks:
																								break
																		lowercase__,	lowercase__				:      str=       bigram
																		lowercase__				:      Optional[Any]=       []
																		lowercase__				:      Optional[Any]=       0
																		while i < len(snake_case__			):
																								try:
																														lowercase__				:      Optional[int]=       word.index(snake_case__       ,  snake_case__			)
																								except ValueError:
																														new_word.extend(word[i:]			)
																														break
																								else:
																														new_word.extend(word[i:j]			)
																														lowercase__				:      List[Any]=       j

																								if word[i] == first and i < len(snake_case__			) - 1 and word[i + 1] == second:
																														new_word.append(first + second			)
																														i += 2
																								else:
																														new_word.append(word[i]			)
																														i += 1
																		lowercase__				:      Tuple=       tuple(snake_case__			)
																		lowercase__				:      Optional[Any]=       new_word
																		if len(snake_case__			) == 1:
																								break
																		else:
																								lowercase__				:      Dict=       get_pairs(snake_case__			)
												lowercase__				:      str=       " ".join(snake_case__			)
												lowercase__				:      List[str]=       word
												return word




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__=None			):
												'''simple docstring'''
												if self.add_bos_token:
																		lowercase__				:      str=       [self.bos_token_id]
												else:
																		lowercase__				:      List[Any]=       []

												lowercase__				:      Optional[int]=       bos_token_ids + token_ids_a

												if token_ids_a is None:
																		return output

												return output + bos_token_ids + token_ids_a




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       []
												for token in re.findall(self.pat       ,  snake_case__			):
																		lowercase__				:      List[str]=       "".join(
																		    self.byte_encoder[b] for b in token.encode("utf-8"			)			)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																		bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case__			).split(" "			)			)
												return bpe_tokens




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												return self.encoder.get(snake_case__       ,  self.encoder.get(self.unk_token			)			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												return self.decoder.get(snake_case__			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												lowercase__				:      Any=       "".join(snake_case__			)
												lowercase__				:      Any=       bytearray([self.byte_decoder[c] for c in text]			).decode("utf-8"       ,  errors=self.errors			)
												return text




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__ = None			):
												'''simple docstring'''
												if not os.path.isdir(snake_case__			):
																		logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
																		return
												lowercase__				:      int=       os.path.join(
												    snake_case__       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]			)
												lowercase__				:      Any=       os.path.join(
												    snake_case__       ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]			)

												with open(snake_case__       ,  "w"       ,  encoding="utf-8"			) as f:
																		f.write(json.dumps(self.encoder       ,  indent=2       ,  sort_keys=snake_case__       ,  ensure_ascii=snake_case__			) + "\n"			)

												lowercase__				:      Any=       0
												with open(snake_case__       ,  "w"       ,  encoding="utf-8"			) as writer:
																		writer.write("#version: 0.2\n"			)
																		for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,  key=lambda snake_case__			: kv[1]			):
																								if index != token_index:
																														logger.warning(
																														    F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																														    " Please check that the tokenizer is not corrupted!"			)
																														lowercase__				:      Union[str, Any]=       token_index
																								writer.write(" ".join(snake_case__			) + "\n"			)
																								index += 1

												return vocab_file, merge_file




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__=False       ,  **snake_case__			):
												'''simple docstring'''
												lowercase__				:      str=       kwargs.pop("add_prefix_space"       ,  self.add_prefix_space			)
												if is_split_into_words or add_prefix_space:
																		lowercase__				:      Dict=       " " + text
												return (text, kwargs)




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__ = False       ,  snake_case__ = None       ,  snake_case__ = None       ,  **snake_case__       ,  ):
												'''simple docstring'''
												lowercase__				:      List[Any]=       super()._decode(
												    token_ids=snake_case__       ,  skip_special_tokens=snake_case__       ,  clean_up_tokenization_spaces=snake_case__       ,  **snake_case__       ,  )

												if truncate_before_pattern is not None and len(snake_case__			) > 0:
																		lowercase__				:      Optional[Any]=       self.truncate(snake_case__       ,  snake_case__			)

												return decoded_text




						def 		UpperCAmelCase_       (   self       ,  snake_case__       ,  snake_case__			):
												'''simple docstring'''
												def find_re(snake_case__       ,  snake_case__       ,  snake_case__			):
																		lowercase__				:      Optional[Any]=       pattern.search(snake_case__       ,  snake_case__			)
																		return m.start() if m else -1

												lowercase__				:      List[Any]=       [re.compile(snake_case__       ,  re.MULTILINE			) for pattern in truncate_before_pattern]

												lowercase__				:      Union[str, Any]=       list(re.finditer("^print"       ,  snake_case__       ,  re.MULTILINE			)			)

												if len(snake_case__			) > 1:
																		lowercase__				:      Union[str, Any]=       completion[: prints[1].start()]

												lowercase__				:      Tuple=       list(re.finditer("^def"       ,  snake_case__       ,  re.MULTILINE			)			)

												if len(snake_case__			) > 1:
																		lowercase__				:      Optional[Any]=       completion[: defs[1].start()]

												lowercase__				:      Dict=       0

												lowercase__				:      Optional[Any]=       [
												    pos for pos in [find_re(snake_case__       ,  snake_case__       ,  snake_case__			) for terminal in terminals] if pos != -1
												]

												if len(snake_case__			) > 0:
																		return completion[: min(snake_case__			)]
												else:
																		return completion














"""simple docstring"""







import unittest

from transformers import is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device


if is_torch_available():
			import torch

			from transformers import XLMProphetNetForConditionalGeneration, XLMProphetNetTokenizer


@require_torch
class     __UpperCAmelCase(   unittest.TestCase							):




						"""simple docstring"""




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Optional[int]=       XLMProphetNetForConditionalGeneration.from_pretrained("microsoft/xprophetnet-large-wiki100-cased"			)
												model.to(snake_case__			)

												# encoder-decoder outputs
												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      Optional[Any]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      Dict=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      int=       output[0]
												lowercase__				:      str=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-6.39_86, -8.23_91, 12.51_89], [-6.32_89, -8.08_64, 12.62_11], [-6.24_18, -8.04_45, 12.79_68]]]			).to(snake_case__			)
												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# encoder outputs
												lowercase__				:      Tuple=       model.prophetnet.encoder(snake_case__			)[0]
												lowercase__				:      Union[str, Any]=       torch.tensor(
												    [[[-1.42_60, -0.76_28, 0.84_53], [-1.47_19, -0.13_91, 0.78_07], [-1.76_78, 0.01_14, 0.46_46]]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       torch.Size((1, 4, 1024)			)
												self.assertEqual(encoder_outputs.shape       ,  snake_case__			)
												self.assertTrue(torch.allclose(encoder_outputs[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)

												# decoder outputs
												lowercase__				:      int=       model.prophetnet.decoder(
												    snake_case__       ,  encoder_hidden_states=snake_case__       ,  )
												lowercase__				:      Union[str, Any]=       decoder_outputs[1].view(1       ,  model.config.ngram       ,  14       ,  -1			)
												lowercase__				:      List[Any]=       model.lm_head(snake_case__			)
												lowercase__				:      str=       predicting_streams_logits[:, 0]
												self.assertTrue(torch.allclose(next_first_stream_logits[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Union[str, Any]=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)

												lowercase__				:      Dict=       torch.tensor([[17, 96208, 103471, 2]]			).to(snake_case__			)
												lowercase__				:      List[str]=       torch.tensor(
												    [[2, 250, 9953, 34, 69489, 1620, 32, 118424, 624, 210, 105, 2913, 1032, 351]]			).to(snake_case__			)
												lowercase__				:      List[Any]=       model(
												    input_ids=snake_case__       ,  attention_mask=snake_case__       ,  encoder_outputs=snake_case__       ,  decoder_input_ids=snake_case__			)
												lowercase__				:      str=       output[0]
												lowercase__				:      Optional[Any]=       torch.Size((1, 14, 250012)			)
												self.assertEqual(output_predited_logis.shape       ,  snake_case__			)
												# compare the actual values for a slice.
												lowercase__				:      str=       torch.tensor(
												    [[[-9.22_53, -9.71_73, -6.35_29], [-7.67_01, -9.01_45, -1.93_82], [-8.01_95, -7.00_04, -0.15_23]]]			).to(snake_case__			)

												self.assertTrue(torch.allclose(output_predited_logis[:, :3, :3]       ,  snake_case__       ,  atol=1e-4			)			)




						@slow
						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												lowercase__				:      Tuple=       XLMProphetNetForConditionalGeneration.from_pretrained(
												    "microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)
												model.to(snake_case__			)
												lowercase__				:      Optional[int]=       512

												lowercase__				:      int=       XLMProphetNetTokenizer.from_pretrained("microsoft/xprophetnet-large-wiki100-cased-xglue-ntg"			)

												lowercase__				:      str=       (
												    "Microsoft Corporation intends to officially end free support for the Windows 7 operating system after"
												    " January 14, 2020, according to the official portal of the organization. From that day, users of this"
												    " system will not be able to receive security updates, which could make their computers vulnerable to"
												    " cyber attacks."
												)
												lowercase__				:      Tuple=       (
												    "орпорация Microsoft намерена официально прекратить бесплатную поддержку операционной системы Windows 7"
												    " после 14 января 2020 года, сообщается на официальном портале организации . С указанного дня пользователи"
												    " этой системы не смогут получать обновления безопасности, из-за чего их компьютеры могут стать уязвимыми"
												    " к кибератакам."
												)
												lowercase__				:      Optional[int]=       (
												    "根据该组织的官方门户网站，微软公司打算在2020年1月14日之后正式终止对Windows 7操作系统的免费支持。从那时起，该系统的用户将无法接收安全更新，这可能会使他们的计算机容易受到网络攻击。"
												)

												lowercase__				:      Dict=       tokenizer(
												    [EN_SENTENCE, RU_SENTENCE, ZH_SENTENCE]       ,  padding=snake_case__       ,  max_length=255       ,  return_tensors="pt"			).input_ids
												lowercase__				:      Any=       input_ids.to(snake_case__			)

												lowercase__				:      List[Any]=       model.generate(
												    snake_case__       ,  num_beams=10       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      str=       [tokenizer.decode(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids]
												lowercase__				:      Dict=       "Microsoft to end Windows 7 free support after January 14, 2020"
												lowercase__				:      int=       "Microsoft намерена прекратить бесплатную поддержку Windows 7 после 14 января 2020 года"
												lowercase__				:      List[str]=       "微软打算终止对Windows 7操作系统的免费支持"
												self.assertListEqual(
												    [EXPECTED_TITLE_EN, EXPECTED_TITLE_RU, EXPECTED_TITLE_ZH]       ,  snake_case__       ,  )

												lowercase__				:      List[str]=       model.generate(
												    snake_case__       ,  num_beams=1       ,  length_penalty=1.0       ,  no_repeat_ngram_size=3       ,  early_stopping=snake_case__			)
												lowercase__				:      Optional[Any]=       [
												    tokenizer.convert_ids_to_tokens(snake_case__       ,  skip_special_tokens=snake_case__			) for g in summary_ids_beama
												]
												lowercase__				:      List[str]=       "▁Microsoft ▁to ▁end ▁free ▁support ▁for ▁Windows ▁7".split(" "			)
												lowercase__				:      List[Any]=       "▁Microsoft ▁намерен а ▁прекрати ть ▁бес плат ную ▁поддержку ▁Windows ▁7 ▁после ▁14 ▁января ▁2020 ▁года".split(
												    " "			)
												lowercase__				:      int=       "微软 公司 打算 终止 对 Windows ▁7 操作 系统的 免费 支持".split(" "			)
												self.assertListEqual(
												    [EXPECTED_TITLE_EN_BEAM1_TOK, EXPECTED_TITLE_RU_BEAM1_TOK, EXPECTED_TITLE_ZH_BEAM1_TOK]       ,  snake_case__       ,  )






