






import itertools
import json
import os
import unittest

from transformers import AddedToken, RobertaTokenizer, RobertaTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


@require_tokenizers
class     __SCREAMING_SNAKE_CASE					(   __lowercase		,					unittest.TestCase):
						_SCREAMING_SNAKE_CASE	:       str   =					RobertaTokenizer
						_SCREAMING_SNAKE_CASE	:       List[str]   =					RobertaTokenizerFast
						_SCREAMING_SNAKE_CASE	:       Tuple   =					True
						_SCREAMING_SNAKE_CASE	:       Tuple   =					{'''cls_token''': '''<s>'''}






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												super().setUp()

												# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
												lowerCAmelCase__    						=						[
												    'l',
												    'o',
												    'w',
												    'e',
												    'r',
												    's',
												    't',
												    'i',
												    'd',
												    'n',
												    '\u0120',
												    '\u0120l',
												    '\u0120n',
												    '\u0120lo',
												    '\u0120low',
												    'er',
												    '\u0120lowest',
												    '\u0120newer',
												    '\u0120wider',
												    '<unk>',
												]
												lowerCAmelCase__    						=						dict(zip(_UpperCamelCase     ,						range(len(_UpperCamelCase   )   )   )   )
												lowerCAmelCase__    						=						['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
												lowerCAmelCase__    						=						{'unk_token': '<unk>'}

												lowerCAmelCase__    						=						os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES['vocab_file']   )
												lowerCAmelCase__    						=						os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES['merges_file']   )
												with open(self.vocab_file     ,						'w'     ,						encoding='utf-8'   ) as fp:
																		fp.write(json.dumps(_UpperCamelCase   ) + '\n'   )
												with open(self.merges_file     ,						'w'     ,						encoding='utf-8'   ) as fp:
																		fp.write('\n'.join(_UpperCamelCase   )   )






						def       UpperCamelCase__      (	self     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												kwargs.update(self.special_tokens_map   )
												return self.tokenizer_class.from_pretrained(self.tmpdirname     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												kwargs.update(self.special_tokens_map   )
												return RobertaTokenizerFast.from_pretrained(self.tmpdirname     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						'lower newer'
												lowerCAmelCase__    						=						'lower newer'
												return input_text, output_text






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.tokenizer_class(self.vocab_file     ,						self.merges_file     ,						**self.special_tokens_map   )
												lowerCAmelCase__    						=						'lower newer'
												lowerCAmelCase__    						=						['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
												lowerCAmelCase__    						=						tokenizer.tokenize(_UpperCamelCase   )  # , add_prefix_space=True)
												self.assertListEqual(_UpperCamelCase     ,						_UpperCamelCase   )

												lowerCAmelCase__    						=						tokens + [tokenizer.unk_token]
												lowerCAmelCase__    						=						[0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
												self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase   )     ,						_UpperCamelCase   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.get_tokenizer()

												self.assertListEqual(tokenizer.encode('Hello world!'     ,						add_special_tokens=_UpperCamelCase   )     ,						[0, 3_14_14, 2_32, 3_28, 2]   )
												self.assertListEqual(
												    tokenizer.encode('Hello world! cécé herlolip 418'     ,						add_special_tokens=_UpperCamelCase   )     ,						[0, 3_14_14, 2_32, 3_28, 7_40, 11_40, 1_26_95, 69, 4_60_78, 15_88, 2]     ,						)






						@slow
						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.tokenizer_class.from_pretrained('roberta-base'   )

												lowerCAmelCase__    						=						tokenizer.encode('sequence builders'     ,						add_special_tokens=_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.encode('multi-sequence build'     ,						add_special_tokens=_UpperCamelCase   )

												lowerCAmelCase__    						=						tokenizer.encode(
												    'sequence builders'     ,						add_special_tokens=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.encode(
												    'sequence builders'     ,						'multi-sequence build'     ,						add_special_tokens=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase   )

												lowerCAmelCase__    						=						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.build_inputs_with_special_tokens(_UpperCamelCase     ,						_UpperCamelCase   )

												assert encoded_sentence == encoded_text_from_decode
												assert encoded_pair == encoded_pair_from_decode






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self.get_tokenizer()

												lowerCAmelCase__    						=						'Encode this sequence.'
												lowerCAmelCase__    						=						tokenizer.byte_encoder[' '.encode('utf-8'   )[0]]

												# Testing encoder arguments
												lowerCAmelCase__    						=						tokenizer.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
												self.assertNotEqual(_UpperCamelCase     ,						_UpperCamelCase   )

												lowerCAmelCase__    						=						tokenizer.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.convert_ids_to_tokens(encoded[0]   )[0]
												self.assertEqual(_UpperCamelCase     ,						_UpperCamelCase   )

												tokenizer.add_special_tokens({'bos_token': '<s>'}   )
												lowerCAmelCase__    						=						tokenizer.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.convert_ids_to_tokens(encoded[1]   )[0]
												self.assertNotEqual(_UpperCamelCase     ,						_UpperCamelCase   )

												# Testing spaces after special tokens
												lowerCAmelCase__    						=						'<mask>'
												tokenizer.add_special_tokens(
												    {'mask_token': AddedToken(_UpperCamelCase     ,						lstrip=_UpperCamelCase     ,						rstrip=_UpperCamelCase   )}   )  # mask token has a left space
												lowerCAmelCase__    						=						tokenizer.convert_tokens_to_ids(_UpperCamelCase   )

												lowerCAmelCase__    						=						'Encode <mask> sequence'
												lowerCAmelCase__    						=						'Encode <mask>sequence'

												lowerCAmelCase__    						=						tokenizer.encode(_UpperCamelCase   )
												lowerCAmelCase__    						=						encoded.index(_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
												self.assertEqual(_UpperCamelCase     ,						_UpperCamelCase   )

												lowerCAmelCase__    						=						tokenizer.encode(_UpperCamelCase   )
												lowerCAmelCase__    						=						encoded.index(_UpperCamelCase   )
												lowerCAmelCase__    						=						tokenizer.convert_ids_to_tokens(encoded[mask_loc + 1]   )[0]
												self.assertNotEqual(_UpperCamelCase     ,						_UpperCamelCase   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												pass






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(_UpperCamelCase     ,						**_UpperCamelCase   )
																								lowerCAmelCase__    						=						self.tokenizer_class.from_pretrained(_UpperCamelCase     ,						**_UpperCamelCase   )
																								lowerCAmelCase__    						=						'A, <mask> AllenNLP sentence.'
																								lowerCAmelCase__    						=						tokenizer_r.encode_plus(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase     ,						return_token_type_ids=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_p.encode_plus(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase     ,						return_token_type_ids=_UpperCamelCase   )

																								# token_type_ids should put 0 everywhere
																								self.assertEqual(sum(tokens_r['token_type_ids']   )     ,						sum(tokens_p['token_type_ids']   )   )

																								# attention_mask should put 1 everywhere, so sum over length should be 1
																								self.assertEqual(
																								    sum(tokens_r['attention_mask']   ) / len(tokens_r['attention_mask']   )     ,						sum(tokens_p['attention_mask']   ) / len(tokens_p['attention_mask']   )     ,						)

																								lowerCAmelCase__    						=						tokenizer_r.convert_ids_to_tokens(tokens_r['input_ids']   )
																								lowerCAmelCase__    						=						tokenizer_p.convert_ids_to_tokens(tokens_p['input_ids']   )

																								# Rust correctly handles the space before the mask while python doesnt
																								self.assertSequenceEqual(tokens_p['input_ids']     ,						[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )
																								self.assertSequenceEqual(tokens_r['input_ids']     ,						[0, 2_50, 6, 5_02_64, 38_23, 4_87, 2_19_92, 36_45, 4, 2]   )

																								self.assertSequenceEqual(
																								    _UpperCamelCase     ,						['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']   )
																								self.assertSequenceEqual(
																								    _UpperCamelCase     ,						['<s>', 'A', ',', '<mask>', 'ĠAllen', 'N', 'LP', 'Ġsentence', '.', '</s>']   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												for trim_offsets, add_prefix_space in itertools.product([True, False]     ,						repeat=2   ):
																		lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																		    self.tmpdirname     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )

																		lowerCAmelCase__    						=						json.loads(tokenizer_r.backend_tokenizer.pre_tokenizer.__getstate__()   )
																		lowerCAmelCase__    						=						json.loads(tokenizer_r.backend_tokenizer.post_processor.__getstate__()   )

																		self.assertEqual(pre_tokenizer_state['add_prefix_space']     ,						_UpperCamelCase   )

																		self.assertEqual(post_processor_state['add_prefix_space']     ,						_UpperCamelCase   )
																		self.assertEqual(post_processor_state['trim_offsets']     ,						_UpperCamelCase   )







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												# Test which aims to verify that the offsets are well adapted to the argument `add_prefix_space` and
												# `trim_offsets`
												for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																		with self.subTest(F"{tokenizer.__class__.__name__} ({pretrained_name})"   ):
																								lowerCAmelCase__    						=						'hello'  # `hello` is a token in the vocabulary of `pretrained_name`
																								lowerCAmelCase__    						=						F"{text_of_1_token} {text_of_1_token}"

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(len(_UpperCamelCase   ) + 1, len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(len(_UpperCamelCase   ) + 1, len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(len(_UpperCamelCase   ), len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(len(_UpperCamelCase   ), len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						F" {text}"

																								# tokenizer_r = self.rust_tokenizer_class.from_pretrained(
																								#     pretrained_name, use_fast=True, add_prefix_space=True, trim_offsets=True
																								# )
																								# encoding = tokenizer_r(text, return_offsets_mapping=True, add_special_tokens=False)
																								# self.assertEqual(encoding.offset_mapping[0], (1, 1 + len(text_of_1_token)))
																								# self.assertEqual(
																								#     encoding.offset_mapping[1],
																								#     (1 + len(text_of_1_token) + 1, 1 + len(text_of_1_token) + 1 + len(text_of_1_token)),
																								# )

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(1, 1 + len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(1 + len(_UpperCamelCase   ) + 1, 1 + len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, 1 + len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(1 + len(_UpperCamelCase   ), 1 + len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)

																								lowerCAmelCase__    						=						self.rust_tokenizer_class.from_pretrained(
																								    _UpperCamelCase     ,						use_fast=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						trim_offsets=_UpperCamelCase   )
																								lowerCAmelCase__    						=						tokenizer_r(_UpperCamelCase     ,						return_offsets_mapping=_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   )
																								self.assertEqual(encoding.offset_mapping[0]     ,						(0, 1 + len(_UpperCamelCase   ))   )
																								self.assertEqual(
																								    encoding.offset_mapping[1]     ,						(1 + len(_UpperCamelCase   ), 1 + len(_UpperCamelCase   ) + 1 + len(_UpperCamelCase   ))     ,						)













import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


__snake_case    :    Any         =     logging.get_logger(__name__)

__snake_case    :    Tuple         =     {"""tokenizer_file""": """tokenizer.json"""}

__snake_case    :    str         =     {
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}


class     __SCREAMING_SNAKE_CASE					(   __lowercase):
						_SCREAMING_SNAKE_CASE	:       Dict   =					VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					['''input_ids''', '''attention_mask''']
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					None






						def __init__(	self     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase="<unk>"     ,						_UpperCamelCase="<s>"     ,						_UpperCamelCase="</s>"     ,						_UpperCamelCase="<pad>"     ,						_UpperCamelCase=False     ,						_UpperCamelCase=False     ,						**_UpperCamelCase     ,						):






												"""simple docstring"""
												super().__init__(
												    _UpperCamelCase     ,						_UpperCamelCase     ,						tokenizer_file=_UpperCamelCase     ,						unk_token=_UpperCamelCase     ,						bos_token=_UpperCamelCase     ,						eos_token=_UpperCamelCase     ,						pad_token=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						clean_up_tokenization_spaces=_UpperCamelCase     ,						**_UpperCamelCase     ,						)
												lowerCAmelCase__    						=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
												if pre_tok_state.get('add_prefix_space'     ,						_UpperCamelCase   ) != add_prefix_space:
																		lowerCAmelCase__    						=						getattr(_UpperCamelCase     ,						pre_tok_state.pop('type'   )   )
																		lowerCAmelCase__    						=						add_prefix_space
																		lowerCAmelCase__    						=						pre_tok_class(**_UpperCamelCase   )

												lowerCAmelCase__    						=						add_prefix_space






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )
												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._batch_encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )

												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase = None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self._tokenizer.model.save(_UpperCamelCase     ,						name=_UpperCamelCase   )
												return tuple(_UpperCamelCase   )







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						[]
												for is_user, text in conversation.iter_texts():
																		input_ids.extend(self.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   ) + [self.eos_token_id]   )

												if len(_UpperCamelCase   ) > self.model_max_length:
																		lowerCAmelCase__    						=						input_ids[-self.model_max_length :]
												return input_ids





