






import unittest

from transformers.utils.backbone_utils import (
    BackboneMixin,
    get_aligned_output_features_output_indices,
    verify_out_features_out_indices,
)


class     __SCREAMING_SNAKE_CASE					(   unittest.TestCase):






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						['a', 'b', 'c']

												# Defaults to last layer if both are None
												lowerCAmelCase__						,       lowerCAmelCase__    						=						get_aligned_output_features_output_indices(_UpperCamelCase     ,						_UpperCamelCase     ,						_UpperCamelCase   )
												self.assertEqual(_UpperCamelCase     ,						['c']   )
												self.assertEqual(_UpperCamelCase     ,						[2]   )

												# Out indices set to match out features
												lowerCAmelCase__						,       lowerCAmelCase__    						=						get_aligned_output_features_output_indices(['a', 'c']     ,						_UpperCamelCase     ,						_UpperCamelCase   )
												self.assertEqual(_UpperCamelCase     ,						['a', 'c']   )
												self.assertEqual(_UpperCamelCase     ,						[0, 2]   )

												# Out features set to match out indices
												lowerCAmelCase__						,       lowerCAmelCase__    						=						get_aligned_output_features_output_indices(_UpperCamelCase     ,						[0, 2]     ,						_UpperCamelCase   )
												self.assertEqual(_UpperCamelCase     ,						['a', 'c']   )
												self.assertEqual(_UpperCamelCase     ,						[0, 2]   )

												# Out features selected from negative indices
												lowerCAmelCase__						,       lowerCAmelCase__    						=						get_aligned_output_features_output_indices(_UpperCamelCase     ,						[-3, -1]     ,						_UpperCamelCase   )
												self.assertEqual(_UpperCamelCase     ,						['a', 'c']   )
												self.assertEqual(_UpperCamelCase     ,						[-3, -1]   )






						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												# Stage names must be set
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(['a', 'b']     ,						(0, 1)     ,						_UpperCamelCase   )

												# Out features must be a list
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(('a', 'b')     ,						(0, 1)     ,						['a', 'b']   )

												# Out features must be a subset of stage names
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(['a', 'b']     ,						(0, 1)     ,						['a']   )

												# Out indices must be a list or tuple
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(_UpperCamelCase     ,						0     ,						['a', 'b']   )

												# Out indices must be a subset of stage names
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(_UpperCamelCase     ,						(0, 1)     ,						['a']   )

												# Out features and out indices must be the same length
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(['a', 'b']     ,						(0,)     ,						['a', 'b', 'c']   )

												# Out features should match out indices
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(['a', 'b']     ,						(0, 2)     ,						['a', 'b', 'c']   )

												# Out features and out indices should be in order
												with self.assertRaises(_UpperCamelCase   ):
																		verify_out_features_out_indices(['b', 'a']     ,						(0, 1)     ,						['a', 'b']   )

												# Check passes with valid inputs
												verify_out_features_out_indices(['a', 'b', 'd']     ,						(0, 1, -1)     ,						['a', 'b', 'c', 'd']   )







						def       UpperCamelCase__      (	self   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						BackboneMixin()

												lowerCAmelCase__    						=						['a', 'b', 'c']
												lowerCAmelCase__    						=						['a', 'c']
												lowerCAmelCase__    						=						[0, 2]

												# Check that the output features and indices are set correctly
												self.assertEqual(backbone.out_features     ,						['a', 'c']   )
												self.assertEqual(backbone.out_indices     ,						[0, 2]   )

												# Check out features and indices are updated correctly
												lowerCAmelCase__    						=						['a', 'b']
												self.assertEqual(backbone.out_features     ,						['a', 'b']   )
												self.assertEqual(backbone.out_indices     ,						[0, 1]   )

												lowerCAmelCase__    						=						[-3, -1]
												self.assertEqual(backbone.out_features     ,						['a', 'c']   )
												self.assertEqual(backbone.out_indices     ,						[-3, -1]   )













import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


__snake_case    :    Any         =     logging.get_logger(__name__)

__snake_case    :    Tuple         =     {"""tokenizer_file""": """tokenizer.json"""}

__snake_case    :    str         =     {
    """tokenizer_file""": {
        """bigscience/tokenizer""": """https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json""",
        """bigscience/bloom-560m""": """https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json""",
        """bigscience/bloom-1b1""": """https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json""",
        """bigscience/bloom-1b7""": """https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json""",
        """bigscience/bloom-3b""": """https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json""",
        """bigscience/bloom-7b1""": """https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json""",
        """bigscience/bloom""": """https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json""",
    },
}


class     __SCREAMING_SNAKE_CASE					(   __lowercase):
						_SCREAMING_SNAKE_CASE	:       Dict   =					VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					['''input_ids''', '''attention_mask''']
						_SCREAMING_SNAKE_CASE	:       Union[str, Any]   =					None






						def __init__(	self     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase=None     ,						_UpperCamelCase="<unk>"     ,						_UpperCamelCase="<s>"     ,						_UpperCamelCase="</s>"     ,						_UpperCamelCase="<pad>"     ,						_UpperCamelCase=False     ,						_UpperCamelCase=False     ,						**_UpperCamelCase     ,						):






												"""simple docstring"""
												super().__init__(
												    _UpperCamelCase     ,						_UpperCamelCase     ,						tokenizer_file=_UpperCamelCase     ,						unk_token=_UpperCamelCase     ,						bos_token=_UpperCamelCase     ,						eos_token=_UpperCamelCase     ,						pad_token=_UpperCamelCase     ,						add_prefix_space=_UpperCamelCase     ,						clean_up_tokenization_spaces=_UpperCamelCase     ,						**_UpperCamelCase     ,						)
												lowerCAmelCase__    						=						json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
												if pre_tok_state.get('add_prefix_space'     ,						_UpperCamelCase   ) != add_prefix_space:
																		lowerCAmelCase__    						=						getattr(_UpperCamelCase     ,						pre_tok_state.pop('type'   )   )
																		lowerCAmelCase__    						=						add_prefix_space
																		lowerCAmelCase__    						=						pre_tok_class(**_UpperCamelCase   )

												lowerCAmelCase__    						=						add_prefix_space






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )
												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._batch_encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						*_UpperCamelCase     ,						**_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						kwargs.get('is_split_into_words'     ,						_UpperCamelCase   )

												if not (self.add_prefix_space or not is_split_into_words):
																		raise Exception(
																		    F"You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"
																		    ' pretokenized inputs.'   )

												return super()._encode_plus(*_UpperCamelCase     ,						**_UpperCamelCase   )






						def       UpperCamelCase__      (	self     ,						_UpperCamelCase     ,						_UpperCamelCase = None   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						self._tokenizer.model.save(_UpperCamelCase     ,						name=_UpperCamelCase   )
												return tuple(_UpperCamelCase   )







						def       UpperCamelCase__      (	self     ,						_UpperCamelCase   ):






												"""simple docstring"""
												lowerCAmelCase__    						=						[]
												for is_user, text in conversation.iter_texts():
																		input_ids.extend(self.encode(_UpperCamelCase     ,						add_special_tokens=_UpperCamelCase   ) + [self.eos_token_id]   )

												if len(_UpperCamelCase   ) > self.model_max_length:
																		lowerCAmelCase__    						=						input_ids[-self.model_max_length :]
												return input_ids





