


'''simple docstring'''
import unittest

from transformers.models.xlm_prophetnet.tokenization_xlm_prophetnet import SPIECE_UNDERLINE, XLMProphetNetTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase						=					get_tests_dir("""fixtures/test_sentencepiece.model""")






@require_sentencepiece
class        lowerCAmelCase_  (     UpperCAmelCase      ,							unittest.TestCase	):
       __UpperCAmelCase					=XLMProphetNetTokenizer
       __UpperCAmelCase					=False
       __UpperCAmelCase					=True






       def   UpperCamelCase						( self   )->      List[str]:
            super().setUp()

            # We have a SentencePiece fixture for testing
            _A				=							XLMProphetNetTokenizer(_UpperCamelCase			,     keep_accents=_UpperCamelCase   )
            tokenizer.save_pretrained(self.tmpdirname   )






       def   UpperCamelCase						( self   )->      Dict:
            _A				=							'[PAD]'
            _A				=							0

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(_UpperCamelCase   )			,     _UpperCamelCase   )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(_UpperCamelCase   )			,     _UpperCamelCase   )






       def   UpperCamelCase						( self   )->      Dict:
            _A				=							list(self.get_tokenizer().get_vocab().keys()   )

            self.assertEqual(vocab_keys[0]			,     '[PAD]'   )
            self.assertEqual(vocab_keys[1]			,     '[CLS]'   )
            self.assertEqual(vocab_keys[-1]			,     'j'   )
            self.assertEqual(len(_UpperCamelCase   )			,     1012   )






       def   UpperCamelCase						( self   )->      int:
            self.assertEqual(self.get_tokenizer().vocab_size			,     1012   )






       def   UpperCamelCase						( self   )->      str:
            _A				=							XLMProphetNetTokenizer(_UpperCamelCase			,     keep_accents=_UpperCamelCase   )

            _A				=							tokenizer.tokenize('This is a test'   )
            self.assertListEqual(_UpperCamelCase			,     ['▁This', '▁is', '▁a', '▁t', 'est']   )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(_UpperCamelCase   )			,     [value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]			,     )

            _A				=							tokenizer.tokenize('I was born in 92000, and this is falsé.'   )
            self.assertListEqual(
                _UpperCamelCase			,     [
                    SPIECE_UNDERLINE + 'I',
                    SPIECE_UNDERLINE + 'was',
                    SPIECE_UNDERLINE + 'b',
                    'or',
                    'n',
                    SPIECE_UNDERLINE + 'in',
                    SPIECE_UNDERLINE + '',
                    '9',
                    '2',
                    '0',
                    '0',
                    '0',
                    ',',
                    SPIECE_UNDERLINE + 'and',
                    SPIECE_UNDERLINE + 'this',
                    SPIECE_UNDERLINE + 'is',
                    SPIECE_UNDERLINE + 'f',
                    'al',
                    's',
                    'é',
                    '.',
                ]			,     )
            _A				=							tokenizer.convert_tokens_to_ids(_UpperCamelCase   )
            self.assertListEqual(
                _UpperCamelCase			,     [
                    value + tokenizer.fairseq_offset
                    for value in [8, 21, 84, 55, 24, 19, 7, -9, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, -9, 4]
                ]			,     )

            _A				=							tokenizer.convert_ids_to_tokens(_UpperCamelCase   )
            self.assertListEqual(
                _UpperCamelCase			,     [
                    SPIECE_UNDERLINE + 'I',
                    SPIECE_UNDERLINE + 'was',
                    SPIECE_UNDERLINE + 'b',
                    'or',
                    'n',
                    SPIECE_UNDERLINE + 'in',
                    SPIECE_UNDERLINE + '',
                    '[UNK]',
                    '2',
                    '0',
                    '0',
                    '0',
                    ',',
                    SPIECE_UNDERLINE + 'and',
                    SPIECE_UNDERLINE + 'this',
                    SPIECE_UNDERLINE + 'is',
                    SPIECE_UNDERLINE + 'f',
                    'al',
                    's',
                    '[UNK]',
                    '.',
                ]			,     )






       @cached_property
       def   UpperCamelCase						( self   )->      Any:
            return XLMProphetNetTokenizer.from_pretrained('microsoft/xprophetnet-large-wiki100-cased'   )






       @slow
       def   UpperCamelCase						( self   )->      Optional[int]:
            _A				=							'Hello World!'
            _A				=							[3_5389, 6672, 49, 2]
            self.assertListEqual(_UpperCamelCase			,     self.big_tokenizer.encode(_UpperCamelCase   )   )






       @slow
       def   UpperCamelCase						( self   )->      int:
            # fmt: off
            _A				=							{'input_ids': [[1_1073, 8_2783, 18, 26, 8_2783, 549, 5_1540, 248, 1_7209, 1301, 217, 20, 21_5186, 1325, 147, 1_7209, 1301, 217, 20, 5_6370, 53, 12_2020, 20, 1_6477, 27, 8_7355, 4548, 20, 4728, 7_8392, 17, 15_9969, 18, 26, 2_4491, 629, 15, 538, 2_2704, 5439, 15, 2788, 2_4491, 9885, 15, 4_3534, 605, 15, 814, 1_8403, 3_3200, 29, 15, 4_3534, 2_4458, 1_2410, 111, 2_4966, 8_3669, 9637, 14_4068, 26, 850, 2_2346, 27, 147, 2_4966, 8_3669, 8_3490, 26, 3_9113, 735, 27, 689, 656, 2800, 1339, 4600, 53, 12_2020, 11_5785, 34, 816, 1339, 4_6887, 18, 147, 5_3905, 1951, 4_2238, 4_1170, 1_7732, 834, 436, 15, 2_7523, 9_8733, 217, 147, 5542, 4981, 930, 1_7347, 16, 2], [2_0091, 629, 94, 8_2786, 58, 490, 20, 1528, 84, 5_3905, 344, 8_0592, 11_0128, 1_8822, 5267, 1306, 62, 15_2537, 308, 7997, 401, 12_4427, 549, 3_5442, 225, 109, 1_5055, 2_5748, 147, 7119, 4_3712, 34, 767, 13_5366, 18, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [592, 6_3784, 11_9466, 17, 14_7808, 8_8214, 18, 656, 81, 32, 3296, 1_0280, 16, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=_UpperCamelCase			,     model_name='microsoft/xprophetnet-large-wiki100-cased'			,     revision='1acad1643ddd54a44df6a1b797ada8373685d90e'			,     )









'''simple docstring'''
class        lowerCAmelCase_  :






       def __init__( self			,     _UpperCamelCase   )->      None:
            _A				=							size
            _A				=							[0] * size
            _A				=							[0] * size






       @staticmethod
       def   UpperCamelCase						( _UpperCamelCase   )->      int:
            return index | (index + 1)






       @staticmethod
       def   UpperCamelCase						( _UpperCamelCase   )->      int:
            return (index & (index + 1)) - 1






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase   )->      None:
            _A				=							value
            while index < self.size:
                 _A				=							self.get_prev(_UpperCamelCase   ) + 1
                 if current_left_border == index:
                      _A				=							value
                 else:
                      _A				=							max(_UpperCamelCase			,     _UpperCamelCase			,     _UpperCamelCase   )
                 _A				=							self.get_next(_UpperCamelCase   )






       def   UpperCamelCase						( self			,     _UpperCamelCase			,     _UpperCamelCase   )->      int:
            right -= 1  # Because of right is exclusive
            _A				=							0
            while left <= right:
                 _A				=							self.get_prev(_UpperCamelCase   )
                 if left <= current_left:
                      _A				=							max(_UpperCamelCase			,     self.tree[right]   )
                      _A				=							current_left
                 else:
                      _A				=							max(_UpperCamelCase			,     self.arr[right]   )
                      right -= 1
            return result


if __name__ == "__main__":
    import doctest

    doctest.testmod()





