

import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


lowerCAmelCase     :   List[Any]						     =  get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
    from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowerCAmelCase     :   str						     =  25_60_47
lowerCAmelCase     :   List[str]						     =  25_61_45
@require_sentencepiece
@require_tokenizers
class  _A		(  __magic_name__   ,     unittest.TestCase):
  SCREAMING_SNAKE_CASE    :      Tuple       =  NllbTokenizer
  SCREAMING_SNAKE_CASE    :      Dict       =  NllbTokenizerFast
  SCREAMING_SNAKE_CASE    :      List[str]       =  True
  SCREAMING_SNAKE_CASE    :      Any       =  True
  SCREAMING_SNAKE_CASE    :      List[Any]       =  {}

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        super().setUp()

        # We have a SentencePiece fixture for testing
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     NllbTokenizer(_SCREAMING_SNAKE_CASE ,					keep_accents=_SCREAMING_SNAKE_CASE					)
        tokenizer.save_pretrained(self.tmpdirname					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[str]           =     NllbTokenizer(_SCREAMING_SNAKE_CASE ,					keep_accents=_SCREAMING_SNAKE_CASE					)

        SCREAMING_SNAKE_CASE_				:  Any           =     tokenizer.tokenize('This is a test'					)
        self.assertListEqual(_SCREAMING_SNAKE_CASE ,					['▁This', '▁is', '▁a', '▁t', 'est']					)

        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					) ,					[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]] ,					)

        SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.tokenize('I was born in 92000, and this is falsé.'					)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE ,					[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '9',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                'é',
                '.',
            ] ,					)
        SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE ,					[
                value + tokenizer.fairseq_offset
                for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
            ] ,					)

        SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE					)
        self.assertListEqual(
            _SCREAMING_SNAKE_CASE ,					[
                SPIECE_UNDERLINE + 'I',
                SPIECE_UNDERLINE + 'was',
                SPIECE_UNDERLINE + 'b',
                'or',
                'n',
                SPIECE_UNDERLINE + 'in',
                SPIECE_UNDERLINE + '',
                '<unk>',
                '2',
                '0',
                '0',
                '0',
                ',',
                SPIECE_UNDERLINE + 'and',
                SPIECE_UNDERLINE + 'this',
                SPIECE_UNDERLINE + 'is',
                SPIECE_UNDERLINE + 'f',
                'al',
                's',
                '<unk>',
                '.',
            ] ,					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     (self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     self.rust_tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  str           =     self.tokenizer_class.from_pretrained(_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     tempfile.mkdtemp()

                    SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Checks it save with the same files + the tokenizer.json file for the fast one
                    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files					)					)
                    SCREAMING_SNAKE_CASE_				:  int           =     tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f					)
                    self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

                    # Checks everything loads correctly in the same way
                    SCREAMING_SNAKE_CASE_				:  int           =     tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)					)

                    shutil.rmtree(_SCREAMING_SNAKE_CASE					)

                    # Save tokenizer rust, legacy_format=True
                    SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     tempfile.mkdtemp()

                    SCREAMING_SNAKE_CASE_				:  Tuple           =     tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,					legacy_format=_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Checks it save with the same files
                    self.assertSequenceEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

                    # Checks everything loads correctly in the same way
                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)					)

                    shutil.rmtree(_SCREAMING_SNAKE_CASE					)

                    # Save tokenizer rust, legacy_format=False
                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     tempfile.mkdtemp()

                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     tokenizer_r.save_pretrained(_SCREAMING_SNAKE_CASE ,					legacy_format=_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tokenizer_p.save_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Checks it saved the tokenizer.json file
                    self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files					)					)

                    # Checks everything loads correctly in the same way
                    SCREAMING_SNAKE_CASE_				:  Optional[int]           =     tokenizer_r.from_pretrained(_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tokenizer_p.from_pretrained(_SCREAMING_SNAKE_CASE					)

                    # Check special tokens are set accordingly on Rust and Python
                    for key in tokenizer_pp.special_tokens_map:
                          self.assertTrue(hasattr(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)					)

                    shutil.rmtree(_SCREAMING_SNAKE_CASE					)

  @require_torch
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        if not self.test_seqaseq:
              return

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     self.get_tokenizers()
        for tokenizer in tokenizers:
              with self.subTest(f"{tokenizer.__class__.__name__}"					):
                    # Longer text that will definitely require truncation.
                    SCREAMING_SNAKE_CASE_				:  Dict           =     [
                        ' UN Chief Says There Is No Military Solution in Syria',
                        ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                        ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                        ' will only worsen the violence and misery for millions of people.',
                    ]
                    SCREAMING_SNAKE_CASE_				:  Any           =     [
                        'Şeful ONU declară că nu există o soluţie militară în Siria',
                        'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                        ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                        ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                    ]
                    try:
                          SCREAMING_SNAKE_CASE_				:  int           =     tokenizer.prepare_seqaseq_batch(
                              src_texts=_SCREAMING_SNAKE_CASE ,					tgt_texts=_SCREAMING_SNAKE_CASE ,					max_length=3 ,					max_target_length=10 ,					return_tensors='pt' ,					src_lang='eng_Latn' ,					tgt_lang='ron_Latn' ,					)
                    except NotImplementedError:
                          return
                    self.assertEqual(batch.input_ids.shape[1] ,					3					)
                    self.assertEqual(batch.labels.shape[1] ,					10					)
                    # max_target_length will default to max_length if not specified
                    SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer.prepare_seqaseq_batch(
                        _SCREAMING_SNAKE_CASE ,					tgt_texts=_SCREAMING_SNAKE_CASE ,					max_length=3 ,					return_tensors='pt'					)
                    self.assertEqual(batch.input_ids.shape[1] ,					3					)
                    self.assertEqual(batch.labels.shape[1] ,					3					)

                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer.prepare_seqaseq_batch(
                        src_texts=_SCREAMING_SNAKE_CASE ,					max_length=3 ,					max_target_length=10 ,					return_tensors='pt'					)
                    self.assertEqual(batch_encoder_only.input_ids.shape[1] ,					3					)
                    self.assertEqual(batch_encoder_only.attention_mask.shape[1] ,					3					)
                    self.assertNotIn('decoder_input_ids' ,					_SCREAMING_SNAKE_CASE					)

  @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'					)
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        pass



  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f"{tokenizer.__class__.__name__} ({pretrained_name})"					):
                    SCREAMING_SNAKE_CASE_				:  Tuple           =     [AddedToken('<special>' ,					lstrip=_SCREAMING_SNAKE_CASE					)]

                    SCREAMING_SNAKE_CASE_				:  Dict           =     self.rust_tokenizer_class.from_pretrained(
                        _SCREAMING_SNAKE_CASE ,					additional_special_tokens=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)
                    SCREAMING_SNAKE_CASE_				:  str           =     tokenizer_r.encode('Hey this is a <special> token'					)

                    SCREAMING_SNAKE_CASE_				:  List[str]           =     tokenizer_r.encode('<special>' ,					add_special_tokens=_SCREAMING_SNAKE_CASE					)[0]

                    self.assertTrue(special_token_id in r_output					)

                    if self.test_slow_tokenizer:
                          SCREAMING_SNAKE_CASE_				:  int           =     self.rust_tokenizer_class.from_pretrained(
                              _SCREAMING_SNAKE_CASE ,					additional_special_tokens=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE ,					)
                          SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     self.tokenizer_class.from_pretrained(
                              _SCREAMING_SNAKE_CASE ,					additional_special_tokens=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

                          SCREAMING_SNAKE_CASE_				:  Dict           =     tokenizer_p.encode('Hey this is a <special> token'					)

                          SCREAMING_SNAKE_CASE_				:  int           =     tokenizer_cr.encode('Hey this is a <special> token'					)

                          self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
                          self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
                          self.assertTrue(special_token_id in p_output					)
                          self.assertTrue(special_token_id in cr_output					)

@require_torch
@require_sentencepiece
@require_tokenizers
class  _A		(  unittest.TestCase):
  SCREAMING_SNAKE_CASE    :      str       =  '''facebook/nllb-200-distilled-600M'''
  SCREAMING_SNAKE_CASE    :      Tuple       =  [
      ''' UN Chief Says There Is No Military Solution in Syria''',
      ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.''',
  ]
  SCREAMING_SNAKE_CASE    :      List[str]       =  [
      '''Şeful ONU declară că nu există o soluţie militară în Siria''',
      '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'''
      ''' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'''
      ''' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
  ]
  SCREAMING_SNAKE_CASE    :      str       =  [
      256_047,
      16_297,
      134_408,
      8_165,
      248_066,
      14_734,
      950,
      1_135,
      105_721,
      3_573,
      83,
      27_352,
      108,
      49_486,
      2,
  ]

  @classmethod
  def  UpperCAmelCase    (				cls					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  NllbTokenizer           =     NllbTokenizer.from_pretrained(
            cls.checkpoint_name ,					src_lang='eng_Latn' ,					tgt_lang='ron_Latn'					)
        SCREAMING_SNAKE_CASE_				:  Any           =     1
        return cls

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab'] ,					25_6001					)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn'] ,					25_6002					)
        self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn'] ,					25_6057					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Any           =     self.tokenizer.batch_encode_plus(self.src_text					).input_ids[0]
        self.assertListEqual(self.expected_src_tokens ,					_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        self.assertIn(_SCREAMING_SNAKE_CASE ,					self.tokenizer.all_special_ids					)
        # fmt: off
        SCREAMING_SNAKE_CASE_				:  Dict           =     [RO_CODE, 4254, 9_8068, 11_2923, 3_9072, 3909, 713, 10_2767, 26, 1_7314, 3_5642, 1_4683, 3_3118, 2022, 6_6987, 2, 25_6047]
        # fmt: on

        SCREAMING_SNAKE_CASE_				:  Dict           =     self.tokenizer.decode(_SCREAMING_SNAKE_CASE ,					skip_special_tokens=_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     self.tokenizer.decode(generated_ids[1:] ,					skip_special_tokens=_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)
        self.assertNotIn(self.tokenizer.eos_token ,					_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     ['this is gunna be a long sentence ' * 20]
        assert isinstance(src_text[0] ,					_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     10
        SCREAMING_SNAKE_CASE_				:  str           =     self.tokenizer(_SCREAMING_SNAKE_CASE ,					max_length=_SCREAMING_SNAKE_CASE ,					truncation=_SCREAMING_SNAKE_CASE					).input_ids[0]
        self.assertEqual(ids[-1] ,					2					)
        self.assertEqual(ids[0] ,					_SCREAMING_SNAKE_CASE					)
        self.assertEqual(len(_SCREAMING_SNAKE_CASE					) ,					_SCREAMING_SNAKE_CASE					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']					) ,					[25_6203, 3]					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     tempfile.mkdtemp()
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     self.tokenizer.fairseq_tokens_to_ids
        self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE					)
        SCREAMING_SNAKE_CASE_				:  int           =     NllbTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE					)
        self.assertDictEqual(new_tok.fairseq_tokens_to_ids ,					_SCREAMING_SNAKE_CASE					)

  @require_torch
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     self.tokenizer(
            self.src_text ,					text_target=self.tgt_text ,					padding=_SCREAMING_SNAKE_CASE ,					truncation=_SCREAMING_SNAKE_CASE ,					max_length=len(self.expected_src_tokens					) ,					return_tensors='pt' ,					)
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     shift_tokens_right(
            batch['labels'] ,					self.tokenizer.pad_token_id ,					self.tokenizer.lang_code_to_id['ron_Latn']					)

        self.assertIsInstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					)

        self.assertEqual((2, 15) ,					batch.input_ids.shape					)
        self.assertEqual((2, 15) ,					batch.attention_mask.shape					)
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     batch.input_ids.tolist()[0]
        self.assertListEqual(self.expected_src_tokens ,					_SCREAMING_SNAKE_CASE					)
        self.assertEqual(_SCREAMING_SNAKE_CASE ,					batch.decoder_input_ids[0, 0]					)  # EOS
        # Test that special tokens are reset
        self.assertEqual(self.tokenizer.prefix_tokens ,					[EN_CODE]					)
        self.assertEqual(self.tokenizer.suffix_tokens ,					[self.tokenizer.eos_token_id]					)

  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  str           =     self.tokenizer(self.src_text ,					padding=_SCREAMING_SNAKE_CASE ,					truncation=_SCREAMING_SNAKE_CASE ,					max_length=3 ,					return_tensors='pt'					)
        SCREAMING_SNAKE_CASE_				:  List[str]           =     self.tokenizer(
            text_target=self.tgt_text ,					padding=_SCREAMING_SNAKE_CASE ,					truncation=_SCREAMING_SNAKE_CASE ,					max_length=10 ,					return_tensors='pt'					)
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     targets['input_ids']
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     shift_tokens_right(
            _SCREAMING_SNAKE_CASE ,					self.tokenizer.pad_token_id ,					decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang] ,					)

        self.assertEqual(batch.input_ids.shape[1] ,					3					)
        self.assertEqual(batch.decoder_input_ids.shape[1] ,					10					)

  @require_torch
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  Any           =     self.tokenizer._build_translation_inputs(
            'A test' ,					return_tensors='pt' ,					src_lang='eng_Latn' ,					tgt_lang='fra_Latn'					)

        self.assertEqual(
            nested_simplify(_SCREAMING_SNAKE_CASE					) ,					{
                # A, test, EOS, en_XX
                'input_ids': [[25_6047, 70, 7356, 2]],
                'attention_mask': [[1, 1, 1, 1]],
                # ar_AR
                'forced_bos_token_id': 25_6057,
            } ,					)



  @require_torch
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        SCREAMING_SNAKE_CASE_				:  List[Any]           =     True
        SCREAMING_SNAKE_CASE_				:  Dict           =     self.tokenizer(
            'UN Chief says there is no military solution in Syria' ,					src_lang='eng_Latn' ,					tgt_lang='fra_Latn'					)
        self.assertEqual(
            inputs.input_ids ,					[1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2, 25_6047]					)

        SCREAMING_SNAKE_CASE_				:  List[Any]           =     False
        SCREAMING_SNAKE_CASE_				:  Any           =     self.tokenizer(
            'UN Chief says there is no military solution in Syria' ,					src_lang='eng_Latn' ,					tgt_lang='fra_Latn'					)
        self.assertEqual(
            inputs.input_ids ,					[25_6047, 1_6297, 13_4408, 2_5653, 6370, 248, 254, 10_3929, 9_4995, 108, 4_9486, 2]					)









from collections import OrderedDict
from typing import Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging
from ..auto import CONFIG_MAPPING


lowerCAmelCase     :   Dict						     =  logging.get_logger(__name__)

lowerCAmelCase     :   str						     =  {
    'microsoft/table-transformer-detection': (
        'https://huggingface.co/microsoft/table-transformer-detection/resolve/main/config.json'
    ),
}
class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      Union[str, Any]       =  '''table-transformer'''
  SCREAMING_SNAKE_CASE    :      Any       =  ['''past_key_values''']
  SCREAMING_SNAKE_CASE    :      Optional[int]       =  {
      '''hidden_size''': '''d_model''',
      '''num_attention_heads''': '''encoder_attention_heads''',
  }

  def __init__(				self ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=None ,					_SCREAMING_SNAKE_CASE=3 ,					_SCREAMING_SNAKE_CASE=100 ,					_SCREAMING_SNAKE_CASE=6 ,					_SCREAMING_SNAKE_CASE=2048 ,					_SCREAMING_SNAKE_CASE=8 ,					_SCREAMING_SNAKE_CASE=6 ,					_SCREAMING_SNAKE_CASE=2048 ,					_SCREAMING_SNAKE_CASE=8 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE="relu" ,					_SCREAMING_SNAKE_CASE=256 ,					_SCREAMING_SNAKE_CASE=0.1 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.0 ,					_SCREAMING_SNAKE_CASE=0.02 ,					_SCREAMING_SNAKE_CASE=1.0 ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE="sine" ,					_SCREAMING_SNAKE_CASE="resnet50" ,					_SCREAMING_SNAKE_CASE=True ,					_SCREAMING_SNAKE_CASE=False ,					_SCREAMING_SNAKE_CASE=1 ,					_SCREAMING_SNAKE_CASE=5 ,					_SCREAMING_SNAKE_CASE=2 ,					_SCREAMING_SNAKE_CASE=1 ,					_SCREAMING_SNAKE_CASE=1 ,					_SCREAMING_SNAKE_CASE=5 ,					_SCREAMING_SNAKE_CASE=2 ,					_SCREAMING_SNAKE_CASE=0.1 ,					**_SCREAMING_SNAKE_CASE ,					):





        """simple docstring"""







        if backbone_config is not None and use_timm_backbone:
              raise ValueError('You can\'t specify both `backbone_config` and `use_timm_backbone`.'					)

        if not use_timm_backbone:
              if backbone_config is None:
                    logger.info('`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.'					)
                    SCREAMING_SNAKE_CASE_				:  List[Any]           =     CONFIG_MAPPING['resnet'](out_features=['stage4']					)
              elif isinstance(_SCREAMING_SNAKE_CASE ,					_SCREAMING_SNAKE_CASE					):
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     backbone_config.get('model_type'					)
                    SCREAMING_SNAKE_CASE_				:  str           =     CONFIG_MAPPING[backbone_model_type]
                    SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     config_class.from_dict(_SCREAMING_SNAKE_CASE					)
              # set timm attributes to None
              SCREAMING_SNAKE_CASE_       ,						SCREAMING_SNAKE_CASE_       ,						SCREAMING_SNAKE_CASE_				:  Optional[int]           =     None, None, None

        SCREAMING_SNAKE_CASE_				:  Any           =     use_timm_backbone
        SCREAMING_SNAKE_CASE_				:  int           =     backbone_config
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     num_channels
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     num_queries
        SCREAMING_SNAKE_CASE_				:  int           =     d_model
        SCREAMING_SNAKE_CASE_				:  int           =     encoder_ffn_dim
        SCREAMING_SNAKE_CASE_				:  Any           =     encoder_layers
        SCREAMING_SNAKE_CASE_				:  str           =     encoder_attention_heads
        SCREAMING_SNAKE_CASE_				:  List[str]           =     decoder_ffn_dim
        SCREAMING_SNAKE_CASE_				:  str           =     decoder_layers
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     decoder_attention_heads
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     dropout
        SCREAMING_SNAKE_CASE_				:  int           =     attention_dropout
        SCREAMING_SNAKE_CASE_				:  Dict           =     activation_dropout
        SCREAMING_SNAKE_CASE_				:  Optional[int]           =     activation_function
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     init_std
        SCREAMING_SNAKE_CASE_				:  int           =     init_xavier_std
        SCREAMING_SNAKE_CASE_				:  str           =     encoder_layerdrop
        SCREAMING_SNAKE_CASE_				:  Any           =     decoder_layerdrop
        SCREAMING_SNAKE_CASE_				:  Any           =     encoder_layers
        SCREAMING_SNAKE_CASE_				:  List[str]           =     auxiliary_loss
        SCREAMING_SNAKE_CASE_				:  List[str]           =     position_embedding_type
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     backbone
        SCREAMING_SNAKE_CASE_				:  Tuple           =     use_pretrained_backbone
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     dilation
        # Hungarian matcher
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     class_cost
        SCREAMING_SNAKE_CASE_				:  List[Any]           =     bbox_cost
        SCREAMING_SNAKE_CASE_				:  Union[str, Any]           =     giou_cost
        # Loss coefficients
        SCREAMING_SNAKE_CASE_				:  Optional[Any]           =     mask_loss_coefficient
        SCREAMING_SNAKE_CASE_				:  str           =     dice_loss_coefficient
        SCREAMING_SNAKE_CASE_				:  Dict           =     bbox_loss_coefficient
        SCREAMING_SNAKE_CASE_				:  int           =     giou_loss_coefficient
        SCREAMING_SNAKE_CASE_				:  Any           =     eos_coefficient
        super().__init__(is_encoder_decoder=_SCREAMING_SNAKE_CASE ,					**_SCREAMING_SNAKE_CASE					)

  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.encoder_attention_heads



  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return self.d_model

class  _A		(  __magic_name__):
  SCREAMING_SNAKE_CASE    :      List[Any]       =  version.parse('''1.11''')

  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return OrderedDict(
            [
                ('pixel_values', {0: 'batch', 1: 'num_channels', 2: 'height', 3: 'width'}),
                ('pixel_mask', {0: 'batch'}),
            ]					)

  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return 1e-5



  @property
  def  UpperCAmelCase    (				self					):





        """simple docstring"""







        return 12






