


import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


a_					:    Optional[Any]          =    get_tests_dir("fixtures/test_sentencepiece.model")


if is_torch_available():
 from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

a_					:    Tuple          =    2_5_6_0_4_7
a_					:    str          =    2_5_6_1_4_5




@require_sentencepiece
@require_tokenizers
class  a     (     _SCREAMING_SNAKE_CASE  ,				unittest.TestCase       ):
 _lowerCAmelCase 					=	NllbTokenizer
 _lowerCAmelCase 					=	NllbTokenizerFast
 _lowerCAmelCase 					=	True
 _lowerCAmelCase 					=	True
 _lowerCAmelCase 					=	{}
 def 							__UpperCAmelCase		( self						)					->							Any:
      super().setUp()

      # We have a SentencePiece fixture for testing
      _a			      =							NllbTokenizer(__magic_name__      ,				keep_accents=__magic_name__						)
      tokenizer.save_pretrained(self.tmpdirname						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      _a			      =							NllbTokenizer(__magic_name__      ,				keep_accents=__magic_name__						)

      _a			      =							tokenizer.tokenize('This is a test'						)
      self.assertListEqual(__magic_name__      ,				['▁This', '▁is', '▁a', '▁t', 'est']						)

      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(__magic_name__						)      ,				[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]      ,				)

      _a			      =							tokenizer.tokenize('I was born in 92000, and this is falsé.'						)
      self.assertListEqual(
          __magic_name__      ,				[
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '9',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              'é',
              '.',
          ]      ,				)
      _a			      =							tokenizer.convert_tokens_to_ids(__magic_name__						)
      self.assertListEqual(
          __magic_name__      ,				[
              value + tokenizer.fairseq_offset
              for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
          ]      ,				)

      _a			      =							tokenizer.convert_ids_to_tokens(__magic_name__						)
      self.assertListEqual(
          __magic_name__      ,				[
              SPIECE_UNDERLINE + 'I',
              SPIECE_UNDERLINE + 'was',
              SPIECE_UNDERLINE + 'b',
              'or',
              'n',
              SPIECE_UNDERLINE + 'in',
              SPIECE_UNDERLINE + '',
              '<unk>',
              '2',
              '0',
              '0',
              '0',
              ',',
              SPIECE_UNDERLINE + 'and',
              SPIECE_UNDERLINE + 'this',
              SPIECE_UNDERLINE + 'is',
              SPIECE_UNDERLINE + 'f',
              'al',
              's',
              '<unk>',
              '.',
          ]      ,				)
 def 							__UpperCAmelCase		( self						)					->							Any:
      _a			      =							(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
                _a			      =							self.rust_tokenizer_class.from_pretrained(__magic_name__      ,				**__magic_name__						)
                _a			      =							self.tokenizer_class.from_pretrained(__magic_name__      ,				**__magic_name__						)

                _a			      =							tempfile.mkdtemp()

                _a			      =							tokenizer_r.save_pretrained(__magic_name__						)
                _a			      =							tokenizer_p.save_pretrained(__magic_name__						)

                # Checks it save with the same files + the tokenizer.json file for the fast one
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files						)						)
                _a			      =							tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f						)
                self.assertSequenceEqual(__magic_name__      ,				__magic_name__						)

                # Checks everything loads correctly in the same way
                _a			      =							tokenizer_r.from_pretrained(__magic_name__						)
                _a			      =							tokenizer_p.from_pretrained(__magic_name__						)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__      ,				__magic_name__						)						)

                shutil.rmtree(__magic_name__						)

                # Save tokenizer rust, legacy_format=True
                _a			      =							tempfile.mkdtemp()

                _a			      =							tokenizer_r.save_pretrained(__magic_name__      ,				legacy_format=__magic_name__						)
                _a			      =							tokenizer_p.save_pretrained(__magic_name__						)

                # Checks it save with the same files
                self.assertSequenceEqual(__magic_name__      ,				__magic_name__						)

                # Checks everything loads correctly in the same way
                _a			      =							tokenizer_r.from_pretrained(__magic_name__						)
                _a			      =							tokenizer_p.from_pretrained(__magic_name__						)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__      ,				__magic_name__						)						)

                shutil.rmtree(__magic_name__						)

                # Save tokenizer rust, legacy_format=False
                _a			      =							tempfile.mkdtemp()

                _a			      =							tokenizer_r.save_pretrained(__magic_name__      ,				legacy_format=__magic_name__						)
                _a			      =							tokenizer_p.save_pretrained(__magic_name__						)

                # Checks it saved the tokenizer.json file
                self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files						)						)

                # Checks everything loads correctly in the same way
                _a			      =							tokenizer_r.from_pretrained(__magic_name__						)
                _a			      =							tokenizer_p.from_pretrained(__magic_name__						)

                # Check special tokens are set accordingly on Rust and Python
                for key in tokenizer_pp.special_tokens_map:
                     self.assertTrue(hasattr(__magic_name__      ,				__magic_name__						)						)

                shutil.rmtree(__magic_name__						)
 @require_torch
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      if not self.test_seqaseq:
           return

      _a			      =							self.get_tokenizers()
      for tokenizer in tokenizers:
           with self.subTest(f'{tokenizer.__class__.__name__}'						):
                # Longer text that will definitely require truncation.
                _a			      =							[
                    ' UN Chief Says There Is No Military Solution in Syria',
                    ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                    ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                    ' will only worsen the violence and misery for millions of people.',
                ]
                _a			      =							[
                    'Şeful ONU declară că nu există o soluţie militară în Siria',
                    'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                    ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                    ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                ]
                try:
                     _a			      =							tokenizer.prepare_seqaseq_batch(
                         src_texts=__magic_name__      ,				tgt_texts=__magic_name__      ,				max_length=3      ,				max_target_length=10      ,				return_tensors='pt'      ,				src_lang='eng_Latn'      ,				tgt_lang='ron_Latn'      ,				)
                except NotImplementedError:
                     return
                self.assertEqual(batch.input_ids.shape[1]      ,				3						)
                self.assertEqual(batch.labels.shape[1]      ,				10						)
                # max_target_length will default to max_length if not specified
                _a			      =							tokenizer.prepare_seqaseq_batch(
                    __magic_name__      ,				tgt_texts=__magic_name__      ,				max_length=3      ,				return_tensors='pt'						)
                self.assertEqual(batch.input_ids.shape[1]      ,				3						)
                self.assertEqual(batch.labels.shape[1]      ,				3						)

                _a			      =							tokenizer.prepare_seqaseq_batch(
                    src_texts=__magic_name__      ,				max_length=3      ,				max_target_length=10      ,				return_tensors='pt'						)
                self.assertEqual(batch_encoder_only.input_ids.shape[1]      ,				3						)
                self.assertEqual(batch_encoder_only.attention_mask.shape[1]      ,				3						)
                self.assertNotIn('decoder_input_ids'      ,				__magic_name__						)
 @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'						)
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      pass




 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
           with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'						):
                _a			      =							[AddedToken('<special>'      ,				lstrip=__magic_name__						)]

                _a			      =							self.rust_tokenizer_class.from_pretrained(
                    __magic_name__      ,				additional_special_tokens=__magic_name__      ,				**__magic_name__						)
                _a			      =							tokenizer_r.encode('Hey this is a <special> token'						)

                _a			      =							tokenizer_r.encode('<special>'      ,				add_special_tokens=__magic_name__						)[0]

                self.assertTrue(special_token_id in r_output						)

                if self.test_slow_tokenizer:
                     _a			      =							self.rust_tokenizer_class.from_pretrained(
                         __magic_name__      ,				additional_special_tokens=__magic_name__      ,				**__magic_name__      ,				)
                     _a			      =							self.tokenizer_class.from_pretrained(
                         __magic_name__      ,				additional_special_tokens=__magic_name__      ,				**__magic_name__						)

                     _a			      =							tokenizer_p.encode('Hey this is a <special> token'						)

                     _a			      =							tokenizer_cr.encode('Hey this is a <special> token'						)

                     self.assertEqual(__magic_name__      ,				__magic_name__						)
                     self.assertEqual(__magic_name__      ,				__magic_name__						)
                     self.assertTrue(special_token_id in p_output						)
                     self.assertTrue(special_token_id in cr_output						)





@require_torch
@require_sentencepiece
@require_tokenizers
class  a     (     unittest.TestCase       ):
 _lowerCAmelCase 					=	"""facebook/nllb-200-distilled-600M"""
 _lowerCAmelCase 					=	[
     """ UN Chief Says There Is No Military Solution in Syria""",
     """ Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.""",
 ]
 _lowerCAmelCase 					=	[
     """Şeful ONU declară că nu există o soluţie militară în Siria""",
     """Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"""
     """ pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"""
     """ face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.""",
 ]
 _lowerCAmelCase 					=	[
     2_5_6_0_4_7,
     1_6_2_9_7,
     1_3_4_4_0_8,
     8_1_6_5,
     2_4_8_0_6_6,
     1_4_7_3_4,
     9_5_0,
     1_1_3_5,
     1_0_5_7_2_1,
     3_5_7_3,
     8_3,
     2_7_3_5_2,
     1_0_8,
     4_9_4_8_6,
     2,
 ]
 @classmethod
 def 							__UpperCAmelCase		( cls						)					->							Union[str, Any]:
      _a			      =							NllbTokenizer.from_pretrained(
          cls.checkpoint_name      ,				src_lang='eng_Latn'      ,				tgt_lang='ron_Latn'						)
      _a			      =							1
      return cls
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']      ,				25_60_01						)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']      ,				25_60_02						)
      self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']      ,				25_60_57						)
 def 							__UpperCAmelCase		( self						)					->							List[str]:
      _a			      =							self.tokenizer.batch_encode_plus(self.src_text						).input_ids[0]
      self.assertListEqual(self.expected_src_tokens      ,				__magic_name__						)
 def 							__UpperCAmelCase		( self						)					->							int:
      self.assertIn(__magic_name__      ,				self.tokenizer.all_special_ids						)
      # fmt: off
      _a			      =							[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
      # fmt: on

      _a			      =							self.tokenizer.decode(__magic_name__      ,				skip_special_tokens=__magic_name__						)
      _a			      =							self.tokenizer.decode(generated_ids[1:]      ,				skip_special_tokens=__magic_name__						)
      self.assertEqual(__magic_name__      ,				__magic_name__						)
      self.assertNotIn(self.tokenizer.eos_token      ,				__magic_name__						)
 def 							__UpperCAmelCase		( self						)					->							str:
      _a			      =							['this is gunna be a long sentence ' * 20]
      assert isinstance(src_text[0]      ,				__magic_name__						)
      _a			      =							10
      _a			      =							self.tokenizer(__magic_name__      ,				max_length=__magic_name__      ,				truncation=__magic_name__						).input_ids[0]
      self.assertEqual(ids[-1]      ,				2						)
      self.assertEqual(ids[0]      ,				__magic_name__						)
      self.assertEqual(len(__magic_name__						)      ,				__magic_name__						)
 def 							__UpperCAmelCase		( self						)					->							Union[str, Any]:
      self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']						)      ,				[25_62_03, 3]						)
 def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
      _a			      =							tempfile.mkdtemp()
      _a			      =							self.tokenizer.fairseq_tokens_to_ids
      self.tokenizer.save_pretrained(__magic_name__						)
      _a			      =							NllbTokenizer.from_pretrained(__magic_name__						)
      self.assertDictEqual(new_tok.fairseq_tokens_to_ids      ,				__magic_name__						)
 @require_torch
 def 							__UpperCAmelCase		( self						)					->							str:
      _a			      =							self.tokenizer(
          self.src_text      ,				text_target=self.tgt_text      ,				padding=__magic_name__      ,				truncation=__magic_name__      ,				max_length=len(self.expected_src_tokens						)      ,				return_tensors='pt'      ,				)
      _a			      =							shift_tokens_right(
          batch['labels']      ,				self.tokenizer.pad_token_id      ,				self.tokenizer.lang_code_to_id['ron_Latn']						)

      self.assertIsInstance(__magic_name__      ,				__magic_name__						)

      self.assertEqual((2, 15)      ,				batch.input_ids.shape						)
      self.assertEqual((2, 15)      ,				batch.attention_mask.shape						)
      _a			      =							batch.input_ids.tolist()[0]
      self.assertListEqual(self.expected_src_tokens      ,				__magic_name__						)
      self.assertEqual(__magic_name__      ,				batch.decoder_input_ids[0, 0]						)  # EOS
      # Test that special tokens are reset
      self.assertEqual(self.tokenizer.prefix_tokens      ,				[EN_CODE]						)
      self.assertEqual(self.tokenizer.suffix_tokens      ,				[self.tokenizer.eos_token_id]						)
 def 							__UpperCAmelCase		( self						)					->							Dict:
      _a			      =							self.tokenizer(self.src_text      ,				padding=__magic_name__      ,				truncation=__magic_name__      ,				max_length=3      ,				return_tensors='pt'						)
      _a			      =							self.tokenizer(
          text_target=self.tgt_text      ,				padding=__magic_name__      ,				truncation=__magic_name__      ,				max_length=10      ,				return_tensors='pt'						)
      _a			      =							targets['input_ids']
      _a			      =							shift_tokens_right(
          __magic_name__      ,				self.tokenizer.pad_token_id      ,				decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]      ,				)

      self.assertEqual(batch.input_ids.shape[1]      ,				3						)
      self.assertEqual(batch.decoder_input_ids.shape[1]      ,				10						)
 @require_torch
 def 							__UpperCAmelCase		( self						)					->							Optional[int]:
      _a			      =							self.tokenizer._build_translation_inputs(
          'A test'      ,				return_tensors='pt'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'						)

      self.assertEqual(
          nested_simplify(__magic_name__						)      ,				{
              # A, test, EOS, en_XX
              'input_ids': [[25_60_47, 70, 73_56, 2]],
              'attention_mask': [[1, 1, 1, 1]],
              # ar_AR
              'forced_bos_token_id': 25_60_57,
          }      ,				)




 @require_torch
 def 							__UpperCAmelCase		( self						)					->							Dict:
      _a			      =							True
      _a			      =							self.tokenizer(
          'UN Chief says there is no military solution in Syria'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'						)
      self.assertEqual(
          inputs.input_ids      ,				[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]						)

      _a			      =							False
      _a			      =							self.tokenizer(
          'UN Chief says there is no military solution in Syria'      ,				src_lang='eng_Latn'      ,				tgt_lang='fra_Latn'						)
      self.assertEqual(
          inputs.input_ids      ,				[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]						)





'''simple docstring'''






import logging
import os
from dataclasses import dataclass
from typing import List, Optional, Union

import tqdm
from filelock import FileLock

from transformers import (
    BartTokenizer,
    BartTokenizerFast,
    DataProcessor,
    PreTrainedTokenizer,
    RobertaTokenizer,
    RobertaTokenizerFast,
    XLMRobertaTokenizer,
    is_tf_available,
    is_torch_available,
)


a_					:    Optional[Any]          =    logging.getLogger(__name__)




@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None





@dataclass(frozen=_SCREAMING_SNAKE_CASE       )
class  a     :
 _lowerCAmelCase 					=	42
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None
 _lowerCAmelCase 					=	None


if is_torch_available():
 import torch
 from torch.utils.data import Dataset




 class  a     (     _SCREAMING_SNAKE_CASE       ):
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = None      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							Optional[int]:
       _a			      =							hans_processors[task]()

       _a			      =							os.path.join(
           __magic_name__      ,				'cached_{}_{}_{}_{}'.format(
               'dev' if evaluate else 'train'      ,				tokenizer.__class__.__name__      ,				str(__magic_name__						)      ,				__magic_name__      ,				)      ,				)
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       # Make sure only the first process in distributed training processes the dataset,
       # and the others will use the cache.
       _a			      =							cached_features_file + '.lock'
       with FileLock(__magic_name__						):
            if os.path.exists(__magic_name__						) and not overwrite_cache:
                 logger.info(f'Loading features from cached file {cached_features_file}'						)
                 _a			      =							torch.load(__magic_name__						)
            else:
                 logger.info(f'Creating features from dataset file at {data_dir}'						)

                 _a			      =							(
                     processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
                 )

                 logger.info('Training examples: %s'      ,				len(__magic_name__						)						)
                 _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)
                 logger.info('Saving features into cached file %s'      ,				__magic_name__						)
                 torch.save(self.features      ,				__magic_name__						)
  def __len__( self						)					->							List[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							Optional[Any]:
       return self.label_list


if is_tf_available():
 import tensorflow as tf




 class  a     :
  _lowerCAmelCase 					=	42
  def __init__( self      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__ = 1_28      ,				__magic_name__=False      ,				__magic_name__ = False      ,				)					->							str:
       _a			      =							hans_processors[task]()
       _a			      =							processor.get_labels()
       if tokenizer.__class__ in (
           RobertaTokenizer,
           RobertaTokenizerFast,
           XLMRobertaTokenizer,
           BartTokenizer,
           BartTokenizerFast,
       ):
            # HACK(label indices are swapped in RoBERTa pretrained model)
            _a      ,   _a			      =							label_list[2], label_list[1]
       _a			      =							label_list

       _a			      =							processor.get_dev_examples(__magic_name__						) if evaluate else processor.get_train_examples(__magic_name__						)
       _a			      =							hans_convert_examples_to_features(__magic_name__      ,				__magic_name__      ,				__magic_name__      ,				__magic_name__						)

       def gen():
            for ex_index, ex in tqdm.tqdm(enumerate(self.features						)      ,				desc='convert examples to features'						):
                 if ex_index % 1_00_00 == 0:
                      logger.info('Writing example %d of %d' % (ex_index, len(__magic_name__						))						)

                 yield (
                     {
                         "example_id": 0,
                         "input_ids": ex.input_ids,
                         "attention_mask": ex.attention_mask,
                         "token_type_ids": ex.token_type_ids,
                     },
                     ex.label,
                 )

       _a			      =							tf.data.Dataset.from_generator(
           __magic_name__      ,				(
               {
                   'example_id': tf.intaa,
                   'input_ids': tf.intaa,
                   'attention_mask': tf.intaa,
                   'token_type_ids': tf.intaa,
               },
               tf.intaa,
           )      ,				(
               {
                   'example_id': tf.TensorShape([]						),
                   'input_ids': tf.TensorShape([None, None]						),
                   'attention_mask': tf.TensorShape([None, None]						),
                   'token_type_ids': tf.TensorShape([None, None]						),
               },
               tf.TensorShape([]						),
           )      ,				)
  def 							__UpperCAmelCase		( self						)					->							Tuple:
       return self.dataset
  def __len__( self						)					->							Optional[Any]:
       return len(self.features						)
  def __getitem__( self      ,				__magic_name__						)					->							InputFeatures:
       return self.features[i]




  def 							__UpperCAmelCase		( self						)					->							List[Any]:
       return self.label_list




class  a     (     _SCREAMING_SNAKE_CASE       ):
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							Tuple:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_train_set.txt'						)						)      ,				'train'						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							List[Any]:
      return self._create_examples(self._read_tsv(os.path.join(__magic_name__      ,				'heuristics_evaluation_set.txt'						)						)      ,				'dev'						)
 def 							__UpperCAmelCase		( self						)					->							Tuple:
      return ["contradiction", "entailment", "neutral"]




 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__						)					->							Optional[Any]:
      _a			      =							[]
      for i, line in enumerate(__magic_name__						):
           if i == 0:
                continue
           _a			      =							'%s-%s' % (set_type, line[0])
           _a			      =							line[5]
           _a			      =							line[6]
           _a			      =							line[7][2:] if line[7].startswith('ex'						) else line[7]
           _a			      =							line[0]
           examples.append(InputExample(guid=__magic_name__      ,				text_a=__magic_name__      ,				text_b=__magic_name__      ,				label=__magic_name__      ,				pairID=__magic_name__						)						)
      return examples






def 	_A				(lowerCAmelCase__  :List[InputExample]				,						lowerCAmelCase__  :List[str]				,						lowerCAmelCase__  :int				,						lowerCAmelCase__  :PreTrainedTokenizer				,						)   ->  Tuple:




     '''simple docstring'''



     _a			      =							{label: i for i, label in enumerate(lowerCAmelCase__		)}

     _a			      =							[]
     for ex_index, example in tqdm.tqdm(enumerate(lowerCAmelCase__		)				,						desc='convert examples to features'		):
          if ex_index % 1_00_00 == 0:
               logger.info('Writing example %d' % (ex_index)		)

          _a			      =							tokenizer(
              example.text_a				,						example.text_b				,						add_special_tokens=lowerCAmelCase__				,						max_length=lowerCAmelCase__				,						padding='max_length'				,						truncation=lowerCAmelCase__				,						return_overflowing_tokens=lowerCAmelCase__				,						)

          _a			      =							label_map[example.label] if example.label in label_map else 0

          _a			      =							int(example.pairID		)

          features.append(InputFeatures(**lowerCAmelCase__				,						label=lowerCAmelCase__				,						pairID=lowerCAmelCase__		)		)

     for i, example in enumerate(examples[:5]		):
          logger.info('*** Example ***'		)
          logger.info(f'guid: {example}'		)
          logger.info(f'features: {features[i]}'		)

     return features


a_					:    Optional[int]          =    {
    "hans": 3,
}






a_					:    Optional[Any]          =    {
    "hans": HansProcessor,
}

