




import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch



class 			a     (   unittest.TestCase			):

     '''simple docstring'''





     @patch('''sys.argv'''					,    ['''fakeprogrampath''', '''env''']				)
     def   lowerCamelCase_   (	self   :			int				):
         # test transformers-cli env
         import transformers.commands.transformers_cli

         with CaptureStd() as cs:
             transformers.commands.transformers_cli.main()
         self.assertIn('''Python version'''					,    cs.out				)
         self.assertIn('''Platform'''					,    cs.out				)
         self.assertIn('''Using distributed or parallel set-up in script?'''					,    cs.out				)
     @is_pt_tf_cross_test
     @patch(
         '''sys.argv'''					,    ['''fakeprogrampath''', '''pt-to-tf''', '''--model-name''', '''hf-internal-testing/tiny-random-gptj''', '''--no-pr''']				)
     def   lowerCamelCase_   (	self   :			Optional[int]				):
         import transformers.commands.transformers_cli

         shutil.rmtree('''/tmp/hf-internal-testing/tiny-random-gptj'''					,    ignore_errors=__snake_case				)  # cleans potential past runs
         transformers.commands.transformers_cli.main()

         # The original repo has no TF weights -- if they exist, they were created by the CLI
         self.assertTrue(os.path.exists('''/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5'''				)				)
     @require_torch
     @patch('''sys.argv'''					,    ['''fakeprogrampath''', '''download''', '''hf-internal-testing/tiny-random-gptj''', '''--cache-dir''', '''/tmp''']				)
     def   lowerCamelCase_   (	self   :			Union[str, Any]				):
         import transformers.commands.transformers_cli

         # # remove any previously downloaded model to start clean
         shutil.rmtree('''/tmp/models--hf-internal-testing--tiny-random-gptj'''					,    ignore_errors=__snake_case				)

         # run the command
         transformers.commands.transformers_cli.main()

         # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
         self.assertTrue(os.path.exists('''/tmp/models--hf-internal-testing--tiny-random-gptj/blobs'''				)				)
         self.assertTrue(os.path.exists('''/tmp/models--hf-internal-testing--tiny-random-gptj/refs'''				)				)
         self.assertTrue(os.path.exists('''/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots'''				)				)




     @require_torch
     @patch(
         '''sys.argv'''					,    [
             '''fakeprogrampath''',
             '''download''',
             '''hf-internal-testing/test_dynamic_model_with_tokenizer''',
             '''--trust-remote-code''',
             '''--cache-dir''',
             '''/tmp''',
         ]					,    )
     def   lowerCamelCase_   (	self   :			str				):
         import transformers.commands.transformers_cli

         # # remove any previously downloaded model to start clean
         shutil.rmtree('''/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer'''					,    ignore_errors=__snake_case				)

         # run the command
         transformers.commands.transformers_cli.main()

         # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
         self.assertTrue(os.path.exists('''/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs'''				)				)
         self.assertTrue(os.path.exists('''/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs'''				)				)
         self.assertTrue(
             os.path.exists('''/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots'''				)				)











from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy



class 			a     (   _A			):

     '''simple docstring'''





     lowerCAmelCase				:       Any   		=  'MarkupLMFeatureExtractor'
     lowerCAmelCase				:       Optional[Any]   		=  ('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
     lowerCAmelCase				:       List[Any]   		=  True
     def __call__(	self   :			Dict					,    __snake_case   :			Tuple=None					,    __snake_case   :			List[Any]=None					,    __snake_case   :			Tuple=None					,    __snake_case   :			Tuple=None					,    __snake_case   :			Any=None					,    __snake_case   :			bool = True					,    __snake_case   :			Union[bool, str, PaddingStrategy] = False					,    __snake_case   :			Union[bool, str, TruncationStrategy] = None					,    __snake_case   :			Optional[int] = None					,    __snake_case   :			int = 0					,    __snake_case   :			Optional[int] = None					,    __snake_case   :			Optional[bool] = None					,    __snake_case   :			Optional[bool] = None					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = False					,    __snake_case   :			bool = True					,    __snake_case   :			Optional[Union[str, TensorType]] = None					,    **__snake_case   :			Any					,    ):
         # first, create nodes and xpaths
         if self.parse_html:
             if html_strings is None:
                 raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''				)

             if nodes is not None or xpaths is not None or node_labels is not None:
                 raise ValueError(
                     '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''				)

             UpperCAmelCase_							 =     self.feature_extractor(__snake_case				)
             UpperCAmelCase_							 =     features['''nodes''']
             UpperCAmelCase_							 =     features['''xpaths''']
         else:
             if html_strings is not None:
                 raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''				)
             if nodes is None or xpaths is None:
                 raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''				)

        # # second, apply the tokenizer
         if questions is not None and self.parse_html:
             if isinstance(__snake_case					,    __snake_case				):
                 UpperCAmelCase_							 =     [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

         UpperCAmelCase_							 =     self.tokenizer(
             text=questions if questions is not None else nodes					,    text_pair=nodes if questions is not None else None					,    xpaths=__snake_case					,    node_labels=__snake_case					,    add_special_tokens=__snake_case					,    padding=__snake_case					,    truncation=__snake_case					,    max_length=__snake_case					,    stride=__snake_case					,    pad_to_multiple_of=__snake_case					,    return_token_type_ids=__snake_case					,    return_attention_mask=__snake_case					,    return_overflowing_tokens=__snake_case					,    return_special_tokens_mask=__snake_case					,    return_offsets_mapping=__snake_case					,    return_length=__snake_case					,    verbose=__snake_case					,    return_tensors=__snake_case					,    **__snake_case					,    )

         return encoded_inputs
     def   lowerCamelCase_   (	self   :			List[Any]					,    *__snake_case   :			Dict					,    **__snake_case   :			Optional[int]				):
         return self.tokenizer.batch_decode(*__snake_case					,    **__snake_case				)
     def   lowerCamelCase_   (	self   :			Optional[Any]					,    *__snake_case   :			Any					,    **__snake_case   :			Dict				):
         return self.tokenizer.decode(*__snake_case					,    **__snake_case				)




     @property
     def   lowerCamelCase_   (	self   :			Dict				):
         UpperCAmelCase_							 =     self.tokenizer.model_input_names
         return tokenizer_input_names





