





import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


UpperCAmelCase__  :	str     =50_00_00

UpperCAmelCase__ ,UpperCAmelCase__  :	Dict     =os.path.split(__file__)
UpperCAmelCase__  :	str     =os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))





@get_duration
def    _lowercase    (  _UpperCAmelCase )  ->     List[Any]:
				lowerCamelCase		       =dataset.select(range(0	,					len(_UpperCAmelCase )	,					2 ) )





@get_duration
def    _lowercase    (  _UpperCAmelCase )  ->     Tuple:
				lowerCamelCase		       =dataset.sort("""numbers""" )





@get_duration
def    _lowercase    (  _UpperCAmelCase )  ->     Optional[int]:
				lowerCamelCase		       =dataset.shuffle()





@get_duration
def    _lowercase    (  _UpperCAmelCase )  ->     List[Any]:
				lowerCamelCase		       =dataset.train_test_split(0.1 )





@get_duration
def    _lowercase    (  _UpperCAmelCase	,					_UpperCAmelCase=10 )  ->     Optional[int]:
				for shard_id in range(_UpperCAmelCase ):
								lowerCamelCase		       =dataset.shard(_UpperCAmelCase	,					_UpperCAmelCase )





def    _lowercase    (  )  ->     Optional[int]:
				lowerCamelCase		       ={"""num examples""": SPEED_TEST_N_EXAMPLES}
				lowerCamelCase		       =(select, sort, shuffle, train_test_split, shard)
				with tempfile.TemporaryDirectory() as tmp_dir:
								print("""generating dataset""" )
								lowerCamelCase		       =datasets.Features({"""text""": datasets.Value("""string""" ), """numbers""": datasets.Value("""float32""" )} )
								lowerCamelCase		       =generate_example_dataset(
								    os.path.join(_UpperCAmelCase	,					"""dataset.arrow""" )	,					_UpperCAmelCase	,					num_examples=_UpperCAmelCase )
								print("""Functions""" )
								for func in functions:
												print(func.__name__ )
												lowerCamelCase		       =func(_UpperCAmelCase )

				with open(_UpperCAmelCase	,					"""wb""" ) as f:
								f.write(json.dumps(_UpperCAmelCase ).encode("""utf-8""" ) )


if __name__ == "__main__":  # useful to run the profiler
							benchmark_indices_mapping()







import os
import shutil
import unittest
from unittest.mock import patch

from transformers.testing_utils import CaptureStd, is_pt_tf_cross_test, require_torch







class      lowerCamelCase		(  unittest.TestCase  ):





     @patch("sys.argv"				,    ["fakeprogrampath", "env"]   )
     def 		UpperCAmelCase(self      :    Optional[Any]   )						->						Dict:
         # test transformers-cli env
         import transformers.commands.transformers_cli

         with CaptureStd() as cs:
             transformers.commands.transformers_cli.main()
         self.assertIn("Python version"				,    cs.out   )
         self.assertIn("Platform"				,    cs.out   )
         self.assertIn("Using distributed or parallel set-up in script?"				,    cs.out   )





     @is_pt_tf_cross_test
     @patch(
         "sys.argv"				,    ["fakeprogrampath", "pt-to-tf", "--model-name", "hf-internal-testing/tiny-random-gptj", "--no-pr"]   )
     def 		UpperCAmelCase(self      :    Any   )						->						int:
         import transformers.commands.transformers_cli

         shutil.rmtree("/tmp/hf-internal-testing/tiny-random-gptj"				,    ignore_errors=_A   )  # cleans potential past runs
         transformers.commands.transformers_cli.main()

         # The original repo has no TF weights -- if they exist, they were created by the CLI
         self.assertTrue(os.path.exists("/tmp/hf-internal-testing/tiny-random-gptj/tf_model.h5"   )   )





     @require_torch
     @patch("sys.argv"				,    ["fakeprogrampath", "download", "hf-internal-testing/tiny-random-gptj", "--cache-dir", "/tmp"]   )
     def 		UpperCAmelCase(self      :    Dict   )						->						List[Any]:
         import transformers.commands.transformers_cli

         # # remove any previously downloaded model to start clean
         shutil.rmtree("/tmp/models--hf-internal-testing--tiny-random-gptj"				,    ignore_errors=_A   )

         # run the command
         transformers.commands.transformers_cli.main()

         # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--tiny-random-gptj
         self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/blobs"   )   )
         self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/refs"   )   )
         self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--tiny-random-gptj/snapshots"   )   )





     @require_torch
     @patch(
         "sys.argv"				,    [
             "fakeprogrampath",
             "download",
             "hf-internal-testing/test_dynamic_model_with_tokenizer",
             "--trust-remote-code",
             "--cache-dir",
             "/tmp",
         ]				,    )
     def 		UpperCAmelCase(self      :    Any   )						->						List[str]:
         import transformers.commands.transformers_cli

         # # remove any previously downloaded model to start clean
         shutil.rmtree("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer"				,    ignore_errors=_A   )

         # run the command
         transformers.commands.transformers_cli.main()

         # check if the model files are downloaded correctly on /tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer
         self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/blobs"   )   )
         self.assertTrue(os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/refs"   )   )
         self.assertTrue(
             os.path.exists("/tmp/models--hf-internal-testing--test_dynamic_model_with_tokenizer/snapshots"   )   )

