


'''simple docstring'''



import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
__lowercase      :		List[str]			=      'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
__lowercase      :		List[Any]			=      direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
__lowercase      :		Optional[Any]			=      re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
__lowercase      :		List[Any]			=      re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
__lowercase      :		Dict			=      re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
__lowercase      :		Dict			=      [
    ('pretraining', 'MODEL_FOR_PRETRAINING_MAPPING_NAMES', 'AutoModelForPreTraining'),
    ('feature-extraction', 'MODEL_MAPPING_NAMES', 'AutoModel'),
    ('audio-classification', 'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForAudioClassification'),
    ('text-generation', 'MODEL_FOR_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForCausalLM'),
    ('automatic-speech-recognition', 'MODEL_FOR_CTC_MAPPING_NAMES', 'AutoModelForCTC'),
    ('image-classification', 'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForImageClassification'),
    ('image-segmentation', 'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES', 'AutoModelForImageSegmentation'),
    ('fill-mask', 'MODEL_FOR_MASKED_LM_MAPPING_NAMES', 'AutoModelForMaskedLM'),
    ('object-detection', 'MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES', 'AutoModelForObjectDetection'),
    (
        'zero-shot-object-detection',
        'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES',
        'AutoModelForZeroShotObjectDetection',
    ),
    ('question-answering', 'MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES', 'AutoModelForQuestionAnswering'),
    ('text2text-generation', 'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForSeq2SeqLM'),
    ('text-classification', 'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForSequenceClassification'),
    ('automatic-speech-recognition', 'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES', 'AutoModelForSpeechSeq2Seq'),
    (
        'table-question-answering',
        'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForTableQuestionAnswering',
    ),
    ('token-classification', 'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForTokenClassification'),
    ('multiple-choice', 'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES', 'AutoModelForMultipleChoice'),
    (
        'next-sentence-prediction',
        'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES',
        'AutoModelForNextSentencePrediction',
    ),
    (
        'audio-frame-classification',
        'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForAudioFrameClassification',
    ),
    ('audio-xvector', 'MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES', 'AutoModelForAudioXVector'),
    (
        'document-question-answering',
        'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForDocumentQuestionAnswering',
    ),
    (
        'visual-question-answering',
        'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForVisualQuestionAnswering',
    ),
    ('image-to-text', 'MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES', 'AutoModelForVision2Seq'),
    (
        'zero-shot-image-classification',
        'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForZeroShotImageClassification',
    ),
    ('depth-estimation', 'MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES', 'AutoModelForDepthEstimation'),
    ('video-classification', 'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForVideoClassification'),
    ('mask-generation', 'MODEL_FOR_MASK_GENERATION_MAPPING_NAMES', 'AutoModelForMaskGeneration'),
]






def     lowerCamelCase				(_SCREAMING_SNAKE_CASE :  Dict  ):
					__a     :			str          =    re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'		, A__  )
					return [m.group(0  ) for m in matches]






def     lowerCamelCase				():
					__a     :			List[Any]          =    transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
					__a     :			Union[str, Any]          =    {
					    config.replace('Config'		, ''  ): model_type for model_type, config in config_maping_names.items()
					}

					# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
					__a     :			Any          =    collections.defaultdict(A__  )
					__a     :			str          =    collections.defaultdict(A__  )
					__a     :			Optional[int]          =    collections.defaultdict(A__  )

					# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
					for attr_name in dir(A__  ):
										__a     :			Tuple          =    None
										if _re_tf_models.match(A__  ) is not None:
															__a     :			List[str]          =    tf_models
															__a     :			Tuple          =    _re_tf_models.match(A__  ).groups()[0]
										elif _re_flax_models.match(A__  ) is not None:
															__a     :			Union[str, Any]          =    flax_models
															__a     :			Optional[int]          =    _re_flax_models.match(A__  ).groups()[0]
										elif _re_pt_models.match(A__  ) is not None:
															__a     :			Dict          =    pt_models
															__a     :			Optional[Any]          =    _re_pt_models.match(A__  ).groups()[0]

										if lookup_dict is not None:
															while len(A__  ) > 0:
																				if attr_name in model_prefix_to_model_type:
																									__a     :			int          =    True
																									break
																				# Try again after removing the last word in the name
																				__a     :			Optional[Any]          =    ''.join(camel_case_split(A__  )[:-1]  )

					__a     :			Tuple          =    set(list(pt_models.keys()  ) + list(tf_models.keys()  ) + list(flax_models.keys()  )  )
					__a     :			int          =    list(A__  )
					all_models.sort()

					__a     :			Optional[Any]          =    {'model_type': all_models}
					__a     :			List[Any]          =    [pt_models[t] for t in all_models]
					__a     :			Optional[Any]          =    [tf_models[t] for t in all_models]
					__a     :			int          =    [flax_models[t] for t in all_models]

					# Now let's use the auto-mapping names to make sure
					__a     :			Any          =    {}
					for t in all_models:
										if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
															__a     :			int          =    'AutoProcessor'
										elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
															__a     :			str          =    'AutoTokenizer'
										elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
															__a     :			Optional[Any]          =    'AutoFeatureExtractor'
										else:
															# Default to AutoTokenizer if a model has nothing, for backward compatibility.
															__a     :			Dict          =    'AutoTokenizer'

					__a     :			Optional[int]          =    [processors[t] for t in all_models]

					return pd.DataFrame(A__  )






def     lowerCamelCase				(_SCREAMING_SNAKE_CASE :  Any  ):
					__a     :			Dict          =    [
					    transformers_module.models.auto.modeling_auto,
					    transformers_module.models.auto.modeling_tf_auto,
					    transformers_module.models.auto.modeling_flax_auto,
					]
					for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
										__a     :			Any          =    [model_mapping, F"""TF_{model_mapping}""", F"""FLAX_{model_mapping}"""]
										__a     :			Dict          =    [auto_class, F"""TF_{auto_class}""", F"""Flax_{auto_class}"""]
										# Loop through all three frameworks
										for module, cls, mapping in zip(A__		, A__		, A__  ):
															# The type of pipeline may not exist in this framework
															if not hasattr(A__		, A__  ):
																				continue
															# First extract all model_names
															__a     :			int          =    []
															for name in getattr(A__		, A__  ).values():
																				if isinstance(A__		, A__  ):
																									model_names.append(A__  )
																				else:
																									model_names.extend(list(A__  )  )

            # Add pipeline tag and auto model class for those models
															table.update({model_name: (pipeline_tag, cls) for model_name in model_names}  )

					return table






def     lowerCamelCase				(_SCREAMING_SNAKE_CASE :  int		, _SCREAMING_SNAKE_CASE :  str  ):
					__a     :			Any          =    get_frameworks_table()
					__a     :			List[str]          =    Dataset.from_pandas(A__  )

					__a     :			Dict          =    hf_hub_download(
					    'huggingface/transformers-metadata'		, 'pipeline_tags.json'		, repo_type='dataset'		, token=A__  )
					__a     :			Tuple          =    Dataset.from_json(A__  )
					__a     :			Optional[int]          =    {
					    tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
					    for i in range(len(A__  )  )
					}
					__a     :			List[str]          =    update_pipeline_and_auto_class_table(A__  )

					# Sort the model classes to avoid some nondeterministic updates to create false update commits.
					__a     :			Optional[Any]          =    sorted(table.keys()  )
					__a     :			Any          =    pd.DataFrame(
					    {
					        'model_class': model_classes,
					        'pipeline_tag': [table[m][0] for m in model_classes],
					        'auto_class': [table[m][1] for m in model_classes],
					    }  )
					__a     :			Any          =    Dataset.from_pandas(A__  )

					with tempfile.TemporaryDirectory() as tmp_dir:
										frameworks_dataset.to_json(os.path.join(A__		, 'frameworks.json'  )  )
										tags_dataset.to_json(os.path.join(A__		, 'pipeline_tags.json'  )  )

										if commit_sha is not None:
															__a     :			List[str]          =    (
															    F"""Update with commit {commit_sha}\n\nSee: """
															    F"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
															)
										else:
															__a     :			Optional[Any]          =    'Update'

										upload_folder(
										    repo_id='huggingface/transformers-metadata'		, folder_path=A__		, repo_type='dataset'		, token=A__		, commit_message=A__		, )






def     lowerCamelCase				():
					__a     :			List[Any]          =    {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
					__a     :			Union[str, Any]          =    transformers_module.pipelines.SUPPORTED_TASKS
					__a     :			Any          =    []
					for key in pipeline_tasks:
										if key not in in_table:
															__a     :			List[str]          =    pipeline_tasks[key]['pt']
															if isinstance(A__		, (list, tuple)  ):
																				__a     :			int          =    model[0]
															__a     :			Tuple          =    model.__name__
															if model not in in_table.values():
																				missing.append(A__  )

					if len(A__  ) > 0:
										__a     :			List[str]          =    ', '.join(A__  )
										raise ValueError(
										    'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
										    F"""`utils/update_metadata.py`: {msg}. Please add them!"""  )


if __name__ == "__main__":
	__lowercase      :		Tuple			=      argparse.ArgumentParser()
	parser.add_argument('--token', type=str, help='The token to use to push to the transformers-metadata dataset.')
	parser.add_argument('--commit_sha', type=str, help='The sha of the commit going with this update.')
	parser.add_argument('--check-only', action='store_true', help='Activate to just check all pipelines are present.')
	__lowercase      :		Tuple			=      parser.parse_args()

	if args.check_only:
		check_pipeline_tags()
	else:
		update_metadata(args.token, args.commit_sha)





'''simple docstring'''
from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)






def      UpperCamelCase_				(      A__  ,      A__						):
	a_      				=						[]
	for part_id in partition_order:
		a_      				=						df.where(F'''SPARK_PARTITION_ID() = {part_id}'''						).collect()
		for row_idx, row in enumerate(A__						):
			expected_row_ids_and_row_dicts.append((F'''{part_id}_{row_idx}''', row.asDict())						)
	return expected_row_ids_and_row_dicts






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
	# that each partition can hold 2 rows.
	spark_builder._repartition_df_if_needed(max_shard_size=16						)
	# Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
	assert spark_builder.df.rdd.getNumPartitions() == 50






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(2						)
	a_      				=						[1, 0]
	a_      				=						_generate_iterable_examples(A__  ,      A__						)  # Reverse the partitions.
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      A__						)

	for i, (row_id, row_dict) in enumerate(generate_fn()						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(10						).repartition(1						)
	a_      				=						SparkExamplesIterable(A__						)
	assert it.n_shards == 1
	for i, (row_id, row_dict) in enumerate(A__						):
		assert row_id == F'''0_{i}'''
		assert row_dict == {"id": i}






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(30						).repartition(3						)
	# Mock the generator so that shuffle reverses the partition indices.
	with patch("""numpy.random.Generator"""						) as generator_mock:
		a_      				=						lambda A__						: x.reverse()
		a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [2, 1, 0]						)

		a_      				=						SparkExamplesIterable(A__						).shuffle_data_sources(A__						)
		assert shuffled_it.n_shards == 3
		for i, (row_id, row_dict) in enumerate(A__						):
			a_				,  a_      				=						expected_row_ids_and_row_dicts[i]
			assert row_id == expected_row_id
			assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(20						).repartition(4						)

	# Partitions 0 and 2
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=0  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [0, 2]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict

	# Partitions 1 and 3
	a_      				=						SparkExamplesIterable(A__						).shard_data_sources(worker_id=1  ,      num_workers=2						)
	assert shard_it_a.n_shards == 2
	a_      				=						_get_expected_row_ids_and_row_dicts_for_partition_order(A__  ,      [1, 3]						)
	for i, (row_id, row_dict) in enumerate(A__						):
		a_				,  a_      				=						expected_row_ids_and_row_dicts_a[i]
		assert row_id == expected_row_id
		assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def      UpperCamelCase_				(      ):
	a_      				=						pyspark.sql.SparkSession.builder.master("""local[*]"""						).appName("""pyspark"""						).getOrCreate()
	a_      				=						spark.range(1_00						).repartition(1						)
	a_      				=						Spark(A__						)
	# Choose a small max_shard_size for maximum partitioning.
	spark_builder._repartition_df_if_needed(max_shard_size=1						)
	# The new number of partitions should not be greater than the number of rows.
	assert spark_builder.df.rdd.getNumPartitions() == 1_00
