import os
from pathlib import Path
from typing import Dict, List

import fire
import torch

from transformers import AutoModelForSeqaSeqLM, AutoTokenizer
from transformers.utils.logging import get_logger


_lowerCamelCase    :      Tuple	 =						get_logger(__name__)





def 		__lowerCamelCase							(UpperCAmelCase__   :							str					,       UpperCAmelCase__   :							str     ):
					if text.startswith(UpperCAmelCase__     ):
										return text[len(UpperCAmelCase__     ) :]
					return text  # or whatever





def 		__lowerCamelCase							(UpperCAmelCase__   :							Optional[Any]     ):
					return {remove_prefix(UpperCAmelCase__					,       "model."     ): v for k, v in sd.items()}





def 		__lowerCamelCase							(UpperCAmelCase__   :							List[Dict[str, torch.Tensor]]     ):
					SCREAMING_SNAKE_CASE				=				{}
					for k in state_dicts[0].keys():
										SCREAMING_SNAKE_CASE				=				[sd[k] for sd in state_dicts]
										SCREAMING_SNAKE_CASE				=				sum(UpperCAmelCase__     ) / len(UpperCAmelCase__     )
										assert isinstance(UpperCAmelCase__					,       torch.Tensor     )
										SCREAMING_SNAKE_CASE				=				new_t
					return new_sd





def 		__lowerCamelCase							(UpperCAmelCase__   :							str					,       UpperCAmelCase__   :							str					,       UpperCAmelCase__   :							str     ):
					SCREAMING_SNAKE_CASE				=				AutoModelForSeqaSeqLM.from_pretrained(UpperCAmelCase__     )
					if os.path.isfile(UpperCAmelCase__     ):
										SCREAMING_SNAKE_CASE				=				[pl_ckpt_path]
					else:
										assert os.path.isdir(UpperCAmelCase__     )
										SCREAMING_SNAKE_CASE				=				list(Path(UpperCAmelCase__     ).glob("*.ckpt"     )     )
										assert ckpt_files, F"could not find any ckpt files inside the {pl_ckpt_path} directory"

					if len(UpperCAmelCase__     ) > 1:
										logger.info(F"averaging the weights of {ckpt_files}"     )

					SCREAMING_SNAKE_CASE				=				[sanitize(torch.load(UpperCAmelCase__					,       map_location="cpu"     )["state_dict"]     ) for x in ckpt_files]
					SCREAMING_SNAKE_CASE				=				average_state_dicts(UpperCAmelCase__     )

					SCREAMING_SNAKE_CASE	,      SCREAMING_SNAKE_CASE				=				hf_model.load_state_dict(UpperCAmelCase__					,       strict=UpperCAmelCase__     )
					assert not missing, F"missing keys: {missing}"
					hf_model.save_pretrained(UpperCAmelCase__     )
					try:
										SCREAMING_SNAKE_CASE				=				AutoTokenizer.from_pretrained(UpperCAmelCase__     )
										tok.save_pretrained(UpperCAmelCase__     )
					except Exception:
										pass
										# dont copy tokenizer if cant


if __name__ == "__main__":
							fire.Fire(convert_pl_to_hf)





import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_lowerCamelCase    :      int	 =						10_00_00_00_00_00
_lowerCamelCase    :      Union[str, Any]	 =						1_00_00

_lowerCamelCase ,  _lowerCamelCase    :      List[str]	 =						os.path.split(__file__)
_lowerCamelCase    :      Any	 =						os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))





def 		__lowerCamelCase							(UpperCAmelCase__   :							int					,       UpperCAmelCase__   :							int     ):
					SCREAMING_SNAKE_CASE				=				pa.Table.from_pydict({"col": [0] * chunk_size}     )
					SCREAMING_SNAKE_CASE				=				pa.concat_tables([table] * (num_examples // chunk_size)     )
					return datasets.Dataset(UpperCAmelCase__					,       fingerprint="table_100B"     )


@dataclass
class   lowercase			:
							lowercase__  :				int
							lowercase__  :				int
							lowercase__  :				int
							lowercase__  :				int




							def 		__snake_case(  self      :    Tuple					)    -> Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE				=				np.random.default_rng(self.seed					)
												SCREAMING_SNAKE_CASE				=				rng.integers(low=self.low		,							high=self.high		,							size=self.size					).tolist()




							def __iter__(  self      :    Union[str, Any]					)    -> Dict:



												'''simple docstring'''

												return iter(self._sampled_values					)





							def __len__(  self      :    Any					)    -> int:



												'''simple docstring'''

												return self.size





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[0]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[-1]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[range(len(UpperCAmelCase__     ) // 2					,       len(UpperCAmelCase__     ) // 2 + 1_0_2_4     )]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[RandIter(0					,       len(UpperCAmelCase__     )					,       1_0_2_4					,       seed=4_2     )]





def 		__lowerCamelCase							():
					SCREAMING_SNAKE_CASE				=				{"num examples": SPEED_TEST_N_EXAMPLES}
					SCREAMING_SNAKE_CASE				=				(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
					print("generating dataset"     )
					SCREAMING_SNAKE_CASE				=				generate_100B_dataset(num_examples=UpperCAmelCase__					,       chunk_size=UpperCAmelCase__     )
					print("Functions"     )
					for func in functions:
										print(func.__name__     )
										SCREAMING_SNAKE_CASE				=				func(UpperCAmelCase__     )

					with open(UpperCAmelCase__					,       "wb"     ) as f:
										f.write(json.dumps(UpperCAmelCase__     ).encode("utf-8"     )     )


if __name__ == "__main__":  # useful to run the profiler
							benchmark_table_100B()




