




import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


_lowerCAmelCase						:   Dict		=   5_0_0_0_0_0

_lowerCAmelCase						:   List[str]		=   os.path.split(__file__)
_lowerCAmelCase						:   str		=   os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							**_lowerCAmelCase							:   Any	):




 """simple docstring"""







 UpperCAmelCase__							 =						dataset.map(**_lowerCAmelCase	)



@get_duration
def 						lowerCAmelCase		(		_lowerCAmelCase							:   datasets.Dataset						,							**_lowerCAmelCase							:   List[Any]	):




 """simple docstring"""







 UpperCAmelCase__							 =						dataset.filter(**_lowerCAmelCase	)



def 						lowerCAmelCase		(		):




 """simple docstring"""







 UpperCAmelCase__							 =						{"num examples": SPEED_TEST_N_EXAMPLES}
 with tempfile.TemporaryDirectory() as tmp_dir:
  UpperCAmelCase__							 =						datasets.Features({"text": datasets.Value("string"	), "numbers": datasets.Value("float32"	)}	)
  UpperCAmelCase__							 =						generate_example_dataset(
      os.path.join(_lowerCAmelCase						,							"dataset.arrow"	)						,							_lowerCAmelCase						,							num_examples=_lowerCAmelCase	)

  UpperCAmelCase__							 =						transformers.AutoTokenizer.from_pretrained("bert-base-cased"						,							use_fast=_lowerCAmelCase	)

  def tokenize(_lowerCAmelCase							:   int	):
   return tokenizer(examples["text"]	)

  UpperCAmelCase__							 =						map(_lowerCAmelCase	)

  UpperCAmelCase__							 =						map(_lowerCAmelCase						,							batched=_lowerCAmelCase	)

  UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=lambda _lowerCAmelCase	: None						,							batched=_lowerCAmelCase	)

  with dataset.formatted_as(type="numpy"	):
   UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=lambda _lowerCAmelCase	: None						,							batched=_lowerCAmelCase	)

  with dataset.formatted_as(type="pandas"	):
   UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=lambda _lowerCAmelCase	: None						,							batched=_lowerCAmelCase	)

  with dataset.formatted_as(type="torch"						,							columns="numbers"	):
   UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=lambda _lowerCAmelCase	: None						,							batched=_lowerCAmelCase	)

  with dataset.formatted_as(type="tensorflow"						,							columns="numbers"	):
   UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=lambda _lowerCAmelCase	: None						,							batched=_lowerCAmelCase	)

  UpperCAmelCase__							 =						map(_lowerCAmelCase						,							function=_lowerCAmelCase						,							batched=_lowerCAmelCase	)

  UpperCAmelCase__							 =						filter(_lowerCAmelCase	)

  # Activate later when tokenizer support batched inputs
  # with dataset.formatted_as(type='numpy'):
  #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

 with open(_lowerCAmelCase						,							"wb"	) as f:
  f.write(json.dumps(_lowerCAmelCase	).encode("utf-8"	)	)


if __name__ == "__main__":  # useful to run the profiler
  benchmark_map_filter()











import argparse
import os

import torch

from transformers.utils import WEIGHTS_NAME


_lowerCAmelCase						:   List[Any]		=   ["small", "medium", "large"]

_lowerCAmelCase						:   List[Any]		=   "lm_head.decoder.weight"
_lowerCAmelCase						:   Optional[int]		=   "lm_head.weight"



def 						lowerCAmelCase		(		_lowerCAmelCase							:   str						,							_lowerCAmelCase							:   str	):




 """simple docstring"""







 UpperCAmelCase__							 =						torch.load(_lowerCAmelCase	)
 UpperCAmelCase__							 =						d.pop(_lowerCAmelCase	)
 os.makedirs(_lowerCAmelCase						,							exist_ok=_lowerCAmelCase	)
 torch.save(_lowerCAmelCase						,							os.path.join(_lowerCAmelCase						,							_lowerCAmelCase	)	)


if __name__ == "__main__":
  _lowerCAmelCase						:   Dict		=   argparse.ArgumentParser()
  parser.add_argument("--dialogpt_path", default=".", type=str)
  _lowerCAmelCase						:   Union[str, Any]		=   parser.parse_args()
  for MODEL in DIALOGPT_MODELS:
    _lowerCAmelCase						:   Union[str, Any]		=   os.path.join(args.dialogpt_path, F'''{MODEL}_ft.pkl''')
    _lowerCAmelCase						:   List[str]		=   F'''./DialoGPT-{MODEL}'''
    convert_dialogpt_checkpoint(
        checkpoint_path,
        pytorch_dump_folder_path,
    )





