# coding=utf-8
# Copyright 2018 The Google AI Language Team Authors and The HuggingFace Inc. team.
# Copyright (c) 2018, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Fine-tuning the library models for language modeling on a text file (GPT, GPT-2, CTRL, BERT, RoBERTa, XLNet).
GPT, GPT-2 and CTRL are fine-tuned using a causal language modeling (CLM) loss. BERT and RoBERTa are fine-tuned
using a masked language modeling (MLM) loss. XLNet is fine-tuned using a permutation language modeling (PLM) loss.
"""


import logging
import math
import os
from dataclasses import dataclass, field
from typing import Optional
import torch
from transformers.optimization import AdamW, get_linear_schedule_with_warmup

from transformers import (
    CONFIG_MAPPING,
    MODEL_WITH_LM_HEAD_MAPPING,
    AutoConfig,
    AutoModelWithLMHead,
    AutoTokenizer,
    DataCollatorForLanguageModeling,
    DataCollatorForPermutationLanguageModeling,
    HfArgumentParser,
    # LineByLineTextDatasetLabels,
    LineByLineTextDataset,
    PreTrainedTokenizer,
    TextDataset,
    Trainer,
    TrainingArguments,
    set_seed,
)
import ray
from ray import tune
from transformers.file_utils import is_torch_tpu_available
from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
from ray.tune.schedulers import PopulationBasedTraining
from ray.tune import CLIReporter
# if is_wandb_available():
#   import wandb

ray.shutdown()
ray.init(log_to_driver=True, ignore_reinit_error=True)

logger = logging.getLogger(__name__)


MODEL_CONFIG_CLASSES = list(MODEL_WITH_LM_HEAD_MAPPING.keys())
MODEL_TYPES = tuple(conf.model_type for conf in MODEL_CONFIG_CLASSES)


@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune, or train from scratch.
    """

    model_name_or_path: Optional[str] = field(
        default=None,
        metadata={
            "help": "The model checkpoint for weights initialization. Leave None if you want to train a model from scratch."
        },
    )
    model_type: Optional[str] = field(
        default=None,
        metadata={"help": "If training from scratch, pass a model type from the list: " + ", ".join(MODEL_TYPES)},
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
    force_pad_token: bool = field(
        default=False,
        metadata={
            "help": "Whether to force the addition of a padding token to tokenizer that does not already have one."
        },
    )
    debiasing_head: Optional[str] = field(
        default=None, metadata={"help": "The type of de-biasing head to be used"}
    )


@dataclass
class DataTrainingArguments:
    """
    Arguments pertaining to what data we are going to input our model for training and eval.
    """

    train_data_file: Optional[str] = field(
        default=None, metadata={"help": "The input training data file (a text file)."}
    )
    eval_data_file: Optional[str] = field(
        default=None,
        metadata={"help": "An optional input evaluation data file to evaluate the perplexity on (a text file)."},
    )
    line_by_line: bool = field(
        default=False,
        metadata={"help": "Whether distinct lines of text in the dataset are to be handled as distinct sequences."},
    )

    mlm: bool = field(
        default=False, metadata={"help": "Train with masked-language modeling loss instead of language modeling."}
    )
    mlm_probability: float = field(
        default=0.15, metadata={"help": "Ratio of tokens to mask for masked language modeling loss"}
    )
    plm_probability: float = field(
        default=1 / 6,
        metadata={
            "help": "Ratio of length of a span of masked tokens to surrounding context length for permutation language modeling."
        },
    )
    max_span_length: int = field(
        default=5, metadata={"help": "Maximum length of a span of masked tokens for permutation language modeling."}
    )

    block_size: int = field(
        default=-1,
        metadata={
            "help": "Optional input sequence length after tokenization."
            "The training dataset will be truncated in block of this size for training."
            "Default to the model max input length for single sentence inputs (take into account special tokens)."
        },
    )
    overwrite_cache: bool = field(
        default=False, metadata={"help": "Overwrite the cached training and evaluation sets"}
    )


def get_dataset(
    args: DataTrainingArguments,
    tokenizer: PreTrainedTokenizer,
    evaluate: bool = False,
    cache_dir: Optional[str] = None,
):
    file_path = args.eval_data_file if evaluate else args.train_data_file
    if args.line_by_line:
        return LineByLineTextDataset(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
        # return LineByLineTextDatasetLabels(tokenizer=tokenizer, file_path=file_path, block_size=args.block_size)
    else:
        return TextDataset(
            tokenizer=tokenizer,
            file_path=file_path,
            block_size=args.block_size,
            overwrite_cache=args.overwrite_cache,
            cache_dir=cache_dir,
        )


class TuneTransformerTrainer(Trainer):

    def create_optimizer_and_scheduler(self, num_training_steps: int):
        if self.optimizer is None:
            no_decay = ["bias", "LayerNorm.weight"]
            optimizer_grouped_parameters = [
                {
                    "params": [p for n, p in self.model.named_parameters() if not any(nd in n for nd in no_decay)],
                    "weight_decay": self.args.weight_decay,
                },
                {
                    "params": [p for n, p in self.model.named_parameters() if any(nd in n for nd in no_decay)],
                    "weight_decay": 0.0,
                },
            ]
            self.optimizer = AdamW(
                optimizer_grouped_parameters,
                lr=self.args.learning_rate,
                betas=(self.args.adam_beta1, self.args.adam_beta2),
                eps=self.args.adam_epsilon,
            )
        if self.lr_scheduler is None:
            self.lr_scheduler = get_linear_schedule_with_warmup(
                self.optimizer, num_warmup_steps=self.args.warmup_steps, num_training_steps=num_training_steps
            )
        return self.current_optimizer, self.current_scheduler

    def evaluate(self,
                 eval_dataset= None):
        eval_dataloader = self.get_eval_dataloader(eval_dataset)
        output = self.prediction_loop(
            eval_dataloader, description="Evaluation")
        self.log(output.metrics)

        self.save_state()

        tune.report(**output.metrics)

        return output.metrics

    def save_state(self):
        with tune.checkpoint_dir(step=self.global_step) as checkpoint_dir:
            self.args.output_dir = checkpoint_dir
            # This is the directory name that Huggingface requires.
            output_dir = os.path.join(
                self.args.output_dir,
                f"{PREFIX_CHECKPOINT_DIR}-{self.global_step}")
            self.save_model(output_dir)
            self.current_optimizer, self.current_scheduler = self.create_optimizer_and_scheduler(360)
            if self.is_world_master():
                torch.save(self.current_optimizer.state_dict(),
                           os.path.join(output_dir, "optimizer.pt"))
                torch.save(self.current_scheduler.state_dict(),
                           os.path.join(output_dir, "scheduler.pt"))


def recover_checkpoint(tune_checkpoint_dir, model_name=None):
    if tune_checkpoint_dir is None or len(tune_checkpoint_dir) == 0:
        return model_name
    # Get subdirectory used for Huggingface.
    subdirs = [
        os.path.join(tune_checkpoint_dir, name)
        for name in os.listdir(tune_checkpoint_dir)
        if os.path.isdir(os.path.join(tune_checkpoint_dir, name))
    ]
    # There should only be 1 subdir.
    assert len(subdirs) == 1, subdirs
    return subdirs[0]


# def train_transformer(config, checkpoint_dir=None):
#     train_dataset, eval_dataset = get_datasets(config)
#
#     training_args = TrainingArguments(
#         output_dir=tune.get_trial_dir(),
#         learning_rate=config["learning_rate"],
#         do_train=True,
#         do_eval=True,
#         evaluate_during_training=True,
#         # Run eval after every epoch.
#         eval_steps=(len(train_dataset) // config["per_gpu_train_batch_size"]) +
#                    1,
#         # We explicitly set save to 0, and do checkpointing in evaluate instead
#         save_steps=0,
#         num_train_epochs=config["num_epochs"],
#         max_steps=config["max_steps"],
#         per_device_train_batch_size=config["per_gpu_train_batch_size"],
#         per_device_eval_batch_size=config["per_gpu_val_batch_size"],
#         warmup_steps=0,
#         weight_decay=config["weight_decay"],
#         logging_dir="./logs",
#     )
#
#     model_name_or_path = recover_checkpoint(checkpoint_dir, config["model_name"])
#     # num_labels = glue_tasks_num_labels[config["task_name"]]
#
#     config = AutoConfig.from_pretrained(
#         model_name_or_path,
#         num_labels=num_labels,
#         finetuning_task=task_name,
#     )
#     model = AutoModelForSequenceClassification.from_pretrained(
#         model_name_or_path,
#         config=config,
#     )
#
#     # Use our modified TuneTransformerTrainer
#     tune_trainer = TuneTransformerTrainer(
#         model=model,
#         args=training_args,
#         train_dataset=train_dataset,
#         eval_dataset=eval_dataset,
#         compute_metrics=utils.build_compute_metrics_fn(task_name),
#     )
#     tune_trainer.train(model_name_or_path)


def train_transformer(config, checkpoint_dir=None):
    # See all possible arguments in src/transformers/training_args.py
    # or by passing the --help flag to this script.
    # We now keep distinct sets of args, for a cleaner separation of concerns.

    # parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    # model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    if data_args.eval_data_file is None and training_args.do_eval:
        raise ValueError(
            "Cannot do evaluation without an evaluation data file. Either supply a file to --eval_data_file "
            "or remove the --do_eval argument."
        )

    if (
        os.path.exists(training_args.output_dir)
        and os.listdir(training_args.output_dir)
        and training_args.do_train
        and not training_args.overwrite_output_dir
    ):
        raise ValueError(
            f"Output directory ({training_args.output_dir}) already exists and is not empty. Use --overwrite_output_dir to overcome."
        )

    # Setup logging
    logging.basicConfig(
        format="%(asctime)s - %(levelname)s - %(name)s -   %(message)s",
        datefmt="%m/%d/%Y %H:%M:%S",
        level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN,
    )
    logger.warning(
        "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s",
        training_args.local_rank,
        training_args.device,
        training_args.n_gpu,
        bool(training_args.local_rank != -1),
        training_args.fp16,
    )
    logger.info("Training/evaluation parameters %s", training_args)

    # Set seed
    set_seed(training_args.seed)

    # Load pretrained model and tokenizer
    #
    # Distributed training:
    # The .from_pretrained methods guarantee that only one local process can concurrently
    # download model & vocab.

    if model_args.config_name:
        config_in = AutoConfig.from_pretrained(model_args.config_name, cache_dir=model_args.cache_dir)
    elif model_args.model_name_or_path:
        config_in = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
    else:
        config_in = CONFIG_MAPPING[model_args.model_type]()
        logger.warning("You are instantiating a new config instance from scratch.")

    if model_args.tokenizer_name:
        tokenizer = AutoTokenizer.from_pretrained(model_args.tokenizer_name, cache_dir=model_args.cache_dir)
    elif model_args.model_name_or_path:
        tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
    else:
        raise ValueError(
            "You are instantiating a new tokenizer from scratch. This is not supported, but you can do it from another script, save it,"
            "and load it from here, using --tokenizer_name"
        )
    if tokenizer.pad_token_id is None:
        if model_args.force_pad_token:
            # See PR 3388. Some tokenizers don't had pad tokens which causes errors at the encoding step in the collate_fn.
            # We give here the option to force the addition of a pad token. The attention mask is used to ignore this token
            # when feeding to the model.x
            tokenizer.add_special_tokens({"pad_token": "<pad>"})
        else:
            logger.warning(
                "Attempting to train a model whose tokenizer has no padding token. This may result in errors in the encoding step. Set the --force_pad_token flag to fix this."
            )

    model_name_or_path = recover_checkpoint(checkpoint_dir, config["model_name"])

    if model_args.model_name_or_path:
        model = AutoModelWithLMHead.from_pretrained(
            model_args.model_name_or_path,
            from_tf=bool(".ckpt" in model_args.model_name_or_path),
            config=config_in,
            cache_dir=model_args.cache_dir,
        )
    else:
        logger.info("Training new model from scratch")
        model = AutoModelWithLMHead.from_config(config_in)

    special_tokens_dict = {'bos_token': '<bos>', 'eos_token': '<eos>', 'pad_token': '<pad>'}
    num_added_toks = tokenizer.add_special_tokens(special_tokens_dict)

    model.resize_token_embeddings(len(tokenizer))

    if config_in.model_type in ["bert", "roberta", "distilbert", "camembert"] and not data_args.mlm:
        raise ValueError(
            "BERT and RoBERTa-like models do not have LM heads but masked LM heads. They must be run using the"
            "--mlm flag (masked language modeling)."
        )

    if data_args.block_size <= 0:
        data_args.block_size = tokenizer.max_len
        # Our input block size will be the max possible for the model
    else:
        data_args.block_size = min(data_args.block_size, tokenizer.max_len)

    # Get datasets

    train_dataset = (
        get_dataset(data_args, tokenizer=tokenizer, cache_dir=model_args.cache_dir) if training_args.do_train else None
    )
    # print('train_dataset {}'.format(train_dataset.examples[0]))

    eval_dataset = (
        get_dataset(data_args, tokenizer=tokenizer, evaluate=True, cache_dir=model_args.cache_dir)
        if training_args.do_eval
        else None
    )
    if config_in.model_type == "xlnet":
        data_collator = DataCollatorForPermutationLanguageModeling(
            tokenizer=tokenizer,
            plm_probability=data_args.plm_probability,
            max_span_length=data_args.max_span_length,
        )
    else:
        data_collator = DataCollatorForLanguageModeling(
            tokenizer=tokenizer, mlm=data_args.mlm, mlm_probability=data_args.mlm_probability
        )

    training_args = TrainingArguments(
        output_dir=tune.get_trial_dir(),
        learning_rate=config["learning_rate"],
        do_train=True,
        do_eval=True,
        evaluate_during_training=True,
        # Run eval after every epoch.
        eval_steps=(len(train_dataset) // config["per_gpu_train_batch_size"]) + 1,
        # We explicitly set save to 0, and do checkpointing in evaluate instead
        save_steps=0,
        num_train_epochs=config["num_epochs"],
        max_steps=config["max_steps"],
        per_device_train_batch_size=config["per_gpu_train_batch_size"],
        per_device_eval_batch_size=config["per_gpu_val_batch_size"],
        warmup_steps=0,
        weight_decay=config["weight_decay"],
        logging_dir="./logs")

    # Initialize our Trainer
    tune_trainer = TuneTransformerTrainer(
        model=model,
        args=training_args,
        data_collator=data_collator,
        train_dataset=train_dataset,
        eval_dataset=eval_dataset,
        prediction_loss_only=True,
        # compute_metrics=compute_metrics,
    )

    if training_args.do_train:
        model_path = (
            model_args.model_name_or_path
            if model_args.model_name_or_path is not None and os.path.isdir(model_args.model_name_or_path)
            else None
        )
        tune_trainer.train(model_path=model_path)


if __name__ == "__main__":

    parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
    model_args, data_args, training_args = parser.parse_args_into_dataclasses()

    config = {
      # These 3 configs below were defined earlier
      "model_name": model_args.model_name_or_path,
      "task_name": "CLM",
      "data_dir": "",
      "per_gpu_val_batch_size": 32,
      "per_gpu_train_batch_size": tune.choice([16, 32, 64]),
      "learning_rate": tune.uniform(1e-5, 5e-5),
      "weight_decay": tune.uniform(0.0, 0.3),
      "num_epochs": tune.choice([2, 3, 4, 5]),
      "max_steps": -1,  # We use num_epochs instead.
      "wandb": {
          "project": "pbt_transformers",
          "reinit": True,
          "allow_val_change": True
      }
    }

    logger.info(config)
    scheduler = PopulationBasedTraining(
          time_attr="training_iteration",
          metric="eval_loss",
          mode="min",
          perturbation_interval=2,
          hyperparam_mutations={
              "weight_decay": lambda: tune.uniform(0.0, 0.3).func(None),
              "learning_rate": lambda: tune.uniform(1e-5, 5e-5).func(None),
              "per_gpu_train_batch_size": [16, 32, 64],
          })

    reporter = CLIReporter(
          parameter_columns={
              "weight_decay": "w_decay",
              "learning_rate": "lr",
              "per_gpu_train_batch_size": "train_bs/gpu",
              "num_epochs": "num_epochs"
          },
          metric_columns=[
              "eval_acc", "eval_loss", "epoch", "training_iteration"
          ])

    analysis = tune.run(
          train_transformer,
          resources_per_trial={
              "cpu": 1,
              "gpu": 1
          },
          config=config,
          num_samples=3,
          scheduler=scheduler,
          keep_checkpoints_num=3,
          checkpoint_score_attr="training_iteration",
          progress_reporter=reporter,
          local_dir="./ray_results/",
          name="tune_trans")

    best_config = analysis.get_best_config(metric="eval_loss", mode="min")
    print(best_config)

[((57, 0, 57, 14), 'ray.shutdown', 'ray.shutdown', ({}, {}), '()', False, 'import ray\n'), ((58, 0, 58, 54), 'ray.init', 'ray.init', (), '', False, 'import ray\n'), ((60, 9, 60, 36), 'logging.getLogger', 'logging.getLogger', ({(60, 27, 60, 35): '__name__'}, {}), '(__name__)', False, 'import logging\n'), ((63, 28, 63, 61), 'transformers.MODEL_WITH_LM_HEAD_MAPPING.keys', 'MODEL_WITH_LM_HEAD_MAPPING.keys', ({}, {}), '()', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((73, 40, 78, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((83, 33, 85, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((86, 36, 88, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((89, 31, 91, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((92, 28, 97, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((98, 36, 100, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((109, 37, 111, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((112, 36, 115, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((116, 25, 119, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((121, 16, 123, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((124, 29, 126, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((127, 29, 132, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((133, 27, 135, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((137, 22, 144, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((145, 28, 147, 5), 'dataclasses.field', 'field', (), '', False, 'from dataclasses import dataclass, field\n'), ((312, 4, 316, 5), 'logging.basicConfig', 'logging.basicConfig', (), '', False, 'import logging\n'), ((328, 4, 328, 32), 'transformers.set_seed', 'set_seed', ({(328, 13, 328, 31): 'training_args.seed'}, {}), '(training_args.seed)', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((457, 13, 457, 89), 'transformers.HfArgumentParser', 'HfArgumentParser', ({(457, 30, 457, 88): '(ModelArguments, DataTrainingArguments, TrainingArguments)'}, {}), '((ModelArguments, DataTrainingArguments, TrainingArguments))', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((490, 15, 499, 12), 'ray.tune.CLIReporter', 'CLIReporter', (), '', False, 'from ray.tune import CLIReporter\n'), ((501, 15, 514, 28), 'ray.tune.run', 'tune.run', (), '', False, 'from ray import tune\n'), ((158, 15, 158, 106), 'transformers.LineByLineTextDataset', 'LineByLineTextDataset', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((161, 15, 167, 9), 'transformers.TextDataset', 'TextDataset', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((206, 8, 206, 37), 'ray.tune.report', 'tune.report', ({}, {}), '(**output.metrics)', False, 'from ray import tune\n'), ((231, 8, 231, 47), 'os.path.join', 'os.path.join', ({(231, 21, 231, 40): 'tune_checkpoint_dir', (231, 42, 231, 46): 'name'}, {}), '(tune_checkpoint_dir, name)', False, 'import os\n'), ((302, 8, 302, 48), 'os.path.exists', 'os.path.exists', ({(302, 23, 302, 47): 'training_args.output_dir'}, {}), '(training_args.output_dir)', False, 'import os\n'), ((303, 12, 303, 48), 'os.listdir', 'os.listdir', ({(303, 23, 303, 47): 'training_args.output_dir'}, {}), '(training_args.output_dir)', False, 'import os\n'), ((337, 20, 337, 102), 'transformers.AutoConfig.from_pretrained', 'AutoConfig.from_pretrained', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((345, 20, 345, 108), 'transformers.AutoTokenizer.from_pretrained', 'AutoTokenizer.from_pretrained', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((375, 16, 375, 58), 'transformers.AutoModelWithLMHead.from_config', 'AutoModelWithLMHead.from_config', ({(375, 48, 375, 57): 'config_in'}, {}), '(config_in)', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((407, 24, 411, 9), 'transformers.DataCollatorForPermutationLanguageModeling', 'DataCollatorForPermutationLanguageModeling', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((413, 24, 415, 9), 'transformers.DataCollatorForLanguageModeling', 'DataCollatorForLanguageModeling', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((466, 34, 466, 59), 'ray.tune.choice', 'tune.choice', ({(466, 46, 466, 58): '[16, 32, 64]'}, {}), '([16, 32, 64])', False, 'from ray import tune\n'), ((467, 23, 467, 47), 'ray.tune.uniform', 'tune.uniform', ({(467, 36, 467, 40): '(1e-05)', (467, 42, 467, 46): '(5e-05)'}, {}), '(1e-05, 5e-05)', False, 'from ray import tune\n'), ((468, 22, 468, 44), 'ray.tune.uniform', 'tune.uniform', ({(468, 35, 468, 38): '(0.0)', (468, 40, 468, 43): '(0.3)'}, {}), '(0.0, 0.3)', False, 'from ray import tune\n'), ((469, 20, 469, 45), 'ray.tune.choice', 'tune.choice', ({(469, 32, 469, 44): '[2, 3, 4, 5]'}, {}), '([2, 3, 4, 5])', False, 'from ray import tune\n'), ((185, 29, 190, 13), 'transformers.optimization.AdamW', 'AdamW', (), '', False, 'from transformers.optimization import AdamW, get_linear_schedule_with_warmup\n'), ((192, 32, 194, 13), 'transformers.optimization.get_linear_schedule_with_warmup', 'get_linear_schedule_with_warmup', (), '', False, 'from transformers.optimization import AdamW, get_linear_schedule_with_warmup\n'), ((211, 13, 211, 55), 'ray.tune.checkpoint_dir', 'tune.checkpoint_dir', (), '', False, 'from ray import tune\n'), ((214, 25, 216, 62), 'os.path.join', 'os.path.join', ({(215, 16, 215, 36): 'self.args.output_dir', (216, 16, 216, 61): 'f"""{PREFIX_CHECKPOINT_DIR}-{self.global_step}"""'}, {}), "(self.args.output_dir,\n    f'{PREFIX_CHECKPOINT_DIR}-{self.global_step}')", False, 'import os\n'), ((232, 20, 232, 51), 'os.listdir', 'os.listdir', ({(232, 31, 232, 50): 'tune_checkpoint_dir'}, {}), '(tune_checkpoint_dir)', False, 'import os\n'), ((339, 20, 339, 109), 'transformers.AutoConfig.from_pretrained', 'AutoConfig.from_pretrained', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((347, 20, 347, 112), 'transformers.AutoTokenizer.from_pretrained', 'AutoTokenizer.from_pretrained', (), '', False, 'from transformers import CONFIG_MAPPING, MODEL_WITH_LM_HEAD_MAPPING, AutoConfig, AutoModelWithLMHead, AutoTokenizer, DataCollatorForLanguageModeling, DataCollatorForPermutationLanguageModeling, HfArgumentParser, LineByLineTextDataset, PreTrainedTokenizer, TextDataset, Trainer, TrainingArguments, set_seed\n'), ((418, 19, 418, 39), 'ray.tune.get_trial_dir', 'tune.get_trial_dir', ({}, {}), '()', False, 'from ray import tune\n'), ((233, 25, 233, 64), 'os.path.join', 'os.path.join', ({(233, 38, 233, 57): 'tune_checkpoint_dir', (233, 59, 233, 63): 'name'}, {}), '(tune_checkpoint_dir, name)', False, 'import os\n'), ((449, 61, 449, 105), 'os.path.isdir', 'os.path.isdir', ({(449, 75, 449, 104): 'model_args.model_name_or_path'}, {}), '(model_args.model_name_or_path)', False, 'import os\n'), ((221, 27, 221, 67), 'os.path.join', 'os.path.join', ({(221, 40, 221, 50): 'output_dir', (221, 52, 221, 66): '"""optimizer.pt"""'}, {}), "(output_dir, 'optimizer.pt')", False, 'import os\n'), ((223, 27, 223, 67), 'os.path.join', 'os.path.join', ({(223, 40, 223, 50): 'output_dir', (223, 52, 223, 66): '"""scheduler.pt"""'}, {}), "(output_dir, 'scheduler.pt')", False, 'import os\n'), ((485, 38, 485, 60), 'ray.tune.uniform', 'tune.uniform', ({(485, 51, 485, 54): '0.0', (485, 56, 485, 59): '0.3'}, {}), '(0.0, 0.3)', False, 'from ray import tune\n'), ((486, 39, 486, 63), 'ray.tune.uniform', 'tune.uniform', ({(486, 52, 486, 56): '1e-05', (486, 58, 486, 62): '5e-05'}, {}), '(1e-05, 5e-05)', False, 'from ray import tune\n')]