code: def train(
      save_code: bool = False,
      run_id: int = None,
      base_model: str = "h2oai/h2ogpt-4096-llama2-7b",
      # base_model: str = 'h2oai/h2ogpt-4096-llama2-13b',
      # base_model: str = 'h2oai/h2ogpt-4096-llama2-70b',
      # only needed if base_model is self-exported HF state without tokenizer
      tokenizer_base_model: str = None,
      # tokenizer_base_model: str = 'EleutherAI/gpt-neox-20b',
      data_path: str = "h2oai/openassistant_oasst1_h2ogpt",
      data_col_dict: dict = None,
      # data_path: str = "./dai_docs.train.json",
      prompt_type: Union[
          str, int
      ] = "plain",  # "plain", "instruct", "quality", "human_bot", "dai_faq"
      valid_path: str = None,
      # valid_path: str = "./dai_docs.valid.json",
      # data_mix_in_path: str = "laion/OIG",  # way too big, medium quality
      data_mix_in_path: str = "0-hero/OIG-small-chip2",  # high quality, 50 MB, good enough for now
      data_mix_in_factor: float = 0.0,  # >1: more mix-in data, <1: more of data_path data
      data_mix_in_col_dict: dict = {
          "user": "instruction",
          "chip2": "output",
      },
      data_mix_in_prompt_type: str = "instruct",  # just instruction->output, same as instruct
      output_dir: str = None,
      # LoRA checkpoint continuation
      lora_weights: str = "",
      # batching training hyperparams
      batch_size: int = 128,
      micro_batch_size: int = 4,
      gradient_checkpointing=False,  # unnecessary with gradient accumulation enabled
      bf16=False,  # needed (and automatically enabled) for llama2-7b
      fp16=True,
      train_8bit=False,
      train_4bit=False,
      # general training hyperparams
      num_epochs: float = 1,
      learning_rate: float = 3e-4,
      # validation settings
      val_set_size: int = None,
      val_metrics: List[str] = [],
      eval_steps: int = None,  # to control eval steps via steps
      eval_epochs: float = None,  # to control eval steps via epochs
      # lora hyperparams
      lora_r: int = 8,
      lora_alpha: int = 16,
      lora_dropout: float = 0.05,
      lora_target_modules: List[str] = None,
      llama_type: bool = None,
      llama_flash_attn: bool = False,
      # llm hyperparams
      train_on_inputs: bool = True,  # if False, masks out inputs in loss
      group_by_length: bool = False,  # if True, faster, but produces an odd training loss curve
      resume_from_checkpoint: str = None,  # either training checkpoint or final adapter
      cutoff_len: int = 512,  # larger values use more memory
      drop_truncations: bool = False,  # if True, drop any truncated long sequences
      # torch training params
      ddp: bool = True,  # set to False if OOM with True, for multi-GPU model parallelism
      local_files_only: bool = False,  # else will download new versions, normally unwanted
      resume_download: bool = True,
      use_auth_token: Union[
          str, bool
      ] = False,  # True requires CLI did huggingface-cli login before running
      warmup_steps: int = 100,
      logging_steps: int = 1,
      save_steps: int = None,  # must be round multiple of eval_steps
      save_total_limit: int = 3,
      add_eos_token: bool = False,
  ):
      if llama_flash_attn:
          # Need to call this before importing transformers.
          from src.llama_flash_attn_monkey_patch import (
              replace_llama_attn_with_flash_attn,
          )
  
          replace_llama_attn_with_flash_attn()
      if "llama2-7b" in base_model:
          fp16 = False
          bf16 = True
  
      # allow set token directly
      use_auth_token = os.environ.get(
          "HUGGING_FACE_HUB_TOKEN", use_auth_token
      )
  
      prompt_type = str(prompt_type)  # migration from integers
      assert prompt_type in prompt_types
  
      world_size = int(os.getenv("WORLD_SIZE", 1))
      local_rank = int(os.getenv("LOCAL_RANK", 0))
      rank = int(os.getenv("RANK", 0))
      print(f"local_rank: {local_rank}")
      print(f"global rank: {rank}")
  
      gpus = max(world_size, torch.cuda.device_count())
      run_id = run_id or 0
      if not data_path:
          raise ValueError("No data_path provided")
      if not output_dir:
          output_dir = f"{base_model.split('/')[-1]}.{data_path.replace('/', '')}.{num_epochs}_epochs.{get_githash() or 'nogit'}.{run_id}"
          if os.path.exists(output_dir) and not resume_from_checkpoint:
              raise FileExistsError(
                  f"output_dir {output_dir} based on run_id {run_id} already exists. Please pick a different run_id."
              )
      else:
          if os.path.exists(output_dir) and not resume_from_checkpoint:
              raise FileExistsError(
                  f"output_dir {output_dir} already exists. Please pick a different output_dir, or specify a run_id instead."
              )
      device_map = "auto"
  
      if save_code:
          copy_code(run_id)
      if tokenizer_base_model is None:
          tokenizer_base_model = base_model
      if llama_type is None:
          llama_type = "llama" in base_model.lower()
      if llama_type and llama_flash_attn:
          from importlib.metadata import (
              distribution,
              PackageNotFoundError,
          )
  
          try:
              distribution("flash_attn")
              can_do_flash_attn = True
          except (PackageNotFoundError, AssertionError):
              can_do_flash_attn = False
  
          if not can_do_flash_attn:
              raise RuntimeError(
                  """Flash attention not installed.
              NOTE: for current pytorch 2.0, flash attention requires installing cuda 11.7 via https://developer.nvidia.com/cuda-11-7-0-download-archive?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=runfile_local and then when running, to avoid installing driver, docs, samples, just install toolkit.  Then when pip installing flash attention do:
  
              CUDA_HOME=/usr/local/cuda-11.7 pip install flash-attn"""
              )
      assert (
          base_model
      ), "Please specify a --base_model, e.g. --base_model='decapoda-research/llama-7b-hf'"
      gradient_accumulation_steps = batch_size // micro_batch_size
      assert (
          gradient_accumulation_steps >= world_size
      ), "must increase batch_size for multi-GPU"
  
      device_map = "auto"
  
      locals_dict = locals()
      locals_print = "\n".join(
          ["%s: %s" % (k, v) for k, v in locals_dict.items()]
      )
      log(f"Training model with params:\n{locals_print}")
      log(
          "Command: %s\nHash: %s"
          % (str(" ".join(sys.argv)), get_githash())
      )
  
      max_memory = None
      if gpus > 1:
          if ddp:
              log("Distributed: data parallel")
              device_map = {"": int(os.environ.get("LOCAL_RANK") or 0)}
              gradient_accumulation_steps = (
                  gradient_accumulation_steps // world_size
              )
          else:
              free_in_GB = int(
                  min(torch.cuda.mem_get_info()) / 1024**3
              )
              max_memory = f"{free_in_GB - 2}GB"
              max_memory = {i: max_memory for i in range(gpus)}
              log("world_size: %d" % world_size)
              log("num_gpus: %d" % gpus)
              log("max mem: %s" % max_memory)
  
      model_loader, tokenizer_loader, conditional_type = get_loaders(
          model_name=base_model,
          reward_type=False,
          llama_type=llama_type,
      )
  
      model = model_loader(
          base_model,
          load_in_8bit=train_8bit,
          load_in_4bit=train_4bit,
          device_map=device_map,
          torch_dtype=torch.float16,
          max_memory=max_memory,
          local_files_only=local_files_only,
          trust_remote_code=True,
          resume_download=resume_download,
          token=use_auth_token,
      )
      print(model)
      if gpus > 1:
          if not ddp:
              log("model parallel")
              model.is_parallelizable = True
              model.model_parallel = True
  
      tokenizer = get_tokenizer(
          tokenizer_loader,
          tokenizer_base_model,
          local_files_only,
          resume_download,
          use_auth_token,
      )
  
      if train_8bit or train_4bit:
          from peft import (
              prepare_model_for_kbit_training,
          )
  
          model = prepare_model_for_kbit_training(model)
  
      from peft import (
          LoraConfig,
          get_peft_model,
          set_peft_model_state_dict,
      )
  
      try:
          from peft import utils
  
          lora_mappings = (
              utils.TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING.copy()
          )
      except AttributeError:
          from peft import mapping
  
          lora_mappings = (
              mapping.TRANSFORMERS_MODELS_TO_LORA_TARGET_MODULES_MAPPING.copy()
          )
      lora_mappings["distilgpt2"] = ["c_attn"]
  
      if lora_weights:
          from peft import PeftModel
  
          model = PeftModel.from_pretrained(
              model,
              lora_weights,
              torch_dtype=torch.float16,
              device_map=device_map,
              local_files_only=local_files_only,
              resume_download=resume_download,
              token=use_auth_token,
          )
      elif lora_r > 0:
          if lora_target_modules is None:
              base_model_lower = base_model.lower()
              if base_model_lower in lora_mappings:
                  lora_target_modules_cand = [
                      lora_mappings[base_model_lower]
                  ]
              else:
                  lora_target_modules_cand = [
                      ["query_key_value"],
                      ["q_proj", "v_proj"],
                  ]
          else:
              lora_target_modules_cand = [lora_target_modules]
  
          for lora_target_modules in lora_target_modules_cand:
              try:
                  config = LoraConfig(
                      r=lora_r,
                      lora_alpha=lora_alpha,
                      target_modules=lora_target_modules,
                      lora_dropout=lora_dropout,
                      bias="none",
                      task_type="CAUSAL_LM",
                  )
                  model = get_peft_model(model, config)
                  break
              except ValueError as e:
                  if "Target modules" in str(e) and "not found" in str(
                      e
                  ):
                      continue
                  else:
                      raise
          from peft import PeftModel
  
          assert isinstance(
              model, PeftModel
          ), "LoRA failed. Please provide --lora_target_modules explicitly."
      if resume_from_checkpoint:
          # Check the available weights and load them
          checkpoint_name = os.path.join(
              resume_from_checkpoint, "pytorch_model.bin"
          )  # Full checkpoint
          if not os.path.exists(checkpoint_name):
              checkpoint_name = os.path.join(
                  resume_from_checkpoint, "adapter_model.bin"
              )  # only LoRA model - LoRA config above has to fit
              resume_from_checkpoint = (
                  False  # So the trainer won't try loading its state
              )
          # The two files above have a different name depending on how they were saved, but are actually the same.
          if os.path.exists(checkpoint_name):
              log(f"Restarting from {checkpoint_name}")
              adapters_weights = torch.load(checkpoint_name)
              set_peft_model_state_dict(model, adapters_weights)
          else:
              log(f"Checkpoint {checkpoint_name} not found")
  
      print(model)
      try:
          # only for PeftModel
          model.print_trainable_parameters()  # Be more transparent about the % of trainable params.
      except:
          pass
  
      metrics = {}
      for name in supported_metrics:
          if name in val_metrics:
              import evaluate  # Causes hang for 'python generate.py' on dual 4090 if imported early, 100% reproducible
  
              metrics[name] = evaluate.load(name)
      log("Using Validation Metrics: %s" % str(list(metrics.keys())))
      log("Supported Metrics: %s" % supported_metrics)
  
      if val_set_size is None:
          if len(metrics) == 0:
              val_set_size = 1000
          else:
              val_set_size = 100
          log("Auto set val_set_size %s" % val_set_size)
      elif val_set_size < 1.0 and val_set_size != 0:
          raise RuntimeError(
              "Fractional validation size not supported."
          )
  
      from datasets import load_dataset, concatenate_datasets
  
      if valid_path:
          data = load_dataset(
              "json",
              data_files={"train": data_path, "valid": valid_path},
          )
      else:
          if "json" in data_path:
              data = load_dataset(
                  "json", data_files={"train": data_path}
              )
          else:
              data = load_dataset(data_path)
              data = data.rename_columns(data_col_dict or {})
  
      valid_data = None
      train_data_mix_in = None
      valid_data_mix_in = None
  
      if data_mix_in_path and data_mix_in_factor > 0:
          # get mix-in training/validation data - to keep model "sane"
          num_rows = data["train"].num_rows
          log("Loading mix-in dataset: %s" % data_mix_in_path)
          if "json" in data_mix_in_path:
              data_mix_in = load_dataset(
                  "json", data_files={"train": data_mix_in_path}
              )["train"]
          else:
              data_mix_in = load_dataset(data_mix_in_path)[
                  "train"
              ]  # can be large
          data_mix_in = data_mix_in.rename_columns(
              data_mix_in_col_dict or {}
          )
          mix_in_rows = int(num_rows * data_mix_in_factor)
  
          if mix_in_rows > data_mix_in.num_rows:
              # duplicate rows if mix-in is smaller than required
              log(
                  "Duplicating mixin to compensate for its size for training size and mixin fraction"
              )
              data_mix_in = concatenate_datasets(
                  [data_mix_in]
                  * int(np.ceil(mix_in_rows / data_mix_in.num_rows))
              )
  
          # only get as much as we need to balance
          valid_size = min(data_mix_in.num_rows // 2, val_set_size or 0)
          train_size = max(
              1, min(data_mix_in.num_rows - valid_size, mix_in_rows)
          )
          mixin_small = data_mix_in.train_test_split(
              test_size=train_size + valid_size,
              shuffle=True,
              seed=np.random.randint(10000),
          )["test"]
          if valid_size:
              mixin_train_test = mixin_small.train_test_split(
                  test_size=valid_size,
                  shuffle=False,
              )
              train_data_mix_in = mixin_train_test["train"]
              valid_data_mix_in = mixin_train_test["test"]
          else:
              train_data_mix_in = mixin_small
  
          if "prompt_type" not in train_data_mix_in.column_names:
              train_data_mix_in = train_data_mix_in.add_column(
                  "prompt_type",
                  [data_mix_in_prompt_type]
                  * train_data_mix_in.num_rows,
              )
              log(
                  "Added prompt type %s to mix-in training data"
                  % data_mix_in_prompt_type
              )
          if (
              valid_data_mix_in
              and "prompt_type" not in valid_data_mix_in.column_names
          ):
              valid_data_mix_in = valid_data_mix_in.add_column(
                  "prompt_type",
                  [data_mix_in_prompt_type]
                  * valid_data_mix_in.num_rows,
              )
              log(
                  "Added prompt type %s to mix-in validation data"
                  % data_mix_in_prompt_type
              )
          log(
              "Created mix-in data:\nTrain %s\nValid %s"
              % (train_data_mix_in, valid_data_mix_in)
          )
  
      # get our own training/validation data - for fine-tuning
      if val_set_size > 0 and not valid_path and not data_mix_in_path:
          # create valid split from train
          train_val = data["train"].train_test_split(
              test_size=val_set_size, shuffle=True, seed=42
          )
          train_data = train_val["train"]
          valid_data = train_val["test"]
      else:
          train_data = data["train"]
          if valid_path:
              # use given valid split, has priority over data_mix_in_path
              valid_data = data["valid"]
      if "prompt_type" not in train_data.column_names:
          train_data = train_data.add_column(
              "prompt_type",
              [prompt_type] * train_data.num_rows,
          )
          log("Added prompt type %s to training data" % prompt_type)
      if valid_data and "prompt_type" not in valid_data.column_names:
          valid_data = valid_data.add_column(
              "prompt_type",
              [prompt_type] * valid_data.num_rows,
          )
          log("Added prompt type %s to validation data" % prompt_type)
  
      assert train_data is not None
  
      generate_and_tokenize_prompt_fun = partial(
          generate_and_tokenize_prompt,
          prompt_type=prompt_type,
          train_on_inputs=train_on_inputs,
          add_eos_token=add_eos_token,
          cutoff_len=cutoff_len,
          tokenizer=tokenizer,
      )
  
      # shuffle and tokenize data
      if train_data_mix_in:
          train_data = concatenate_datasets(
              [train_data, train_data_mix_in]
          )
      log("Tokenizing %s training rows" % train_data.num_rows)
      train_data = train_data.shuffle().map(
          generate_and_tokenize_prompt_fun,
          num_proc=os.cpu_count() // torch.cuda.device_count(),
      )
      if drop_truncations:
          log(
              "avoid keeping truncated cases to avoid contaminating model with truncation cases.  Original size: %s"
              % train_data.num_rows
          )
          prune_long_sequences_func = partial(
              prune_long_sequences, cutoff_len=cutoff_len
          )
          train_data = train_data.filter(
              prune_long_sequences_func,
              num_proc=os.cpu_count() // torch.cuda.device_count(),
          )
          log(
              "avoid keeping truncated cases to avoid contaminating model with truncation cases.  New size: %s"
              % train_data.num_rows
          )
      train_set_size = len(train_data)
  
      if valid_data and valid_data_mix_in:
          valid_data = concatenate_datasets(
              [valid_data, valid_data_mix_in]
          )
      elif valid_data_mix_in:
          valid_data = valid_data_mix_in
  
      if valid_data:
          log("Tokenizing %s validation rows" % valid_data.num_rows)
          valid_data = valid_data.shuffle().map(
              generate_and_tokenize_prompt_fun,
              num_proc=os.cpu_count() // torch.cuda.device_count(),
          )
          val_set_size = len(valid_data)
      else:
          val_set_size = 0
      log(
          "Final fine-tuning data:\nTrain %s\nValid %s"
          % (train_data, valid_data)
      )
      sample_row_dict = train_data[:1]
      del sample_row_dict["input_ids"]
      del sample_row_dict["attention_mask"]
      del sample_row_dict["labels"]
      log("Sample input: %s" % sample_row_dict)
  
      try:
          import neptune
          from transformers.integrations import NeptuneCallback
  
          neptune_run = neptune.init_run(
              source_files=[],
          )
          log("Connected to Neptune.")
      except ImportError:
          neptune_run = None
          log("Please pip install neptune for tracking.")
      except neptune.exceptions.NeptuneMissingApiTokenException:
          neptune_run = None
          os.environ["NEPTUNE_MODE"] = "debug"
          log("No neptune configured, set NEPTUNE_API_TOKEN env var.")
  
      if neptune_run:
          neptune_callback = NeptuneCallback(run=neptune_run)
          callbacks = [neptune_callback]
      else:
          from transformers.integrations import (
              TensorBoardCallback,
              is_tensorboard_available,
          )
  
          if is_tensorboard_available:
              # tensorboard --logdir=runs/
              from torch.utils.tensorboard import SummaryWriter
  
              tb_writer = SummaryWriter()
              callbacks = [TensorBoardCallback(tb_writer=tb_writer)]
          else:
              callbacks = []
  
      expected_steps = (train_set_size * num_epochs) // batch_size
      if eval_steps is None and eval_epochs is None:
          # 20 evaluations for a run
          eval_steps = max(1, int(expected_steps / 20))
          log(
              "Auto set eval_steps to %s out of %s total training steps"
              % (eval_steps, expected_steps)
          )
      elif eval_steps is None and eval_epochs is not None:
          eval_steps = max(
              1, int(expected_steps * eval_epochs / num_epochs)
          )
          log(
              "Auto converted eval_epochs=%s to eval_steps %s"
              " out of %s total training steps"
              % (eval_epochs, eval_steps, expected_steps)
          )
      if save_steps is None:
          save_steps = eval_steps
          log("Auto step save_steps to %s" % save_steps)
      elif save_steps > eval_steps:
          # save steps must be round multiple of eval_steps
          save_steps0 = save_steps
          save_steps = max(1, (save_steps // eval_steps)) * eval_steps
          if save_steps0 != save_steps:
              log(
                  "Auto converted save_steps from %s to %s"
                  % (save_steps0, save_steps)
              )
  
      def compute_metrics(eval_preds):
          # e.g. see: https://huggingface.co/docs/transformers/v4.25.1/en/tasks/translation#evaluate
          inputs = eval_preds.inputs
          label_ids = eval_preds.label_ids
          predictions = eval_preds.predictions
  
          # inputs = np.where(inputs != -100, inputs, tokenizer.pad_token_id)
          # decoded_inputs = tokenizer.batch_decode(inputs, skip_special_tokens=True)
          # decoded_inputs = [pred.strip() for pred in decoded_inputs]
  
          label_ids = np.where(
              label_ids != -100, label_ids, tokenizer.pad_token_id
          )
          # tokenizer behavior like generate time
          decoded_labels = tokenizer.batch_decode(
              label_ids,
              skip_special_tokens=True,
              clean_up_tokenization_spaces=True,
          )
          decoded_labels = [pred.strip() for pred in decoded_labels]
  
          predictions = np.argmax(predictions, -1)
          predictions = np.where(
              predictions != -100, predictions, tokenizer.pad_token_id
          )
          # tokenizer behavior like generate time
          decoded_predictions = tokenizer.batch_decode(
              predictions,
              skip_special_tokens=True,
              clean_up_tokenization_spaces=True,
          )
          decoded_predictions = [
              pred.strip() for pred in decoded_predictions
          ]
  
          result = {}
          for metric in metrics.values():
              result1 = metric.compute(
                  predictions=decoded_predictions,
                  references=decoded_labels,
              )
              # get rid of lists, for precision etc., for now
              numeric_results = {
                  k: v
                  for k, v in result1.items()
                  if isinstance(v, (int, float))
              }
              result.update(numeric_results)
          return result
  
      # the callback that computes metrics of interest
      if val_metrics:
          trainer_kwargs = dict(compute_metrics=compute_metrics)
      else:
          trainer_kwargs = dict()
  
      import transformers
  
      trainer = transformers.Trainer(
          model=model,
          tokenizer=tokenizer,
          train_dataset=train_data,
          eval_dataset=valid_data,
          # FIXME: might need Seq2SeqTrainingArguments for some models
          args=transformers.TrainingArguments(
              per_device_train_batch_size=micro_batch_size,
              per_device_eval_batch_size=1,
              eval_accumulation_steps=10,
              # predict_with_generate=True,  # SEQ2SEQ only
              include_inputs_for_metrics=True,
              gradient_accumulation_steps=gradient_accumulation_steps,
              warmup_steps=warmup_steps,
              num_train_epochs=num_epochs,
              learning_rate=learning_rate,
              gradient_checkpointing=gradient_checkpointing,
              bf16=bf16,
              fp16=fp16,
              # cosnider 8-bit adam: https://huggingface.co/docs/transformers/v4.18.0/en/performance#8bit-adam
              optim="adamw_torch",  # consider "adafactor" to save memory
              logging_steps=logging_steps,
              logging_strategy="steps",
              evaluation_strategy="steps" if val_set_size > 0 else "no",
              save_strategy="steps",
              eval_steps=eval_steps if val_set_size > 0 else None,
              save_steps=save_steps,
              output_dir=output_dir,
              save_total_limit=save_total_limit,
              load_best_model_at_end=True
              if val_set_size > 0
              else False,
              ddp_find_unused_parameters=False if ddp else None,
              group_by_length=group_by_length,
              # fsdp=gpus > 1 and not ddp,
              report_to="tensorboard" if not neptune_run else "neptune",
          ),
          data_collator=transformers.DataCollatorForSeq2Seq(
              tokenizer,
              pad_to_multiple_of=8,
              return_tensors="pt",
              padding=True,
          ),
          callbacks=callbacks,
          **trainer_kwargs,
      )
      model.config.use_cache = False
  
      if torch.__version__ >= "2" and sys.platform != "win32":
          model = torch.compile(model)
          # WIP (not generally replacing layers until pytorch 2.1)
          if not llama_flash_attn:
              torch.backends.cuda.enable_flash_sdp(True)
  
      if gpus > 1 and not ddp:
          assert trainer.is_model_parallel
      else:
          assert not trainer.is_model_parallel
      trainer.train(resume_from_checkpoint=resume_from_checkpoint)
  
      model.save_pretrained(output_dir)
  
      log(
          "\n If there's a warning about missing keys above, please disregard :)"
      )
  
file_path: h2ogpt/finetune.py
imports: import os
  import sys
  from functools import partial
  from typing import List, Union
  import numpy
  from src.loaders import get_loaders, get_tokenizer
  from src.prompter import generate_prompt, prompt_types, PromptType
  from src.utils import get_githash, copy_code, H2O_Fire
  import torch
label_desc: Use image node_id 2 for calling a global function with example usage: train(save_code, run_id, base_model, tokenizer_base_model, data_path, data_col_dict, prompt_type, valid_path, data_mix_in_path, data_mix_in_factor, data_mix_in_col_dict, data_mix_in_prompt_type, output_dir, lora_weights, batch_size, micro_batch_size, gradient_checkpointing, bf16, fp16, train_8bit, train_4bit, num_epochs, learning_rate, val_set_size, val_metrics, eval_steps, eval_epochs, lora_r, lora_alpha, lora_dropout, lora_target_modules, llama_type, llama_flash_attn, train_on_inputs, group_by_length, resume_from_checkpoint, cutoff_len, drop_truncations, ddp, local_files_only, resume_download, use_auth_token, warmup_steps, logging_steps, save_steps, save_total_limit, add_eos_token) and returns: result