class_docstr: Performs Textual Inversion fine-tuning on a Determined cluster.
class_docstr_tok: ["Performs","Textual","Inversion","fine-tuning","on","a","Determined","cluster","."]
docstr: Creates a DetSDTextualInversionTrainer instance on the cluster, drawing hyperparameters
  and other needed information from the Determined master.  Expects the `hyperparameters`
  section of the config to be organized into the following sections:
  - `model`: specifies which model weights to use.
  - `concepts`: all pertinent information about the to-be-added concepts.
  - `training`: controls details of the training process.
  - `inference`: optionally generate images during training using parameters specified here.
docstr_tok: ["Creates","a","DetSDTextualInversionTrainer","instance","on","the","cluster",",","drawing","hyperparameters","and","other","needed","information","from","the","Determined","master",".","Expects","the","`","hyperparameters","`","section","of","the","config","to","be","organized","into","the","following","sections",":","-","`","model","`",":","specifies","which","model","weights","to","use",".","-","`","concepts","`",":","all","pertinent","information","about","the","to-be-added","concepts",".","-","`","training","`",":","controls","details","of","the","training","process",".","-","`","inference","`",":","optionally","generate","images","during","training","using","parameters","specified","here","."]
code: def train_on_cluster(cls) -> None:
      """Creates a DetSDTextualInversionTrainer instance on the cluster, drawing hyperparameters
      and other needed information from the Determined master.  Expects the `hyperparameters`
      section of the config to be organized into the following sections:
      - `model`: specifies which model weights to use.
      - `concepts`: all pertinent information about the to-be-added concepts.
      - `training`: controls details of the training process.
      - `inference`: optionally generate images during training using parameters specified here.
      """
      info = det.get_cluster_info()
      assert (
          info is not None
      ), "train_on_cluster() must be called on a Determined cluster."
      hparams = info.trial.hparams
      trial_id = info.trial.trial_id
      latest_checkpoint = info.latest_checkpoint
  
      # Instantiate the trainer and train.
      trainer = cls(
          **hparams["model"],
          **hparams["concepts"],
          **hparams["training"],
          **hparams["inference"],
      )
  
      trainer.logger.info(
          "--------------- Starting Training ---------------"
      )
      trainer.logger.info(
          f"Effective global batch size: {trainer.effective_global_batch_size}"
      )
      trainer.logger.info(
          f"{'(Scaled) ' if trainer.scale_lr else ''}Learning rate: {trainer.learning_rate}"
      )
      trainer.logger.info(
          f"Train dataset size: {len(trainer.train_dataset)}"
      )
  
      try:
          distributed = (
              det.core.DistributedContext.from_torch_distributed()
          )
      except KeyError:
          distributed = None
  
      with det.core.init(
          distributed=distributed,
          tensorboard_mode=det.core.TensorboardMode.MANUAL,
      ) as core_context:
          if latest_checkpoint is not None:
              trainer._restore_latest_checkpoint(
                  core_context=core_context,
                  latest_checkpoint=latest_checkpoint,
                  trial_id=trial_id,
              )
  
          # There will be a single op of len max_length, as defined in the searcher config.
          for op in core_context.searcher.operations():
              while trainer.steps_completed < op.length:
                  for batch in trainer.train_dataloader:
                      # Use the accumulate method for efficient gradient accumulation.
                      with trainer.accelerator.accumulate(
                          trainer.text_encoder
                      ):
                          trainer._train_one_batch(batch)
                      took_sgd_step = trainer.accelerator.sync_gradients
                      if took_sgd_step:
                          trainer.steps_completed += 1
                          trainer.logger.info(
                              f"Step {trainer.steps_completed} completed."
                          )
  
                          is_end_of_training = (
                              trainer.steps_completed == op.length
                          )
                          time_to_report = (
                              trainer.steps_completed
                              % trainer.metric_report_freq
                              == 0
                          )
                          time_to_ckpt = (
                              trainer.steps_completed
                              % trainer.checkpoint_freq
                              == 0
                          )
  
                          # Report metrics, checkpoint, and preempt as appropriate.
                          if (
                              is_end_of_training
                              or time_to_report
                              or time_to_ckpt
                          ):
                              trainer._report_train_metrics(
                                  core_context
                              )
                              # report_progress for Web UI progress-bar rendering.
                              if trainer.accelerator.is_main_process:
                                  op.report_progress(
                                      trainer.steps_completed
                                  )
                          if is_end_of_training or time_to_ckpt:
                              trainer._save(core_context, trial_id)
                              if core_context.preempt.should_preempt():
                                  return
                          if is_end_of_training:
                              break
              if trainer.accelerator.is_main_process:
                  # Report the final mean loss.
                  op.report_completed(trainer.last_mean_loss)
  
imports: import json
  import os
  import pathlib
  from collections import defaultdict
  from typing import List, Literal, Optional, Sequence, Union
  import accelerate
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from detsd import data, defaults, layers, utils
  from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionPipeline, UNet2DConditionModel
  from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
  from torch.utils.data import DataLoader
  from torch.utils.tensorboard import SummaryWriter
  from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
  import determined
  from determined.pytorch import TorchData
label_desc: Use image node_id 2 for calling the DetSDTextualInversionTrainer obj's underlying member method code with example usage: obj.train_on_cluster(cls) without return types