code: def main():
      # Create experiment
      experiment.create(name="transformer")
      # Create configs
      conf = Configs()
      # Override configurations
      experiment.configs(
          conf,
          {
              # Use character level tokenizer
              "tokenizer": "character",
              # Prompt separator is blank
              "prompt_separator": "",
              # Starting prompt for sampling
              "prompt": "It is ",
              # Use Tiny Shakespeare dataset
              "text": "tiny_shakespeare",
              # Use a context size of $256$
              "seq_len": 512,
              # Train for 32 epochs
              "epochs": 32,
              # Batch size $32$
              "batch_size": 16,
              # Switch between training and validation for $10$ times
              # per epoch
              "inner_iterations": 10,
              # Model size
              "d_model": 256,
              "transformer.n_heads": 16,
              "transformer.ffn.d_ff": 1024,
              # Use [Sophia optimizer](../../optimizers/sophia.html)
              "optimizer.optimizer": "Sophia",
              "optimizer.learning_rate": 3e-4,
              "optimizer.rho": 0.03,
          },
      )
  
      # Set models for saving and loading
      experiment.add_pytorch_models({"model": conf.model})
  
      # Start the experiment
      with experiment.start():
          # Run training
          conf.run()
  
file_path: annotated_deep_learning_paper_implementations/labml_nn/transformers/basic/with_sophia.py
imports: import torch
  from labml import experiment, tracker
  from labml_helpers.train_valid import BatchIndex
  from labml_nn.optimizers.sophia import Sophia
  from labml_nn.transformers.basic.autoregressive_experiment import Configs
label_desc: Use image node_id 1 for calling a global function with example usage: main() without return types