class_docstr: Performs Textual Inversion fine-tuning on a Determined cluster.
class_docstr_tok: ["Performs","Textual","Inversion","fine-tuning","on","a","Determined","cluster","."]
docstr: Add new concept tokens to the tokenizer and update the corresponding embedding layers in
  the text encoder.
docstr_tok: ["Add","new","concept","tokens","to","the","tokenizer","and","update","the","corresponding","embedding","layers","in","the","text","encoder","."]
code: def _add_new_tokens_and_update_embeddings(self) -> None:
      """Add new concept tokens to the tokenizer and update the corresponding embedding layers in
      the text encoder.
      """
      for concept_str, initializer_strs in zip(
          self.concept_strs, self.initializer_strs
      ):
          (
              initializer_ids,
              dummy_placeholder_ids,
              dummy_placeholder_strs,
          ) = utils.add_new_tokens_to_tokenizer(
              concept_str=concept_str,
              initializer_strs=initializer_strs,
              tokenizer=self.tokenizer,
          )
          self.concept_to_initializer_strs_map[
              concept_str
          ] = initializer_strs
          self.concept_to_initializer_ids_map[
              concept_str
          ] = initializer_ids
          self.concept_to_dummy_strs_map[
              concept_str
          ] = dummy_placeholder_strs
          self.concept_to_dummy_ids_map[
              concept_str
          ] = dummy_placeholder_ids
          self.logger.info(
              f"Added {len(dummy_placeholder_ids)} new tokens for {concept_str}."
          )
  
      # Create the dummy-to-initializer idx mapping and use the sorted values to generate the
      # updated embedding layer.
      dummy_ids_to_initializer_ids_map = {}
      for concept_str in self.concept_strs:
          for dummy_ids, initializer_ids in zip(
              self.concept_to_dummy_ids_map[concept_str],
              self.concept_to_initializer_ids_map[concept_str],
          ):
              dummy_ids_to_initializer_ids_map[
                  dummy_ids
              ] = initializer_ids
      sorted_dummy_initializer_ids_list = sorted(
          [
              (dummy_ids, init_ids)
              for dummy_ids, init_ids in dummy_ids_to_initializer_ids_map.items()
          ]
      )
      idxs_to_copy = torch.tensor(
          [init_id for _, init_id in sorted_dummy_initializer_ids_list],
          device=self.accelerator.device,
      )
      token_embedding_layer_weight_data = (
          self._get_token_embedding_layer().weight.data
      )
      copied_embedding_weights = (
          token_embedding_layer_weight_data[idxs_to_copy]
          .clone()
          .detach()
          .contiguous()
      )
  
      # Update the embedding layer.
      original_embedding = (
          self.text_encoder.text_model.embeddings.token_embedding
      )
      self.text_encoder.text_model.embeddings.token_embedding = (
          layers.ExtendedEmbedding(
              original_embedding=original_embedding,
              new_embedding_weights=copied_embedding_weights,
          )
      )
  
imports: import json
  import os
  import pathlib
  from collections import defaultdict
  from typing import List, Literal, Optional, Sequence, Union
  import accelerate
  import numpy
  import torch
  import torch.nn
  import torch.nn.functional
  from detsd import data, defaults, layers, utils
  from diffusers import AutoencoderKL, DDPMScheduler, StableDiffusionPipeline, UNet2DConditionModel
  from diffusers.pipelines.stable_diffusion import StableDiffusionSafetyChecker
  from torch.utils.data import DataLoader
  from torch.utils.tensorboard import SummaryWriter
  from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer
  import determined
  from determined.pytorch import TorchData
label_desc: Use image node_id 7 for calling the DetSDTextualInversionTrainer obj's underlying member method code with example usage: obj._add_new_tokens_and_update_embeddings() without return types