docstr: Initialize language model sampler. This sampler replaces words with the ones
  sampled according to the output distribution of the language model. There are
  two modes to use the sampler: ``'parallel'`` and ``'autoregressive'``. In the ``'parallel'``
  mode, all words are replaced simultaneously. In the ``'autoregressive'`` model, the words
  are replaced one by one, starting from left to right. Thus the following words
  are conditioned on the previous predicted words.
  
  Parameters
  ----------
  model
      Transformers masked language model.
  perturb_opts
      Perturbation options.
docstr_tok: ["Initialize","language","model","sampler",".","This","sampler","replaces","words","with","the","ones","sampled","according","to","the","output","distribution","of","the","language","model",".","There","are","two","modes","to","use","the","sampler",":","``","'parallel","'","``","and","``","'autoregressive","'","``",".","In","the","``","'parallel","'","``","mode",",","all","words","are","replaced","simultaneously",".","In","the","``","'autoregressive","'","``","model",",","the","words","are","replaced","one","by","one",",","starting","from","left","to","right",".","Thus","the","following","words","are","conditioned","on","the","previous","predicted","words",".","Parameters","--","--","--","--","--","model","Transformers","masked","language","model",".","perturb_opts","Perturbation","options","."]
code: def __init__(
      self,
      model: LanguageModel,
      perturb_opts: dict,
  ):
      """
      Initialize language model sampler. This sampler replaces words with the ones
      sampled according to the output distribution of the language model. There are
      two modes to use the sampler: ``'parallel'`` and ``'autoregressive'``. In the ``'parallel'``
      mode, all words are replaced simultaneously. In the ``'autoregressive'`` model, the words
      are replaced one by one, starting from left to right. Thus the following words
      are conditioned on the previous predicted words.
  
      Parameters
      ----------
      model
          Transformers masked language model.
      perturb_opts
          Perturbation options.
      """
      super().__init__()
  
      # set language model and perturbation options
      self.model = model
      self.perturb_opts = perturb_opts
  
      # Define language model's vocab
      vocab: Dict[str, int] = self.model.tokenizer.get_vocab()
  
      # Define masking sampling tensor. This tensor is used to avoid sampling
      # certain tokens from the vocabulary such as: subwords, punctuation, etc.
      self.subwords_mask = np.zeros(len(vocab.keys()), dtype=np.bool_)
  
      for token in vocab:
          # Add subwords in the sampling mask. This means that subwords
          # will not be considered when sampling for the masked words.
          if self.model.is_subword_prefix(token):
              self.subwords_mask[vocab[token]] = True
              continue
  
          # Add punctuation in the sampling mask. This means that the
          # punctuation will not be considered when sampling for the masked words.
          sample_punctuation: bool = perturb_opts.get(
              "sample_punctuation", False
          )
          punctuation: str = perturb_opts.get(
              "punctuation", string.punctuation
          )
  
          if (not sample_punctuation) and self.model.is_punctuation(
              token, punctuation
          ):
              self.subwords_mask[vocab[token]] = True
  
      # define head, tail part of the text
      self.head: str = ""
      self.tail: str = ""
      self.head_tokens: List[str] = []
      self.tail_tokens: List[str] = []
  
imports: import string
  from functools import partial
  from typing import Dict, List, Optional, Tuple
  import numpy
  import tensorflow
  from alibi.utils.lang_model import LanguageModel
  from alibi.explainers.anchors.text_samplers import AnchorTextSampler
label_desc: Use image node_id 1 to create a new LanguageModelSampler object from inherited base classes: AnchorTextSampler with example: obj = LanguageModelSampler(model, perturb_opts)