class_docstr: Pretraining machinery using SCRIPT from Nijkamp et al., 2021. Always running sparse prediction.
class_docstr_tok: ["Pretraining","machinery","using","SCRIPT","from","Nijkamp","et","al.",",","2021",".","Always","running","sparse","prediction","."]
code: def _get_binary_logits(self, logits):
      # Convert to binary decision as described in SCRIPT
      # exp_logitsum = torch.exp(disc_logits).sum(dim=-1)  # autocast ok?
      # binary_logits = torch.stack([1 / (exp_logitsum + 1), exp_logitsum / (exp_logitsum + 1)], dim=-1)  # stack minus and plus
      # instead, we can also compute logit[binary_logits], which is
  
      # let y = sum(exp(logits)) / ( sum(exp(logits))+1 ), 1-y = 1 / ( sum(exp(logits))+1 )
      # log(y / (1-y)) = log( sum(exp(logits)) / ( sum(exp(logits))+1 ) * ( sum(exp(logits))+1 ) / 1)
      #                = log(sum(exp(logits))
      # Then, we can use BCEWithLogitsLoss, to safely compute logit probs via sigmoids
      return torch.logsumexp(logits, dim=-1)
  
imports: import torch
  from transformers import PretrainedConfig, PreTrainedModel
  from transformers import AutoConfig, AutoModel, AutoModelForMaskedLM, AutoModelForSequenceClassification, AutoModelForTokenClassification
  from typing import Optional
  from omegaconf import OmegaConf
  from .fff import FFF
  from .components import _get_norm_fn, _get_nonlin_fn, EmbeddingComponent, PoolingComponent, PredictionHeadComponent, GLU, get_extended_attention_mask, _init_module
  from .attention import get_attention_mechanism
label_desc: Use image node_id 4 for calling the ScriptableLMForSCRIPTTraining obj's underlying member method code with example usage: obj._get_binary_logits(logits) and returns: torch