class_docstr: PyTorch implementation containing all the modelling needed for your NLP task. Combines 2 language
  models for representation of 2 sequences and a prediction head. Allows for gradient flow back to the 2 language model components.
class_docstr_tok: ["PyTorch","implementation","containing","all","the","modelling","needed","for","your","NLP","task",".","Combines","2","language","models","for","representation","of","2","sequences","and","a","prediction","head",".","Allows","for","gradient","flow","back","to","the","2","language","model","components","."]
docstr: :param language_model1: Any model that turns token ids into vector representations
  :type language_model1: LanguageModel
  :param language_model2: Any model that turns token ids into vector representations
  :type language_model2: LanguageModel
  :param prediction_heads: A list of models that take 2 sequence embeddings and return logits for a given task
  :type prediction_heads: list
  :param embeds_dropout_prob: The probability that a value in the embeddings returned by any of the 2
     language model will be zeroed.
  :param embeds_dropout_prob: float
  :param lm1_output_types: How to extract the embeddings from the final layer of the first language model. When set
                          to "per_token", one embedding will be extracted per input token. If set to
                          "per_sequence", a single embedding will be extracted to represent the full
                          input sequence. Can either be a single string, or a list of strings,
                          one for each prediction head.
  :type lm1_output_types: list or str
  :param lm2_output_types: How to extract the embeddings from the final layer of the second language model. When set
                          to "per_token", one embedding will be extracted per input token. If set to
                          "per_sequence", a single embedding will be extracted to represent the full
                          input sequence. Can either be a single string, or a list of strings,
                          one for each prediction head.
  :type lm2_output_types: list or str
  :param device: The device on which this model will operate. Either "cpu" or "cuda".
  :param loss_aggregation_fn: Function to aggregate the loss of multiple prediction heads.
                              Input: loss_per_head (list of tensors), global_step (int), batch (dict)
                              Output: aggregated loss (tensor)
                              Default is a simple sum:
                              `lambda loss_per_head, global_step=None, batch=None: sum(tensors)`
                              However, you can pass more complex functions that depend on the
                              current step (e.g. for round-robin style multitask learning) or the actual
                              content of the batch (e.g. certain labels)
                              Note: The loss at this stage is per sample, i.e one tensor of
                              shape (batchsize) per prediction head.
  :type loss_aggregation_fn: function
docstr_tok: [":","param","language_model1",":","Any","model","that","turns","token","ids","into","vector","representations",":","type","language_model1",":","LanguageModel",":","param","language_model2",":","Any","model","that","turns","token","ids","into","vector","representations",":","type","language_model2",":","LanguageModel",":","param","prediction_heads",":","A","list","of","models","that","take","2","sequence","embeddings","and","return","logits","for","a","given","task",":","type","prediction_heads",":","list",":","param","embeds_dropout_prob",":","The","probability","that","a","value","in","the","embeddings","returned","by","any","of","the","2","language","model","will","be","zeroed",".",":","param","embeds_dropout_prob",":","float",":","param","lm1_output_types",":","How","to","extract","the","embeddings","from","the","final","layer","of","the","first","language","model",".","When","set","to","``","per_token","''",",","one","embedding","will","be","extracted","per","input","token",".","If","set","to","``","per_sequence","''",",","a","single","embedding","will","be","extracted","to","represent","the","full","input","sequence",".","Can","either","be","a","single","string",",","or","a","list","of","strings",",","one","for","each","prediction","head",".",":","type","lm1_output_types",":","list","or","str",":","param","lm2_output_types",":","How","to","extract","the","embeddings","from","the","final","layer","of","the","second","language","model",".","When","set","to","``","per_token","''",",","one","embedding","will","be","extracted","per","input","token",".","If","set","to","``","per_sequence","''",",","a","single","embedding","will","be","extracted","to","represent","the","full","input","sequence",".","Can","either","be","a","single","string",",","or","a","list","of","strings",",","one","for","each","prediction","head",".",":","type","lm2_output_types",":","list","or","str",":","param","device",":","The","device","on","which","this","model","will","operate",".","Either","``","cpu","''","or","``","cuda","''",".",":","param","loss_aggregation_fn",":","Function","to","aggregate","the","loss","of","multiple","prediction","heads",".","Input",":","loss_per_head","(","list","of","tensors",")",",","global_step","(","int",")",",","batch","(","dict",")","Output",":","aggregated","loss","(","tensor",")","Default","is","a","simple","sum",":","`","lambda","loss_per_head",",","global_step=None",",","batch=None",":","sum","(","tensors",")","`","However",",","you","can","pass","more","complex","functions","that","depend","on","the","current","step","(","e.g",".","for","round-robin","style","multitask","learning",")","or","the","actual","content","of","the","batch","(","e.g",".","certain","labels",")","Note",":","The","loss","at","this","stage","is","per","sample",",","i.e","one","tensor","of","shape","(","batchsize",")","per","prediction","head",".",":","type","loss_aggregation_fn",":","function"]
code: def __init__(
      self,
      language_model1,
      language_model2,
      prediction_heads,
      embeds_dropout_prob=0.1,
      device="cuda",
      lm1_output_types=["per_sequence"],
      lm2_output_types=["per_sequence"],
      loss_aggregation_fn=None,
  ):
      """
      :param language_model1: Any model that turns token ids into vector representations
      :type language_model1: LanguageModel
      :param language_model2: Any model that turns token ids into vector representations
      :type language_model2: LanguageModel
      :param prediction_heads: A list of models that take 2 sequence embeddings and return logits for a given task
      :type prediction_heads: list
      :param embeds_dropout_prob: The probability that a value in the embeddings returned by any of the 2
         language model will be zeroed.
      :param embeds_dropout_prob: float
      :param lm1_output_types: How to extract the embeddings from the final layer of the first language model. When set
                              to "per_token", one embedding will be extracted per input token. If set to
                              "per_sequence", a single embedding will be extracted to represent the full
                              input sequence. Can either be a single string, or a list of strings,
                              one for each prediction head.
      :type lm1_output_types: list or str
      :param lm2_output_types: How to extract the embeddings from the final layer of the second language model. When set
                              to "per_token", one embedding will be extracted per input token. If set to
                              "per_sequence", a single embedding will be extracted to represent the full
                              input sequence. Can either be a single string, or a list of strings,
                              one for each prediction head.
      :type lm2_output_types: list or str
      :param device: The device on which this model will operate. Either "cpu" or "cuda".
      :param loss_aggregation_fn: Function to aggregate the loss of multiple prediction heads.
                                  Input: loss_per_head (list of tensors), global_step (int), batch (dict)
                                  Output: aggregated loss (tensor)
                                  Default is a simple sum:
                                  `lambda loss_per_head, global_step=None, batch=None: sum(tensors)`
                                  However, you can pass more complex functions that depend on the
                                  current step (e.g. for round-robin style multitask learning) or the actual
                                  content of the batch (e.g. certain labels)
                                  Note: The loss at this stage is per sample, i.e one tensor of
                                  shape (batchsize) per prediction head.
      :type loss_aggregation_fn: function
      """
  
      super(BiAdaptiveModel, self).__init__()
      self.device = device
      self.language_model1 = language_model1.to(device)
      self.lm1_output_dims = language_model1.get_output_dims()
      self.language_model2 = language_model2.to(device)
      self.lm2_output_dims = language_model2.get_output_dims()
      self.dropout1 = nn.Dropout(embeds_dropout_prob)
      self.dropout2 = nn.Dropout(embeds_dropout_prob)
      self.prediction_heads = nn.ModuleList(
          [ph.to(device) for ph in prediction_heads]
      )
      self.lm1_output_types = (
          [lm1_output_types]
          if isinstance(lm1_output_types, str)
          else lm1_output_types
      )
      self.lm2_output_types = (
          [lm2_output_types]
          if isinstance(lm2_output_types, str)
          else lm2_output_types
      )
      self.log_params()
      # default loss aggregation function is a simple sum (without using any of the optional params)
      if not loss_aggregation_fn:
          loss_aggregation_fn = loss_per_head_sum
      self.loss_aggregation_fn = loss_aggregation_fn
  
imports: import copy
  import json
  import logging
  import os
  from argparse import Namespace
  from pathlib import Path
  import multiprocessing
  import numpy
  import torch
  from torch import nn
  from farm.data_handler.processor import TextSimilarityProcessor
  from farm.data_handler.data_silo import DataSilo
  from farm.modeling.language_model import LanguageModel
  from farm.modeling.prediction_head import PredictionHead, TextSimilarityHead
  from farm.modeling.tokenization import Tokenizer
  from farm.utils import MLFlowLogger, stack
label_desc: Use image node_id 1 to create a new BiAdaptiveModel object from inherited base classes: nn, BaseBiAdaptiveModel with example: obj = BiAdaptiveModel(language_model1, language_model2, prediction_heads, embeds_dropout_prob, device, lm1_output_types, lm2_output_types, loss_aggregation_fn)