class_docstr: GPT2 with Retriever Model.
  
  A GPT2 model that receives documents from a retriever.
class_docstr_tok: ["GPT2","with","Retriever","Model",".","A","GPT2","model","that","receives","documents","from","a","retriever","."]
docstr: Override FidModel.encoder to pack all the documents into one input example.
  
  :param input:
      2D [bsz, seqlen] input to the encoder
  :param input_lengths:
      1D [bsz] lengths of each input item
  :param query_vec:
      2D [bsz*n_turns, seqlen] input for the retriever
  :param input_turns_cnt:
      1D [bsz] number of dialogue turns for each input example
  :param input_lengths:
      1D [bsz] lengths of each target item (for each input item)
  
  :return (encoder_out, encoder_mask, input_turns_cnt, top_docs, top_doc_scores):
      encoder_out: *concatenated* encoded representations of context/document pairs
      encoder_mask: new mask for enc_out
      input_turns_cnt: pass along the input turns count for the decoder
      top_docs: List of top Documents for each batch example
      top_doc_scores: scores for each retrieved document.
docstr_tok: ["Override","FidModel.encoder","to","pack","all","the","documents","into","one","input","example",".",":","param","input",":","2D","[","bsz",",","seqlen","]","input","to","the","encoder",":","param","input_lengths",":","1D","[","bsz","]","lengths","of","each","input","item",":","param","query_vec",":","2D","[","bsz","*","n_turns",",","seqlen","]","input","for","the","retriever",":","param","input_turns_cnt",":","1D","[","bsz","]","number","of","dialogue","turns","for","each","input","example",":","param","input_lengths",":","1D","[","bsz","]","lengths","of","each","target","item","(","for","each","input","item",")",":","return","(","encoder_out",",","encoder_mask",",","input_turns_cnt",",","top_docs",",","top_doc_scores",")",":","encoder_out",":","*","concatenated","*","encoded","representations","of","context\/document","pairs","encoder_mask",":","new","mask","for","enc_out","input_turns_cnt",":","pass","along","the","input","turns","count","for","the","decoder","top_docs",":","List","of","top","Documents","for","each","batch","example","top_doc_scores",":","scores","for","each","retrieved","document","."]
code: def encoder(
      self,
      input: torch.LongTensor,
      input_lengths: torch.LongTensor,
      query_vec: torch.LongTensor,
      input_turns_cnt: torch.LongTensor,
      target_lengths: Optional[torch.LongTensor],
      positions: Optional[torch.LongTensor] = None,
      segments: Optional[torch.LongTensor] = None,
  ) -> Tuple[
      torch.Tensor,
      torch.BoolTensor,
      Optional[torch.LongTensor],
      Optional[List[List[Document]]],
      Optional[torch.Tensor],
  ]:
      """
      Override FidModel.encoder to pack all the documents into one input example.
  
      :param input:
          2D [bsz, seqlen] input to the encoder
      :param input_lengths:
          1D [bsz] lengths of each input item
      :param query_vec:
          2D [bsz*n_turns, seqlen] input for the retriever
      :param input_turns_cnt:
          1D [bsz] number of dialogue turns for each input example
      :param input_lengths:
          1D [bsz] lengths of each target item (for each input item)
  
      :return (encoder_out, encoder_mask, input_turns_cnt, top_docs, top_doc_scores):
          encoder_out: *concatenated* encoded representations of context/document pairs
          encoder_mask: new mask for enc_out
          input_turns_cnt: pass along the input turns count for the decoder
          top_docs: List of top Documents for each batch example
          top_doc_scores: scores for each retrieved document.
      """
      (
          enc_out,
          mask,
          input_turns_cnt,
          top_docs,
          top_doc_scores,
      ) = RagModel.encoder(
          self,
          input,
          input_lengths,
          query_vec,
          input_turns_cnt,
          positions,
          segments,
      )  # type: ignore
      seq_len, n_docs = enc_out.size(1), enc_out.size(0) // input.size(
          0
      )
  
      if input_turns_cnt is not None:
          # Input Turns is a tensor of dim [bsz]
          input = input.repeat_interleave(input_turns_cnt, dim=0)  # type: ignore
      doc_starts = (enc_out == self.pad_idx).sum(dim=1)  # left padded
      doc_lens = (
          seq_len - input_lengths.repeat_interleave(n_docs)
      ) - doc_starts
      # if no padding, docs are assumed to be min doc length long
      doc_lens[doc_lens.le(0)] = self.min_doc_len
      new_enc_out = enc_out.clone()
      # BEFORE:
      #  [
      #   pad...doc_0 / in_0
      #   pad...doc_1 / in_0
      #   ...
      #   pad...doc_n / in_m
      #                       ]
      total_length_i = 0
      for i, doc_len in enumerate(doc_lens):
          if i % n_docs == 0:
              total_length_i = 0
          # max doc length is determined by how much space we have after subtracting input length
          input_and_target = input_lengths[i // n_docs] + (
              target_lengths[i // n_docs]
              if target_lengths is not None
              else 0
          )
          max_doc_len = torch.div(
              (seq_len - input_and_target),
              n_docs,
              rounding_mode="floor",
          )
          max_doc_len = max(max_doc_len, self.min_doc_len)  # type: ignore
          doc_len = min(doc_len, max_doc_len)
          total_length_i += doc_len
          if i % n_docs == n_docs - 1:
              # keep the actual input context when processing the last doc.
              clamped_input_length = input_lengths[i // n_docs].clamp(
                  max=self.truncate - total_length_i
              )
              if target_lengths is not None:
                  clamped_input_length = input_lengths[
                      i // n_docs
                  ].clamp(
                      max=self.truncate
                      - total_length_i
                      - target_lengths
                  )
  
              pad_end = seq_len - clamped_input_length - doc_len
          else:
              pad_end = seq_len - doc_len
          # we simply move the portion of the doc we want to keep to the end of the tensor,
          # and mask out everything else.
          mask[i, :pad_end] = False
          new_enc_out[i, pad_end : pad_end + doc_len] = enc_out[
              i, doc_starts[i] : doc_starts[i] + doc_len
          ]
  
      new_out, new_mask = concat_enc_outs(
          input,
          new_enc_out.unsqueeze(-1),
          mask,
          1,
          self.pad_idx,
          right_padded=False,
      )
      # After:
      #  [
      #   doc_0 / doc_1 / doc_2 ... in_0
      #   doc_0 / doc_1 / doc_2 ... in_m
      #                                   ]
      return (
          new_out,
          new_mask,
          input_turns_cnt,
          top_docs,
          top_doc_scores,
      )
  
imports: from typing import Optional, Type, Tuple, List, Dict, Any
  import torch
  import torch.nn
  from parlai.agents.fid.fid import FidModel, concat_enc_outs
  from parlai.agents.hugging_face.gpt2 import GPT2Decoder, Gpt2Agent
  from parlai.agents.rag.args import RetrieverType
  from parlai.agents.rag.modules import RagModel
  from parlai.agents.rag.retrievers import Document, BLANK_DOC, SearchQuerySearchEngineRetriever, retriever_factory, RagRetriever
  from parlai.core.dict import DictionaryAgent
  from parlai.core.opt import Opt
  import parlai.utils.logging
  from projects.seeker.agents.seeker_modules import ComboFidSearchQuerySearchEngineRetriever
  from projects.seeker.agents.seeker_modules import combo_fid_retriever_factory, interleave_fid_combo_outputs, ComboFidModel
label_desc: Use image node_id 5 for calling the GPT2WithRetrieverModel obj's underlying member method code with example usage: obj.encoder(input, input_lengths, query_vec, input_turns_cnt, target_lengths, positions, segments) and returns: new_out,  new_mask,  input_turns_cnt,  top_docs,  top_doc_scores