docstr: Tokenizes and truncates description; combines it with the tokenized context and generates one input sample for bert
docstr_tok: ["Tokenizes","and","truncates","description",";","combines","it","with","the","tokenized","context","and","generates","one","input","sample","for","bert"]
code: def _get_candidate_representation(
      context_tokens,
      candidate_desc,
      tokenizer,
      max_seq_length,
      max_sub_seq_length,
  ):
      """Tokenizes and truncates description; combines it with the tokenized context and generates one input sample for bert"""
      candidate_desc_tokens = tokenizer.tokenize(candidate_desc)
      candidate_desc_tokens = candidate_desc_tokens[:max_sub_seq_length]
  
      tokens = (
          ["[CLS]"]
          + context_tokens
          + ["[SEP]"]
          + candidate_desc_tokens
          + ["[SEP]"]
      )
      segment_ids = [0] * (len(context_tokens) + 2) + [1] * (
          len(candidate_desc_tokens) + 1
      )
      input_ids = tokenizer.convert_tokens_to_ids(tokens)
      input_mask = [1] * len(input_ids)
  
      # Zero-pad up to the sequence length
      padding = [0] * (max_seq_length - len(input_ids))
      input_ids += padding
      input_mask += padding
      segment_ids += padding
  
      assert len(input_ids) == max_seq_length
      assert len(input_mask) == max_seq_length
      assert len(segment_ids) == max_seq_length
  
      return {
          "tokens": tokens,
          "input_ids": input_ids,
          "input_mask": input_mask,
          "segment_ids": segment_ids,
      }
  
imports: import torch
  import os
  import numpy
  from pytorch_transformers.modeling_bert import BertPreTrainedModel, BertConfig, BertModel
  from pytorch_transformers.tokenization_bert import BertTokenizer
  from torch.utils.data import DataLoader, SequentialSampler, TensorDataset
  from torch import nn
  from torch.nn import CrossEntropyLoss, MSELoss
  from tqdm import tqdm
  from pytorch_transformers.optimization import AdamW, WarmupLinearSchedule
  from pytorch_transformers.file_utils import PYTORCH_PRETRAINED_BERT_CACHE
label_desc: Use image node_id 6 for calling the BertReranker obj's underlying member method code with example usage: obj._get_candidate_representation(context_tokens, candidate_desc, tokenizer, max_seq_length, max_sub_seq_length) and returns: dict