code: def drugprot_document_to_tokenlists(
      self,
      pmid: str,
      title_sentences: List[Sentence],
      abstract_sentences: List[Sentence],
      abstract_offset: int,
      entities: Dict[str, Tuple[str, int, int, str]],
      relations: Set[Tuple[str, str, str]],
  ) -> List[conllu.TokenList]:
      tokenlists: List[conllu.TokenList] = []
      sentence_id = 1
      for offset, sents in [
          (0, title_sentences),
          (abstract_offset, abstract_sentences),
      ]:
          for sent in sents:
              assert sent.start_position is not None
              assert sent.end_position is not None
              sent_char_start = sent.start_position + offset
              sent_char_end = sent.end_position + offset
  
              entities_in_sent = set()
              for entity_id, (
                  _,
                  char_start,
                  char_end,
                  _,
              ) in entities.items():
                  if (
                      sent_char_start <= char_start
                      and char_end <= sent_char_end
                  ):
                      entities_in_sent.add(entity_id)
  
              entity_char_spans = [
                  (entities[entity_id][1], entities[entity_id][2])
                  for entity_id in entities_in_sent
              ]
  
              token_offsets = [
                  (
                      sent.start_position
                      + (token.start_position or 0)
                      + offset,
                      sent.start_position
                      + (token.end_position or 0)
                      + offset,
                  )
                  for token in sent.tokens
              ]
              entity_token_spans = self.char_spans_to_token_spans(
                  entity_char_spans, token_offsets
              )
  
              tags_1 = ["O"] * len(sent)
              tags_2 = ["O"] * len(sent)
              entity_id_to_token_idx = {}
  
              ordered_entities = sorted(
                  zip(entities_in_sent, entity_token_spans),
                  key=lambda x: x[1][1] - x[1][0],
                  reverse=True,
              )
  
              for entity_id, entity_span in ordered_entities:
                  entity_id_to_token_idx[entity_id] = entity_span
  
                  # check if first tag row is already occupied
                  token_start, token_end = entity_span
                  tag_1_occupied = False
                  for i in range(token_start, token_end):
                      if tags_1[i] != "O":
                          tag_1_occupied = True
  
                  # if first tag row is occupied, use second tag row
                  tags = tags_2 if tag_1_occupied else tags_1
  
                  tag = entities[entity_id][0]
                  token_start, token_end = entity_span
                  for i in range(token_start, token_end):
                      prefix = "B-" if i == token_start else "I-"
  
                      tags[i] = prefix + tag
  
              token_dicts = []
              for i, (token, tag_1, tag_2) in enumerate(
                  zip(sent, tags_1, tags_2)
              ):
                  # hardcoded mapping TODO: perhaps find nicer solution
                  tag_1 = tag_1.replace("GENE-N", "GENE")
                  tag_1 = tag_1.replace("GENE-Y", "GENE")
                  tag_2 = tag_2.replace("GENE-N", "GENE")
                  tag_2 = tag_2.replace("GENE-Y", "GENE")
  
                  token_dicts.append(
                      Token(
                          {
                              "id": str(i + 1),
                              "form": token.text,
                              "ner": tag_1,
                              "ner-2": tag_2,
                          }
                      )
                  )
  
              relations_in_sent = []
              for relation, ent1, ent2 in [
                  r
                  for r in relations
                  if {r[1], r[2]} <= entities_in_sent
              ]:
                  subj_start = entity_id_to_token_idx[ent1][0]
                  subj_end = entity_id_to_token_idx[ent1][1]
                  obj_start = entity_id_to_token_idx[ent2][0]
                  obj_end = entity_id_to_token_idx[ent2][1]
                  relations_in_sent.append(
                      (
                          subj_start,
                          subj_end,
                          obj_start,
                          obj_end,
                          relation,
                      )
                  )
  
              metadata = {
                  "text": sent.to_original_text(),
                  "doc_id": pmid,
                  "sentence_id": str(sentence_id),
                  "relations": "|".join(
                      [
                          ";".join(
                              [
                                  str(subj_start + 1),
                                  str(subj_end),
                                  str(obj_start + 1),
                                  str(obj_end),
                                  relation,
                              ]
                          )
                          for subj_start, subj_end, obj_start, obj_end, relation in relations_in_sent
                      ]
                  ),
              }
  
              tokenlists.append(
                  conllu.TokenList(
                      tokens=token_dicts, metadata=Metadata(metadata)
                  )
              )
  
              sentence_id += 1
  
      return tokenlists
  
imports: import bisect
  import io
  import json
  import logging
  import os
  import re
  from collections import defaultdict
  from pathlib import Path
  from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union
  import conllu
  import gdown
  from conllu.models import Metadata, Token
  import flair
  from flair.data import Sentence
  from flair.datasets.sequence_labeling import ColumnCorpus
  from flair.file_utils import cached_path
  from flair.splitter import SegtokSentenceSplitter, SentenceSplitter
label_desc: Use image node_id 5 for calling the RE_ENGLISH_DRUGPROT obj's underlying member method code with example usage: obj.drugprot_document_to_tokenlists(pmid, title_sentences, abstract_sentences, abstract_offset, entities, relations) and returns: tokenlists