class_docstr: predicts  classes for string-representation of sentence
class_docstr_tok: ["predicts","classes","for","string-representation","of","sentence"]
code: def merge_tokens(self, annotated_sentence, lang, return_proba):
      if TU.is_chinese(
          lang, strict=False
      ):  # strict=False: workaround for langdetect bug on short chinese texts
          sep = ""
      else:
          sep = " "
  
      current_token = ""
      current_tag = ""
      prob_list = []
      entities = []
      start = None
      last_end = None
  
      for tup in annotated_sentence:
          token = tup[0]
          entity = tup[1]
          if return_proba:
              prob = tup[2]
              offsets = tup[3] if len(tup) > 3 else None
          else:
              offsets = tup[2] if len(tup) > 2 else None
          tag = entity.split("-")[1] if "-" in entity else None
          prefix = entity.split("-")[0] if "-" in entity else None
          # not within entity
          if tag is None and not current_token:
              continue
          # beginning of entity
          # elif tag and prefix=='B':
          elif tag and (
              prefix == "B" or prefix == "I" and not current_token
          ):
              if current_token:  # consecutive entities
                  entities.append(
                      self._build_merge_tuple(
                          current_token,
                          current_tag,
                          start,
                          last_end,
                          prob_list,
                      )
                  )
                  prob_list = []
                  current_token = ""
                  current_tag = None
                  start, end = None, None
              current_token = token
              current_tag = tag
              start = offsets[0] if offsets else None
              last_end = offsets[1] if offsets else None
              if return_proba:
                  prob_list.append(prob)
          # end of entity
          elif tag is None and current_token:
              entities.append(
                  self._build_merge_tuple(
                      current_token,
                      current_tag,
                      start,
                      last_end,
                      prob_list,
                  )
              )
              prob_list = []
              current_token = ""
              current_tag = None
              continue
          # within entity
          elif tag and current_token:  # prefix I
              current_token = current_token + sep + token
              current_tag = tag
              last_end = offsets[1] if offsets else None
              if return_proba:
                  prob_list.append(prob)
      if current_token and current_tag:
          entities.append(
              self._build_merge_tuple(
                  current_token, current_tag, start, last_end, prob_list
              )
          )
      return entities
  
file_path: ktrain/ktrain/text/ner/predictor.py
imports: from ...None import utils
  from ...imports import *
  from ...predictor import Predictor
  from ..None import textutils
  from .preprocessor import NERPreprocessor
label_desc: Use image node_id 4 for calling the NERPredictor obj's underlying member method code with example usage: obj.merge_tokens(annotated_sentence, lang, return_proba) and returns: entities