class_docstr: Named Entity Recognition pipeline using any `ModelForTokenClassification`. See the [named entity recognition
  examples](../task_summary#named-entity-recognition) for more information.
  
  This token recognition pipeline can currently be loaded from [`pipeline`] using the following task identifier:
  `"ner"` (for predicting the classes of tokens in a sequence: person, organisation, location or miscellaneous).
  
  The models that this pipeline can use are models that have been fine-tuned on a token classification task. See the
  up-to-date list of available models on
  [huggingface.co/models](https://huggingface.co/models?filter=token-classification).
class_docstr_tok: ["Named","Entity","Recognition","pipeline","using","any","`","ModelForTokenClassification","`",".","See","the","[","named","entity","recognition","examples","]","(","..","\/task_summary","#","named-entity-recognition",")","for","more","information",".","This","token","recognition","pipeline","can","currently","be","loaded","from","[","`","pipeline","`","]","using","the","following","task","identifier",":","`","``","ner","''","`","(","for","predicting","the","classes","of","tokens","in","a","sequence",":","person",",","organisation",",","location","or","miscellaneous",")",".","The","models","that","this","pipeline","can","use","are","models","that","have","been","fine-tuned","on","a","token","classification","task",".","See","the","up-to-date","list","of","available","models","on","[","huggingface.co\/models","]","(","https",":","\/\/huggingface.co\/models","?","filter=token-classification",")","."]
docstr: Fuse various numpy arrays into dicts with all the information needed for aggregation
docstr_tok: ["Fuse","various","numpy","arrays","into","dicts","with","all","the","information","needed","for","aggregation"]
code: def gather_pre_entities(
      self,
      sentence: str,
      input_ids: np.ndarray,
      scores: np.ndarray,
      offset_mapping: Optional[List[Tuple[int, int]]],
      special_tokens_mask: np.ndarray,
      aggregation_strategy: AggregationStrategy,
  ) -> List[dict]:
      """Fuse various numpy arrays into dicts with all the information needed for aggregation"""
      pre_entities = []
      for idx, token_scores in enumerate(scores):
          # Filter special_tokens, they should only occur
          # at the sentence boundaries since we're not encoding pairs of
          # sentences so we don't have to keep track of those.
          if special_tokens_mask[idx]:
              continue
  
          word = self.tokenizer.convert_ids_to_tokens(
              int(input_ids[idx])
          )
          if offset_mapping is not None:
              start_ind, end_ind = offset_mapping[idx]
              if not isinstance(start_ind, int):
                  if self.framework == "pt":
                      start_ind = start_ind.item()
                      end_ind = end_ind.item()
                  else:
                      start_ind = int(start_ind.numpy())
                      end_ind = int(end_ind.numpy())
              word_ref = sentence[start_ind:end_ind]
              if getattr(
                  self.tokenizer._tokenizer.model,
                  "continuing_subword_prefix",
                  None,
              ):
                  # This is a BPE, word aware tokenizer, there is a correct way
                  # to fuse tokens
                  is_subword = len(word) != len(word_ref)
              else:
                  # This is a fallback heuristic. This will fail most likely on any kind of text + punctuation mixtures that will be considered "words". Non word aware models cannot do better than this unfortunately.
                  if aggregation_strategy in {
                      AggregationStrategy.FIRST,
                      AggregationStrategy.AVERAGE,
                      AggregationStrategy.MAX,
                  }:
                      warnings.warn(
                          "Tokenizer does not support real words, using fallback heuristic",
                          UserWarning,
                      )
                  is_subword = (
                      start_ind > 0
                      and " "
                      not in sentence[start_ind - 1 : start_ind + 1]
                  )
  
              if int(input_ids[idx]) == self.tokenizer.unk_token_id:
                  word = word_ref
                  is_subword = False
          else:
              start_ind = None
              end_ind = None
              is_subword = False
  
          pre_entity = {
              "word": word,
              "scores": token_scores,
              "start": start_ind,
              "end": end_ind,
              "index": idx,
              "is_subword": is_subword,
          }
          pre_entities.append(pre_entity)
      return pre_entities
  
imports: import types
  import warnings
  from typing import List, Optional, Tuple, Union
  import numpy
  from ..models.bert.tokenization_bert import BasicTokenizer
  from ..utils import ExplicitEnum, add_end_docstrings, is_tf_available, is_torch_available
  from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline
label_desc: Use image node_id 7 for calling the TokenClassificationPipeline obj's underlying member method code with example usage: obj.gather_pre_entities(sentence, input_ids, scores, offset_mapping, special_tokens_mask, aggregation_strategy) and returns: pre_entities