docstr: Given the text from a single document (X), and optionally the labels found
  in that document (Y), tokenize the text, and yield chunks
  
  If Y is provided, filter out chunks that do not contain any positive
  examples (labels) at a ratio determined by self.config.max_empty_chunk_ratio
docstr_tok: ["Given","the","text","from","a","single","document","(","X",")",",","and","optionally","the","labels","found","in","that","document","(","Y",")",",","tokenize","the","text",",","and","yield","chunks","If","Y","is","provided",",","filter","out","chunks","that","do","not","contain","any","positive","examples","(","labels",")","at","a","ratio","determined","by","self.config.max_empty_chunk_ratio"]
code: def text_to_tokens_mask(self, X, Y=None, context=None):
      """
      Given the text from a single document (X), and optionally the labels found
      in that document (Y), tokenize the text, and yield chunks
  
      If Y is provided, filter out chunks that do not contain any positive
      examples (labels) at a ratio determined by self.config.max_empty_chunk_ratio
      """
      pad_token = (
          [self.config.pad_token]
          if self.multi_label
          else self.config.pad_token
      )
      out_gen = self._text_to_ids(X, pad_token=pad_token)
  
      for out in out_gen:
          feats = {"tokens": out.token_ids}
          if context is not None:
              tokenized_context = tokenize_context(
                  context, out, self.config
              )
              feats["context"] = tokenized_context
          if Y is None:
              yield feats
          if Y is not None:
              min_starts = min(out.token_starts)
              max_ends = max(out.token_ends)
              filtered_labels = [
                  lab
                  for lab in Y
                  if lab["end"] >= min_starts
                  and lab["start"] <= max_ends
              ]
              empty = len(filtered_labels) == 0
              if (
                  self.config.filter_empty_examples
                  or self.empty_ratio
                  > self.config.max_empty_chunk_ratio
              ) and empty:
                  continue
              self._update_empty_ratio(empty)
              yield feats, self.label_encoder.transform(
                  out, filtered_labels
              )
  
imports: import copy
  import os
  from collections import Counter, defaultdict
  import math
  from typing import Dict, List, Tuple, Union
  import tensorflow
  import numpy
  from finetune.base import BaseModel
  from finetune.encoding.target_encoders import SequenceLabelingEncoder, SequenceMultiLabelingEncoder
  from finetune.nn.target_blocks import sequence_labeler
  from finetune.nn.crf import sequence_decode
  from finetune.encoding.sequence_encoder import finetune_to_indico_sequence
  from finetune.encoding.input_encoder import get_spacy
  from finetune.input_pipeline import BasePipeline
  from finetune.util.metrics import sequences_overlap
  from finetune.encoding.input_encoder import tokenize_context
label_desc: Use image node_id 4 for calling the SequencePipeline obj's underlying member method code with example usage: obj.text_to_tokens_mask(X, Y, context) without return types