docstr: :param corpus:
  :param model:
  :param tokenizer:
  :param feats_from_spacy_doc: if none, use offsets
  :param doc_segmenter:
  :param token_normalizer: Function which takes spacy token and returns a string.
    Not used for embedding calculation, just feature representation. Lowercases all by default.
  :param use_all_tokens_for_category_embeddings: bool, Use all tokens in segment for category embeddings?
docstr_tok: [":","param","corpus",":",":","param","model",":",":","param","tokenizer",":",":","param","feats_from_spacy_doc",":","if","none",",","use","offsets",":","param","doc_segmenter",":",":","param","token_normalizer",":","Function","which","takes","spacy","token","and","returns","a","string",".","Not","used","for","embedding","calculation",",","just","feature","representation",".","Lowercases","all","by","default",".",":","param","use_all_tokens_for_category_embeddings",":","bool",",","Use","all","tokens","in","segment","for","category","embeddings","?"]
code: def __init__(
      self,
      corpus: Union[ParsedCorpus, OffsetCorpus],
      model: PreTrainedModel,
      tokenizer: PreTrainedTokenizerFast,
      feats_from_spacy_doc: Optional[FeatsFromSpacyDoc] = None,
      doc_segmenter: Optional[
          Callable[
              [spacy.tokens.doc.Doc], Iterable[spacy.tokens.span.Span]
          ]
      ] = None,
      token_normalizer: Optional[
          Callable[[spacy.tokens.Token], str]
      ] = None,
      use_all_tokens_for_category_embeddings: bool = True,
  ):
      """
  
      :param corpus:
      :param model:
      :param tokenizer:
      :param feats_from_spacy_doc: if none, use offsets
      :param doc_segmenter:
      :param token_normalizer: Function which takes spacy token and returns a string.
        Not used for embedding calculation, just feature representation. Lowercases all by default.
      :param use_all_tokens_for_category_embeddings: bool, Use all tokens in segment for category embeddings?
      """
      self.corpus = corpus
      self.model = model
      self.tokenizer = tokenizer
      self.feats_from_spacy_doc = feats_from_spacy_doc
      if not isinstance(corpus, OffsetCorpus):
          raise Exception(
              "Since the corpus object is not an offset corpus, feats_from_spacy_doc needs to be passed."
          )
      self.doc_segmenter = (
          doc_sentence_splitter
          if doc_segmenter is None
          else doc_segmenter
      )
      self.token_normalizer = (
          lambda x: x.lower_
          if token_normalizer is None
          else token_normalizer
      )
      self.use_all_tokens_for_category_embeddings = (
          use_all_tokens_for_category_embeddings
      )
  
imports: import dataclasses
  from typing import Iterable, Optional, Callable, Union
  import flashtext
  import numpy
  import spacy
  import pandas
  from scattertext.contextual_embeddings.doc_splitter import doc_sentence_splitter
  from scattertext.features.FeatsFromSpacyDoc import FeatsFromSpacyDoc
  from scattertext.OffsetCorpus import OffsetCorpus
  from tqdm.auto import tqdm
  from scattertext.ParsedCorpus import ParsedCorpus
  from scattertext.contextual_embeddings.corpus_runing_stats import CorpusRunningStats
label_desc: Use image node_id 1 to create a new CorpusRunningStatsFactory object  with example: obj = CorpusRunningStatsFactory(corpus, model, tokenizer, feats_from_spacy_doc, doc_segmenter, token_normalizer, use_all_tokens_for_category_embeddings)