code: """
  :mod:`textacy.vsm.vectorizers`:  Transform a collection of tokenized documents into
  a document-term matrix of shape (# docs, # unique terms), with various ways to filter
  or limit included terms and flexible weighting schemes for their values.
  
  A second option aggregates terms in tokenized documents by provided group labels,
  resulting in a "group-term-matrix" of shape (# unique groups, # unique terms),
  with filtering and weighting functionality as described above.
  
  See the :class:`Vectorizer` and :class:`GroupVectorizer` docstrings for usage
  examples and explanations of the various weighting schemes.
  """
  import collections
  import operator
  from array import array
  from typing import Dict, List
  
  import numpy as np
  import scipy.sparse as sp
  from sklearn.preprocessing import normalize as normalize_mat
  
  from .. import errors
  from .matrix_utils import get_doc_lengths, get_inverse_doc_freqs, filter_terms_by_df
  
  
  BM25_K1 = 1.6  # value typically bounded in [1.2, 2.0]
  BM25_B = 0.75
  
  
  class Vectorizer:
      """
      Transform one or more tokenized documents into a sparse document-term matrix
      of shape (# docs, # unique terms), with flexibly weighted and normalized values.
  
      Stream a corpus with metadata from disk:
  
      .. code-block:: pycon
  
          >>> ds = textacy.datasets.CapitolWords()
          >>> records = ds.records(limit=1000)
          >>> corpus = textacy.Corpus("en", data=records)
          >>> corpus
          Corpus(1000 docs; 538172 tokens)
  
      Tokenize and vectorize the first 600 documents of this corpus:
  
      .. code-block:: pycon
  
          >>> tokenized_docs = (
          ...     doc._.to_terms_list(ngrams=1, entities=True, as_strings=True)
          ...     for doc in corpus[:600])
          >>> vectorizer = Vectorizer(
          ...     apply_idf=True, norm="l2",
          ...     min_df=3, max_df=0.95)
          >>> doc_term_matrix = vectorizer.fit_transform(tokenized_docs)
          >>> doc_term_matrix
          <600x4346 sparse matrix of type '<class 'numpy.float64'>'
                  with 69673 stored elements in Compressed Sparse Row format>
  
      Tokenize and vectorize the remaining 400 documents of the corpus, using only
      the groups, terms, and weights learned in the previous step:
  
      .. code-block:: pycon
  
          >>> tokenized_docs = (
          ...     doc._.to_terms_list(ngrams=1, entities=True, as_strings=True)
          ...     for doc in corpus[600:])
          >>> doc_term_matrix = vectorizer.transform(tokenized_docs)
          >>> doc_term_matrix
          <400x4346 sparse matrix of type '<class 'numpy.float64'>'
                  with 38756 stored elements in Compressed Sparse Row format>
  
      Inspect the terms associated with columns; they're sorted alphabetically:
  
      .. code-block:: pycon
  
          >>> vectorizer.terms_list[:5]
          ['', '$', '$ 1 million', '$ 1.2 billion', '$ 10 billion']
  
      (Btw: That empty string shouldn't be there. Somehow, spaCy is labeling it as
      a named entity...)
  
      If known in advance, limit the terms included in vectorized outputs
      to a particular set of values:
  
      .. code-block:: pycon
  
          >>> tokenized_docs = (
          ...     doc._.to_terms_list(ngrams=1, entities=True, as_strings=True)
          ...     for doc in corpus[:600])
          >>> vectorizer = Vectorizer(
          ...     apply_idf=True, idf_type="smooth", norm="l2",
          ...     min_df=3, max_df=0.95,
          ...     vocabulary_terms=["president", "bill", "unanimous", "distinguished", "american"])
          >>> doc_term_matrix = vectorizer.fit_transform(tokenized_docs)
          >>> doc_term_matrix
          <600x5 sparse matrix of type '<class 'numpy.float64'>'
                  with 844 stored elements in Compressed Sparse Row format>
          >>> vectorizer.terms_list
          ['american', 'bill', 'distinguished', 'president', 'unanimous']
  
      Specify different weighting schemes to determine values in the matrix,
      adding or customizing individual components, as desired:
  
      .. code-block:: pycon
  
          >>> money_idx = vectorizer.vocabulary_terms["$"]
          >>> doc_term_matrix = Vectorizer(
          ...     tf_type="linear", norm=None, min_df=3, max_df=0.95
          ...     ).fit_transform(tokenized_docs)
          >>> print(doc_term_matrix[0:7, money_idx].toarray())
          [[0]
           [0]
           [1]
           [4]
           [0]
           [0]
           [2]]
          >>> doc_term_matrix = Vectorizer(
          ...     tf_type="sqrt", apply_dl=True, dl_type="sqrt", norm=None, min_df=3, max_df=0.95
          ...     ).fit_transform(tokenized_docs)
          >>> print(doc_term_matrix[0:7, money_idx].toarray())
          [[0.        ]
           [0.        ]
           [0.10101525]
           [0.26037782]
           [0.        ]
           [0.        ]
           [0.11396058]]
          >>> doc_term_matrix = Vectorizer(
          ...     tf_type="bm25", apply_idf=True, idf_type="smooth", norm=None, min_df=3, max_df=0.95
          ...     ).fit_transform(tokenized_docs)
          >>> print(doc_term_matrix[0:7, money_idx].toarray())
          [[0.        ]
           [0.        ]
           [3.28353965]
           [5.82763722]
           [0.        ]
           [0.        ]
           [4.83933924]]
  
      If you're not sure what's going on mathematically, :attr:`Vectorizer.weighting`
      gives the formula being used to calculate weights, based on the parameters
      set when initializing the vectorizer:
  
      .. code-block:: pycon
  
          >>> vectorizer.weighting
          '(tf * (k + 1)) / (k + tf) * log((n_docs + 1) / (df + 1)) + 1'
  
      In general, weights may consist of a local component (term frequency),
      a global component (inverse document frequency), and a normalization
      component (document length). Individual components may be modified:
      they may have different scaling (e.g. tf vs. sqrt(tf)) or different behaviors
      (e.g. "standard" idf vs bm25's version). There are *many* possible weightings,
      and some may be better for particular use cases than others. When in doubt,
      though, just go with something standard.
  
      - "tf": Weights are simply the absolute per-document term frequencies (tfs),
        i.e. value (i, j) in an output doc-term matrix corresponds to the number
        of occurrences of term j in doc i. Terms appearing many times in a given
        doc receive higher weights than less common terms.
        Params: ``tf_type="linear", apply_idf=False, apply_dl=False``
      - "tfidf": Doc-specific, *local* tfs are multiplied by their corpus-wide,
        *global* inverse document frequencies (idfs). Terms appearing in many docs
        have higher document frequencies (dfs), correspondingly smaller idfs, and
        in turn, lower weights.
        Params: ``tf_type="linear", apply_idf=True, idf_type="smooth", apply_dl=False``
      - "bm25": This scheme includes a local tf component that increases asymptotically,
        so higher tfs have diminishing effects on the overall weight; a global idf
        component that can go *negative* for terms that appear in a sufficiently
        high proportion of docs; as well as a row-wise normalization that accounts for
        document length, such that terms in shorter docs hit the tf asymptote sooner
        than those in longer docs.
        Params: ``tf_type="bm25", apply_idf=True, idf_type="bm25", apply_dl=True``
      - "binary": This weighting scheme simply replaces all non-zero tfs with 1,
        indicating the presence or absence of a term in a particular doc. That's it.
        Params: ``tf_type="binary", apply_idf=False, apply_dl=False``
  
      Slightly altered versions of these "standard" weighting schemes are common,
      and may have better behavior in general use cases:
  
      - "lucene-style tfidf": Adds a doc-length normalization to the usual local
        and global components.
        Params: ``tf_type="linear", apply_idf=True, idf_type="smooth", apply_dl=True, dl_type="sqrt"``
      - "lucene-style bm25": Uses a smoothed idf instead of the classic bm25 variant
        to prevent weights on terms from going negative.
        Params: ``tf_type="bm25", apply_idf=True, idf_type="smooth", apply_dl=True, dl_type="linear"``
  
      Args:
          tf_type ({"linear", "sqrt", "log", "binary"}): Type of term frequency (tf)
              to use for weights' local component:
  
              - "linear": tf (tfs are already linear, so left as-is)
              - "sqrt": tf => sqrt(tf)
              - "log": tf => log(tf) + 1
              - "binary": tf => 1
  
          apply_idf (bool): If True, apply global idfs to local term weights, i.e.
              divide per-doc term frequencies by the (log of the) total number
              of documents in which they appear; otherwise, don't.
          idf_type ({"standard", "smooth", "bm25"}): Type of inverse document
              frequency (idf) to use for weights' global component:
  
              - "standard": idf = log(n_docs / df) + 1.0
              - "smooth": idf = log(n_docs + 1 / df + 1) + 1.0, i.e. 1 is added
                to all document frequencies, as if a single document containing
                every unique term was added to the corpus. This prevents zero divisions!
              - "bm25": idf = log((n_docs - df + 0.5) / (df + 0.5)), which is
                a form commonly used in information retrieval that allows for
                very common terms to receive negative weights.
  
          apply_dl (bool): If True, normalize local(+global) weights by doc length,
              i.e. divide by the total number of in-vocabulary terms appearing
              in a given doc; otherwise, don't.
          dl_type ({"linear", "sqrt", "log"}): Type of document-length scaling
              to use for weights' normalization component:
  
              - "linear": dl (dls are already linear, so left as-is)
              - "sqrt": dl => sqrt(dl)
              - "log": dl => log(dl)
  
          norm ({"l1", "l2"} or None): If "l1" or "l2", normalize weights by the
              L1 or L2 norms, respectively, of row-wise vectors; otherwise, don't.
          vocabulary_terms (Dict[str, int] or Iterable[str]): Mapping of unique term
              string to unique term id, or an iterable of term strings that gets
              converted into a suitable mapping. Note that, if specified, vectorized
              outputs will include *only* these terms as columns.
          min_df (float or int): If float, value is the fractional proportion of
              the total number of documents, which must be in [0.0, 1.0]. If int,
              value is the absolute number. Filter terms whose document frequency
              is less than ``min_df``.
          max_df (float or int): If float, value is the fractional proportion of
              the total number of documents, which must be in [0.0, 1.0]. If int,
              value is the absolute number. Filter terms whose document frequency
              is greater than ``max_df``.
          max_n_terms (int): Only include terms whose document frequency is within
              the top ``max_n_terms``.
  
      Attributes:
          vocabulary_terms (Dict[str, int]): Mapping of unique term string to unique
              term id, either provided on instantiation or generated by calling
              :meth:`Vectorizer.fit()` on a collection of tokenized documents.
      """
  
      def __init__(
          self,
          *,
          tf_type="linear",
          apply_idf=False,
          idf_type="smooth",
          apply_dl=False,
          dl_type="sqrt",
          norm=None,
          min_df=1,
          max_df=1.0,
          max_n_terms=None,
          vocabulary_terms=None,
      ):
          # sanity check numeric arguments
          if min_df < 0 or max_df < 0:
              raise ValueError("`min_df` and `max_df` must be positive numbers or None")
          if max_n_terms and max_n_terms < 0:
              raise ValueError("`max_n_terms` must be a positive integer or None")
          self.tf_type = tf_type
          self.apply_idf = apply_idf
          self.idf_type = idf_type
          self.apply_dl = apply_dl
          self.dl_type = dl_type
          self.norm = norm
          self.min_df = min_df
          self.max_df = max_df
          self.max_n_terms = max_n_terms
          self.vocabulary_terms, self._fixed_terms = self._validate_vocabulary(
              vocabulary_terms
          )
          self.id_to_term_ = {}
          self._idf_diag = None
          self._avg_doc_length = None
  
      def _validate_vocabulary(self, vocabulary):
          """
          Validate an input vocabulary. If it's a mapping, ensure that term ids
          are unique and compact (i.e. without any gaps between 0 and the number
          of terms in ``vocabulary``. If it's a sequence, sort terms then assign
          integer ids in ascending order.
  
          Args:
              vocabulary_terms (Dict[str, int] or Iterable[str])
  
          Returns:
              Dict[str, int]
              bool
          """
          if vocabulary is not None:
              if not isinstance(vocabulary, collections.abc.Mapping):
                  vocab = {}
                  for i, term in enumerate(sorted(vocabulary)):
                      if vocab.setdefault(term, i) != i:
                          raise ValueError(
                              f"Terms in `vocabulary` must be unique, but '{term}' "
                              "was found more than once."
                          )
                  vocabulary = vocab
              else:
                  ids = set(vocabulary.values())
                  if len(ids) != len(vocabulary):
                      counts = collections.Counter(vocabulary.values())
                      n_dupe_term_ids = sum(
                          1
                          for term_id, term_id_count in counts.items()
                          if term_id_count > 1
                      )
                      raise ValueError(
                          "Term ids in `vocabulary` must be unique, but "
                          f"{n_dupe_term_ids} ids were assigned to more than one term."
                      )
                  for i in range(len(vocabulary)):
                      if i not in ids:
                          raise ValueError(
                              "Term ids in `vocabulary` must be compact, i.e. "
                              f"not have any gaps, but term id {i} is missing from "
                              f"a vocabulary of {len(vocabulary)} terms"
                          )
              if not vocabulary:
                  raise ValueError("`vocabulary` must not be empty.")
              is_fixed = True
          else:
              is_fixed = False
          return vocabulary, is_fixed
  
      def _check_vocabulary(self):
          """
          Check that instance has a valid vocabulary mapping;
          if not, raise a ValueError.
          """
          if not isinstance(self.vocabulary_terms, collections.abc.Mapping):
              raise ValueError("vocabulary hasn't been built; call `Vectorizer.fit()`")
          if len(self.vocabulary_terms) == 0:
              raise ValueError("vocabulary is empty")
  
      @property
      def id_to_term(self) -> Dict[int, str]:
          """
          Mapping of unique term id (int) to unique term string (str), i.e.
          the inverse of :attr:`Vectorizer.vocabulary`. This attribute is only
          generated if needed, and it is automatically kept in sync with the
          corresponding vocabulary.
          """
          if len(self.id_to_term_) != self.vocabulary_terms:
              self.id_to_term_ = {
                  term_id: term_str for term_str, term_id in self.vocabulary_terms.items()
              }
          return self.id_to_term_
  
      # TODO: Do we *want* to allow setting to this property?
      # @id_to_term.setter
      # def id_to_term(self, new_id_to_term):
      #     self.id_to_term_ = new_id_to_term
      #     self.vocabulary_terms = {
      #         term_str: term_id for term_id, term_str in new_id_to_term.items()}
  
      @property
      def terms_list(self) -> List[str]:
          """
          List of term strings in column order of vectorized outputs. For example,
          ``terms_list[0]`` gives the term assigned to the first column in an
          output doc-term-matrix, ``doc_term_matrix[:, 0]``.
          """
          self._check_vocabulary()
          return [
              term_str
              for term_str, _ in sorted(
                  self.vocabulary_terms.items(), key=operator.itemgetter(1)
              )
          ]
  
      def fit(self, tokenized_docs):
          """
          Count terms in ``tokenized_docs`` and, if not already provided, build up
          a vocabulary based those terms. Fit and store global weights (IDFs)
          and, if needed for term weighting, the average document length.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
          Returns:
              :class:`Vectorizer`: The instance that has just been fit.
          """
          _ = self._fit(tokenized_docs)
          return self
  
      def fit_transform(self, tokenized_docs):
          """
          Count terms in ``tokenized_docs`` and, if not already provided, build up
          a vocabulary based those terms. Fit and store global weights (IDFs)
          and, if needed for term weighting, the average document length.
          Transform ``tokenized_docs`` into a document-term matrix with values
          weighted according to the parameters in :class:`Vectorizer` initialization.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
          Returns:
              :class:`scipy.sparse.csr_matrix`: The transformed document-term matrix.
              Rows correspond to documents and columns correspond to terms.
          """
          # count terms and fit global weights
          doc_term_matrix = self._fit(tokenized_docs)
          # re-weight values in doc-term matrix, as specified in init
          doc_term_matrix = self._reweight_values(doc_term_matrix)
          return doc_term_matrix
  
      def transform(self, tokenized_docs):
          """
          Transform ``tokenized_docs`` into a document-term matrix with values
          weighted according to the parameters in :class:`Vectorizer` initialization
          and the global weights computed by calling :meth:`Vectorizer.fit()`.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
          Returns:
              :class:`scipy.sparse.csr_matrix`: The transformed document-term matrix.
              Rows correspond to documents and columns correspond to terms.
  
          Note:
              For best results, the tokenization used to produce ``tokenized_docs``
              should be the same as was applied to the docs used in fitting this
              vectorizer or in generating a fixed input vocabulary.
  
              Consider an extreme case where the docs used in fitting consist of
              lowercased (non-numeric) terms, while the docs to be transformed are
              all uppercased: The output doc-term-matrix will be empty.
          """
          self._check_vocabulary()
          doc_term_matrix, _ = self._count_terms(tokenized_docs, True)
          return self._reweight_values(doc_term_matrix)
  
      def _fit(self, tokenized_docs):
          """
          Count terms and, if :attr:`Vectorizer.fixed_terms` is False, build up
          a vocabulary based on the terms found in ``tokenized_docs``. Transform
          ``tokenized_docs`` into a document-term matrix with absolute tf weights.
          Store global weights (IDFs) and, if :attr:`Vectorizer.doc_length_norm`
          is not None, the average doc length.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]])
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
          """
          # count terms and, if not provided on init, build up a vocabulary
          doc_term_matrix, vocabulary_terms = self._count_terms(
              tokenized_docs, self._fixed_terms
          )
  
          if self._fixed_terms is False:
              # filter terms by doc freq or info content, as specified in init
              doc_term_matrix, vocabulary_terms = self._filter_terms(
                  doc_term_matrix, vocabulary_terms
              )
              # sort features alphabetically (vocabulary_terms modified in-place)
              doc_term_matrix = self._sort_vocab_and_matrix(
                  doc_term_matrix, vocabulary_terms, axis="columns"
              )
              # *now* vocabulary_terms are known and fixed
              self.vocabulary_terms = vocabulary_terms
              self._fixed_terms = True
  
          n_docs, n_terms = doc_term_matrix.shape
  
          if self.apply_idf is True:
              # store the global weights as a diagonal sparse matrix of idfs
              idfs = get_inverse_doc_freqs(doc_term_matrix, type_=self.idf_type)
              self._idf_diag = sp.spdiags(
                  idfs, diags=0, m=n_terms, n=n_terms, format="csr"
              )
  
          if self.tf_type == "bm25" and self.apply_dl is True:
              # store the avg document length, used in bm25 weighting to normalize
              # term weights by the length of the containing documents
              self._avg_doc_length = get_doc_lengths(
                  doc_term_matrix, type_=self.dl_type
              ).mean()
  
          return doc_term_matrix
  
      def _count_terms(self, tokenized_docs, fixed_vocab):
          """
          Count terms found in ``tokenized_docs`` and, if ``fixed_vocab`` is False,
          build up a vocabulary based on those terms.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]])
              fixed_vocab (bool)
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
              Dict[str, int]
          """
          if fixed_vocab is False:
              # add a new value when a new term is seen
              vocabulary = collections.defaultdict()
              vocabulary.default_factory = vocabulary.__len__
          else:
              vocabulary = self.vocabulary_terms
  
          indices = array(str("i"))
          indptr = array(str("i"), [0])
          for terms in tokenized_docs:
              for term in terms:
                  try:
                      indices.append(vocabulary[term])
                  except KeyError:
                      # ignore out-of-vocabulary terms when _fixed_terms=True
                      continue
              indptr.append(len(indices))
  
          if fixed_vocab is False:
              # we no longer want defaultdict behaviour
              vocabulary = dict(vocabulary)
  
          indices = np.frombuffer(indices, dtype=np.intc)
          indptr = np.frombuffer(indptr, dtype=np.intc)
          data = np.ones(len(indices))
  
          # build the matrix, then consolidate duplicate entries
          # by adding them together, in-place
          doc_term_matrix = sp.csr_matrix(
              (data, indices, indptr),
              shape=(len(indptr) - 1, len(vocabulary)),
              dtype=np.int32,
          )
          doc_term_matrix.sum_duplicates()
  
          # pretty sure this is a good thing to do... o_O
          doc_term_matrix.sort_indices()
  
          return doc_term_matrix, vocabulary
  
      def _filter_terms(self, doc_term_matrix, vocabulary):
          """
          Filter terms in ``vocabulary`` by their document frequency or information
          content, as specified in :class:`Vectorizer` initialization.
  
          Args:
              doc_term_matrix (:class:`sp.sparse.csr_matrix`)
              vocabulary (Dict[str, int])
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
              Dict[str, int]
          """
          if self.max_df != 1.0 or self.min_df != 1 or self.max_n_terms is not None:
              doc_term_matrix, vocabulary = filter_terms_by_df(
                  doc_term_matrix,
                  vocabulary,
                  max_df=self.max_df,
                  min_df=self.min_df,
                  max_n_terms=self.max_n_terms,
              )
          return doc_term_matrix, vocabulary
  
      def _sort_vocab_and_matrix(self, matrix, vocabulary, axis):
          """
          Sort terms in ``vocabulary`` alphabetically, modifying the vocabulary
          in-place, and returning a correspondingly reordered ``matrix`` along
          its rows or columns, depending on ``axis``.
  
          Args:
              matrix (:class:`sp.sparse.csr_matrix`)
              vocabulary (Dict[str, int])
              axis ({'rows', 'columns'} or {0, 1})
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
          """
          sorted_vocab = sorted(vocabulary.items())
          new_idx_array = np.empty(len(sorted_vocab), dtype=np.int32)
          for new_idx, (term, old_idx) in enumerate(sorted_vocab):
              new_idx_array[new_idx] = old_idx
              vocabulary[term] = new_idx
          # use fancy indexing to reorder rows or columns
          if axis == "rows" or axis == 0:
              return matrix[new_idx_array, :]
          elif axis == "columns" or axis == 1:
              return matrix[:, new_idx_array]
          else:
              raise ValueError(
                  errors.value_invalid_msg("axis", axis, {"rows", "columns", 0, 1})
              )
  
      def _reweight_values(self, doc_term_matrix):
          """
          Re-weight values in a doc-term matrix according to parameters specified
          in :class:`Vectorizer` initialization: binary or tf-idf weighting,
          sublinear term-frequency, document-normalized weights.
  
          Args:
              doc_term_matrix (:class:`sp.sparse.csr_matrix`)
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
          """
          # re-weight the local components (term freqs)
          if self.tf_type == "binary":
              doc_term_matrix.data.fill(1)
          elif self.tf_type == "bm25":
              if self.apply_dl is False:
                  doc_term_matrix.data = (
                      doc_term_matrix.data
                      * (BM25_K1 + 1.0)
                      / (BM25_K1 + doc_term_matrix.data)
                  )
              else:
                  dls = get_doc_lengths(doc_term_matrix, type_=self.dl_type)
                  length_norm = (1 - BM25_B) + (BM25_B * (dls / self._avg_doc_length))
                  doc_term_matrix = doc_term_matrix.tocoo(copy=False)
                  doc_term_matrix.data = (
                      doc_term_matrix.data
                      * (BM25_K1 + 1.0)
                      / (
                          doc_term_matrix.data
                          + (BM25_K1 * length_norm[doc_term_matrix.row])
                      )
                  )
                  doc_term_matrix = doc_term_matrix.tocsr(copy=False)
          elif self.tf_type == "sqrt":
              _ = np.sqrt(doc_term_matrix.data, doc_term_matrix.data, casting="unsafe")
          elif self.tf_type == "log":
              _ = np.log(doc_term_matrix.data, doc_term_matrix.data, casting="unsafe")
              doc_term_matrix.data += 1.0
          elif self.tf_type == "linear":
              pass  # tfs are already linear
          else:
              # this should never raise, i'm just being a worrywart
              raise ValueError(
                  errors.value_invalid_msg(
                      "tf_type", self.tf_type, {"binary", "bm25", "sqrt", "log", "linear"}
                  )
              )
  
          # apply the global component (idfs), column-wise
          if self.apply_idf is True:
              doc_term_matrix = doc_term_matrix * self._idf_diag
  
          # apply normalizations, row-wise
          # unless we've already handled it for bm25-style tf
          if self.apply_dl is True and self.tf_type != "bm25":
              n_docs, _ = doc_term_matrix.shape
              dls = get_doc_lengths(doc_term_matrix, type_=self.dl_type)
              dl_diag = sp.spdiags(1.0 / dls, diags=0, m=n_docs, n=n_docs, format="csr")
              doc_term_matrix = dl_diag * doc_term_matrix
          if self.norm is not None:
              doc_term_matrix = normalize_mat(
                  doc_term_matrix, norm=self.norm, axis=1, copy=False
              )
  
          return doc_term_matrix
  
      @property
      def weighting(self):
          """
          str: A mathematical representation of the overall weighting scheme
          used to determine values in the vectorized matrix, depending on the
          params used to initialize the :class:`Vectorizer`.
          """
          w = []
          tf_types = {
              "binary": "1",
              "linear": "tf",
              "sqrt": "sqrt(tf)",
              "log": "log(tf)",
              "bm25": {
                  True: "(tf * (k + 1)) / (tf + k * (1 - b + b * (length / avg(lengths)))",
                  False: "(tf * (k + 1)) / (tf + k)",
              },
          }
          idf_types = {
              "standard": "log(n_docs / df) + 1",
              "smooth": "log((n_docs + 1) / (df + 1)) + 1",
              "bm25": "log((n_docs - df + 0.5) / (df + 0.5))",
          }
          dl_types = {
              "linear": "1/length",
              "sqrt": "1/sqrt(length)",
              "log": "1/log(length) + 1",
          }
          if self.tf_type == "bm25":
              w.append(tf_types[self.tf_type][self.apply_dl])
          else:
              w.append(tf_types[self.tf_type])
          if self.apply_idf:
              w.append(idf_types[self.idf_type])
          if self.apply_dl and self.tf_type != "bm25":
              w.append(dl_types[self.dl_type])
          return " * ".join(w)
  
  
  class GroupVectorizer(Vectorizer):
      """
      Transform one or more tokenized documents into a group-term matrix of
      shape (# groups, # unique terms), with tf-, tf-idf, or binary-weighted values.
  
      This is an extension of typical document-term matrix vectorization, where
      terms are grouped by the documents in which they co-occur. It allows for
      customized grouping, such as by a shared author or publication year, that
      may span multiple documents, without forcing users to merge those documents
      themselves.
  
      Stream a corpus with metadata from disk::
  
          >>> ds = textacy.datasets.CapitolWords()
          >>> records = ds.records(limit=1000)
          >>> corpus = textacy.Corpus("en", data=records)
          >>> corpus
          Corpus(1000 docs; 538172 tokens)
  
      Tokenize and vectorize the first 600 documents of this corpus, where terms
      are grouped not by documents but by a categorical value in the docs' metadata::
  
          >>> tokenized_docs, groups = textacy.io.unzip(
          ...     (doc._.to_terms_list(ngrams=1, entities=True, as_strings=True),
          ...      doc._.meta["speaker_name"])
          ...     for doc in corpus[:600])
          >>> vectorizer = GroupVectorizer(
          ...     apply_idf=True, idf_type="smooth", norm="l2",
          ...     min_df=3, max_df=0.95)
          >>> grp_term_matrix = vectorizer.fit_transform(tokenized_docs, groups)
          >>> grp_term_matrix
          <5x1793 sparse matrix of type '<class 'numpy.float64'>'
                  with 6075 stored elements in Compressed Sparse Row format>
  
      Tokenize and vectorize the remaining 400 documents of the corpus, using only
      the groups, terms, and weights learned in the previous step::
  
          >>> tokenized_docs, groups = textacy.io.unzip(
          ...     (doc._.to_terms_list(ngrams=1, entities=True, as_strings=True),
          ...      doc._.meta["speaker_name"])
          ...     for doc in corpus[600:])
          >>> grp_term_matrix = vectorizer.transform(tokenized_docs, groups)
          >>> grp_term_matrix
          <5x1793 sparse matrix of type '<class 'numpy.float64'>'
                  with 4440 stored elements in Compressed Sparse Row format>
  
      Inspect the terms associated with columns and groups associated with rows;
      they're sorted alphabetically::
  
          >>> vectorizer.terms_list[:5]
          ['$ 1 million', '$ 160 million', '$ 7 billion', '0', '1 minute']
          >>> vectorizer.grps_list
          ['Bernie Sanders', 'John Kasich', 'Joseph Biden', 'Lindsey Graham', 'Rick Santorum']
  
      If known in advance, limit the terms and/or groups included in vectorized outputs
      to a particular set of values::
  
          >>> tokenized_docs, groups = textacy.io.unzip(
          ...     (doc._.to_terms_list(ngrams=1, entities=True, as_strings=True),
          ...      doc._.meta["speaker_name"])
          ...     for doc in corpus[:600])
          >>> vectorizer = GroupVectorizer(
          ...     apply_idf=True, idf_type="smooth", norm="l2",
          ...     min_df=3, max_df=0.95,
          ...     vocabulary_terms=["legislation", "federal government", "house", "constitutional"],
          ...     vocabulary_grps=["Bernie Sanders", "Lindsey Graham", "Rick Santorum"])
          >>> grp_term_matrix = vectorizer.fit_transform(tokenized_docs, groups)
          >>> grp_term_matrix
          <3x4 sparse matrix of type '<class 'numpy.float64'>'
                  with 12 stored elements in Compressed Sparse Row format>
          >>> vectorizer.terms_list
          ['constitutional', 'federal government', 'house', 'legislation']
          >>> vectorizer.grps_list
          ['Bernie Sanders', 'Lindsey Graham', 'Rick Santorum']
  
      For a discussion of the various weighting schemes that can be applied, check
      out the :class:`Vectorizer` docstring.
  
      Args:
          tf_type ({"linear", "sqrt", "log", "binary"}): Type of term frequency (tf)
              to use for weights' local component:
  
              - "linear": tf (tfs are already linear, so left as-is)
              - "sqrt": tf => sqrt(tf)
              - "log": tf => log(tf) + 1
              - "binary": tf => 1
  
          apply_idf (bool): If True, apply global idfs to local term weights, i.e.
              divide per-doc term frequencies by the total number of documents
              in which they appear (well, the log of that number); otherwise, don't.
          idf_type ({"standard", "smooth", "bm25"}): Type of inverse document
              frequency (idf) to use for weights' global component:
  
              - "standard": idf = log(n_docs / df) + 1.0
              - "smooth": idf = log(n_docs + 1 / df + 1) + 1.0, i.e. 1 is added
                to all document frequencies, as if a single document containing
                every unique term was added to the corpus.
              - "bm25": idf = log((n_docs - df + 0.5) / (df + 0.5)), which is
                a form commonly used in information retrieval that allows for
                very common terms to receive negative weights.
  
          apply_dl (bool): If True, normalize local(+global) weights by doc length,
              i.e. divide by the total number of in-vocabulary terms appearing
              in a given doc; otherwise, don't.
          dl_type ({"linear", "sqrt", "log"}): Type of document-length scaling
              to use for weights' normalization component:
  
              - "linear": dl (dls are already linear, so left as-is)
              - "sqrt": dl => sqrt(dl)
              - "log": dl => log(dl)
  
          norm ({"l1", "l2"} or None): If "l1" or "l2", normalize weights by the
              L1 or L2 norms, respectively, of row-wise vectors; otherwise, don't.
          vocabulary_terms (Dict[str, int] or Iterable[str]): Mapping of unique term
              string to unique term id, or an iterable of term strings that gets
              converted into a suitable mapping. Note that, if specified, vectorized
              outputs will include *only* these terms as columns.
          vocabulary_grps (Dict[str, int] or Iterable[str]): Mapping of unique group
              string to unique group id, or an iterable of group strings that gets
              converted into a suitable mapping. Note that, if specified, vectorized
              outputs will include *only* these groups as rows.
          min_df (float or int): If float, value is the fractional proportion of
              the total number of documents, which must be in [0.0, 1.0]. If int,
              value is the absolute number. Filter terms whose document frequency
              is less than ``min_df``.
          max_df (float or int): If float, value is the fractional proportion of
              the total number of documents, which must be in [0.0, 1.0]. If int,
              value is the absolute number. Filter terms whose document frequency
              is greater than ``max_df``.
          max_n_terms (int): Only include terms whose document frequency is within
              the top ``max_n_terms``.
  
      Attributes:
          vocabulary_terms (Dict[str, int]): Mapping of unique term string to unique
              term id, either provided on instantiation or generated by calling
              :meth:`GroupVectorizer.fit()` on a collection of tokenized documents.
          vocabulary_grps (Dict[str, int]): Mapping of unique group string to unique
              group id, either provided on instantiation or generated by calling
              :meth:`GroupVectorizer.fit()` on a collection of tokenized documents.
          id_to_term (Dict[int, str]): Mapping of unique term id to unique term
              string, i.e. the inverse of :attr:`GroupVectorizer.vocabulary_terms`.
              This mapping is only generated as needed.
          terms_list (List[str]): List of term strings in column order of
              vectorized outputs.
  
      See Also:
          :class:`Vectorizer`
      """
  
      def __init__(
          self,
          *,
          tf_type="linear",
          apply_idf=False,
          idf_type="smooth",
          apply_dl=False,
          dl_type="linear",
          norm=None,
          min_df=1,
          max_df=1.0,
          max_n_terms=None,
          vocabulary_terms=None,
          vocabulary_grps=None,
      ):
          super().__init__(
              tf_type=tf_type,
              apply_idf=apply_idf,
              idf_type=idf_type,
              apply_dl=apply_dl,
              dl_type=dl_type,
              norm=norm,
              min_df=min_df,
              max_df=max_df,
              max_n_terms=max_n_terms,
              vocabulary_terms=vocabulary_terms,
          )
          # now do the same thing for grps as was done for terms
          self.vocabulary_grps, self._fixed_grps = self._validate_vocabulary(
              vocabulary_grps
          )
          self.id_to_grp_ = {}
  
      @property
      def id_to_grp(self) -> Dict[int, str]:
          """
          Mapping of unique group id (int) to unique group string (str), i.e.
          the inverse of :attr:`GroupVectorizer.vocabulary_grps`. This attribute
          is only generated if needed, and it is automatically kept in sync
          with the corresponding vocabulary.
          """
          if len(self.id_to_grp_) != self.vocabulary_grps:
              self.id_to_grp_ = {
                  grp_id: grp_str for grp_str, grp_id in self.vocabulary_grps.items()
              }
          return self.id_to_grp_
  
      # @id_to_grp.setter
      # def id_to_grp(self, new_id_to_grp):
      #     self.id_to_grp_ = new_id_to_grp
      #     self.vocabulary_grps = {
      #         grp_str: grp_id for grp_id, grp_str in new_id_to_grp.items()}
  
      @property
      def grps_list(self) -> List[str]:
          """
          List of group strings in row order of vectorized outputs. For example,
          ``grps_list[0]`` gives the group assigned to the first row in an
          output group-term-matrix, ``grp_term_matrix[0, :]``.
          """
          self._check_vocabulary()
          return [
              grp_str
              for grp_str, _ in sorted(
                  self.vocabulary_grps.items(), key=operator.itemgetter(1)
              )
          ]
  
      def fit(self, tokenized_docs, grps):
          """
          Count terms in ``tokenized_docs`` and, if not already provided, build up
          a vocabulary based those terms; do the same for the groups in ``grps``.
          Fit and store global weights (IDFs) and, if needed for term weighting,
          the average document length.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
              grps (Iterable[str]): Sequence of group names by which the terms in
                  ``tokenized_docs`` are aggregated, where the first item in ``grps``
                  corresponds to the first item in ``tokenized_docs``, and so on.
  
          Returns:
              :class:`GroupVectorizer`: The instance that has just been fit.
          """
          _ = self._fit(tokenized_docs, grps)
          return self
  
      def fit_transform(self, tokenized_docs, grps):
          """
          Count terms in ``tokenized_docs`` and, if not already provided, build up
          a vocabulary based those terms; do the same for the groups in ``grps``.
          Fit and store global weights (IDFs) and, if needed for term weighting,
          the average document length. Transform ``tokenized_docs`` into a
          group-term matrix with values weighted according to the parameters in
          :class:`GroupVectorizer` initialization.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
              grps (Iterable[str]): Sequence of group names by which the terms in
                  ``tokenized_docs`` are aggregated, where the first item in ``grps``
                  corresponds to the first item in ``tokenized_docs``, and so on.
  
          Returns:
              :class:`scipy.sparse.csr_matrix`: The transformed group-term matrix.
              Rows correspond to groups and columns correspond to terms.
          """
          # count terms and fit global weights
          grp_term_matrix = self._fit(tokenized_docs, grps)
          # re-weight values in group-term matrix, as specified in init
          grp_term_matrix = self._reweight_values(grp_term_matrix)
          return grp_term_matrix
  
      def transform(self, tokenized_docs, grps):
          """
          Transform ``tokenized_docs`` and ``grps`` into a group-term matrix with
          values weighted according to the parameters in :class:`GroupVectorizer`
          initialization and the global weights computed by calling
          :meth:`GroupVectorizer.fit()`.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]]): A sequence of tokenized
                  documents, where each is a sequence of (str) terms. For example::
  
                      >>> ([tok.lemma_ for tok in spacy_doc]
                      ...  for spacy_doc in spacy_docs)
                      >>> ((ne.text for ne in extract.entities(doc))
                      ...  for doc in corpus)
                      >>> (doc._.to_terms_list(as_strings=True)
                      ...  for doc in docs)
  
              grps (Iterable[str]): Sequence of group names by which the terms in
                  ``tokenized_docs`` are aggregated, where the first item in ``grps``
                  corresponds to the first item in ``tokenized_docs``, and so on.
  
          Returns:
              :class:`scipy.sparse.csr_matrix`: The transformed group-term matrix.
              Rows correspond to groups and columns correspond to terms.
  
          Note:
              For best results, the tokenization used to produce ``tokenized_docs``
              should be the same as was applied to the docs used in fitting this
              vectorizer or in generating a fixed input vocabulary.
  
              Consider an extreme case where the docs used in fitting consist of
              lowercased (non-numeric) terms, while the docs to be transformed are
              all uppercased: The output group-term-matrix will be empty.
          """
          self._check_vocabulary()
          grp_term_matrix, _, _ = self._count_terms(tokenized_docs, grps, True, True)
          return self._reweight_values(grp_term_matrix)
  
      def _fit(self, tokenized_docs, grps):
          """
          Count terms and, if :attr:`Vectorizer.fixed_terms` is False, build up
          a vocabulary based on the terms found in ``tokenized_docs``. Transform
          ``tokenized_docs`` into a document-term matrix with absolute tf weights.
          Store global weights (IDFs) and, if :attr:`Vectorizer.doc_length_norm`
          is not None, the average doc length.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]])
              grps (Iterable[str])
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
          """
          # count terms and, if not provided on init, build up a vocabulary
          grp_term_matrix, vocabulary_terms, vocabulary_grps = self._count_terms(
              tokenized_docs, grps, self._fixed_terms, self._fixed_grps
          )
  
          if self._fixed_terms is False:
              # filter terms by doc freq or info content, as specified in init
              grp_term_matrix, vocabulary_terms = self._filter_terms(
                  grp_term_matrix, vocabulary_terms
              )
              # sort features alphabetically (vocabulary_terms modified in-place)
              grp_term_matrix = self._sort_vocab_and_matrix(
                  grp_term_matrix, vocabulary_terms, axis="columns"
              )
              # *now* vocabulary_terms are known and fixed
              self.vocabulary_terms = vocabulary_terms
              self._fixed_terms = True
          if self._fixed_grps is False:
              # sort groups alphabetically (vocabulary_grps modified in-place)
              grp_term_matrix = self._sort_vocab_and_matrix(
                  grp_term_matrix, vocabulary_grps, axis="rows"
              )
              # *now* vocabulary_grps are known and fixed
              self.vocabulary_grps = vocabulary_grps
              self._fixed_grps = True
  
          n_grps, n_terms = grp_term_matrix.shape
  
          if self.apply_idf is True:
              # store the global weights as a diagonal sparse matrix of idfs
              idfs = get_inverse_doc_freqs(grp_term_matrix, type_=self.idf_type)
              self._idf_diag = sp.spdiags(
                  idfs, diags=0, m=n_terms, n=n_terms, format="csr"
              )
  
          if self.tf_type == "bm25" and self.apply_dl is True:
              # store the avg document length, used in bm25 weighting to normalize
              # term weights by the length of the containing documents
              self._avg_doc_length = get_doc_lengths(
                  grp_term_matrix, type_=self.dl_type
              ).mean()
  
          return grp_term_matrix
  
      def _count_terms(self, tokenized_docs, grps, fixed_vocab_terms, fixed_vocab_grps):
          """
          Count terms and build up a vocabulary based on the terms found in the
          ``tokenized_docs`` and the groups found in ``grps``.
  
          Args:
              tokenized_docs (Iterable[Iterable[str]])
              grps (Iterable[str])
              fixed_vocab_terms (bool)
              fixed_vocab_grps (bool)
  
          Returns:
              :class:`scipy.sparse.csr_matrix`
              Dict[str, int]
              Dict[str, int]
          """
          # TODO: can we adapt the optimization from `Vectorizer._count_terms()` here?
          if fixed_vocab_terms is False:
              # add a new value when a new term is seen
              vocabulary_terms = collections.defaultdict()
              vocabulary_terms.default_factory = vocabulary_terms.__len__
          else:
              vocabulary_terms = self.vocabulary_terms
  
          if fixed_vocab_grps is False:
              # add a new value when a new group is seen
              vocabulary_grps = collections.defaultdict()
              vocabulary_grps.default_factory = vocabulary_grps.__len__
          else:
              vocabulary_grps = self.vocabulary_grps
  
          data = array(str("i"))
          cols = array(str("i"))
          rows = array(str("i"))
          for grp, terms in zip(grps, tokenized_docs):
  
              try:
                  grp_idx = vocabulary_grps[grp]
              except KeyError:
                  # ignore out-of-vocabulary groups when fixed_grps=True
                  continue
  
              term_counter = collections.defaultdict(int)
              for term in terms:
                  try:
                      term_idx = vocabulary_terms[term]
                      term_counter[term_idx] += 1
                  except KeyError:
                      # ignore out-of-vocabulary terms when fixed_terms=True
                      continue
  
              data.extend(term_counter.values())
              cols.extend(term_counter.keys())
              rows.extend(grp_idx for _ in range(len(term_counter)))
  
          # do we still want defaultdict behaviour?
          if fixed_vocab_terms is False:
              vocabulary_terms = dict(vocabulary_terms)
          if fixed_vocab_grps is False:
              vocabulary_grps = dict(vocabulary_grps)
  
          data = np.frombuffer(data, dtype=np.intc)
          rows = np.frombuffer(rows, dtype=np.intc)
          cols = np.frombuffer(cols, dtype=np.intc)
  
          grp_term_matrix = sp.csr_matrix(
              (data, (rows, cols)),
              shape=(len(vocabulary_grps), len(vocabulary_terms)),
              dtype=np.int32,
          )
          grp_term_matrix.sort_indices()
  
          return grp_term_matrix, vocabulary_terms, vocabulary_grps
  
apis: scipy.sparse.spdiags numpy.log sklearn.preprocessing.normalize numpy.sqrt numpy.frombuffer