code: def whole_corpus_productivity_scores(
      tdm: TermDocMatrixWithoutCategories,
  ) -> pd.DataFrame:
      term_freqs = pd.Series(
          tdm.get_term_freqs(), index=tdm.get_terms()
      )
      ngrams_mask = [" " in x for x in tdm.get_terms()]
  
      data = []
      for ngram, ngram_freq in term_freqs[ngrams_mask].items():
          for term in ngram.split():
              try:
                  tdm.get_term_index(term)
              except KeyError:
                  continue
              data.append([tdm.get_term_index(term), ngram_freq])
  
      ngram_freq_df = pd.DataFrame(
          data, columns=["TermIndex", "NgramFreq"]
      )
  
      productivity_df = (
          pd.merge(
              ngram_freq_df,
              ngram_freq_df.groupby(["TermIndex"])
              .sum()
              .rename(columns={"NgramFreq": "TermCatFreq"})
              .reset_index(),
              on=["TermIndex"],
          )
          .assign(P=lambda df: df.NgramFreq / df.TermCatFreq)
          .dropna()
          .groupby(["TermIndex"])
          .apply(lambda df: -np.sum(df.P * np.log(df.P) / np.log(2)))
          .reset_index()
          .rename(columns={0: "Productivity"})
          .assign(
              Term=lambda df: np.array(tdm.get_terms())[df.TermIndex],
              Frequency=lambda df: term_freqs.loc[
                  df.Term.values
              ].values,
          )[lambda df: [c for c in df.columns if c != "TermIndex"]]
          .set_index("Term")
      )
      return productivity_df
  
file_path: scattertext/scattertext/termscoring/Productivity.py
imports: import pandas
  import numpy
  from scattertext.TermDocMatrixWithoutCategories import TermDocMatrixWithoutCategories
  from scattertext.termscoring.CorpusBasedTermScorer import CorpusBasedTermScorer
label_desc: Use image node_id 1 for calling a global function with example usage: whole_corpus_productivity_scores(tdm) and returns: productivity_df