class_docstr: Generates ngram features from text features.
  
  Parameters
  ----------
  vectorizer : :class:`sklearn.feature_extraction.text.CountVectorizer` or :class:`sklearn.feature_extraction.text.TfidfVectorizer`, default CountVectorizer(min_df=30, ngram_range=(1, 3), max_features=10000, dtype=np.uint8)  # noqa
      sklearn CountVectorizer which is used to generate the ngrams given the text data.
      Can also specify a TfidfVectorizer, but note that memory usage will increase by 4-8x relative to CountVectorizer.
  vectorizer_strategy : str, default 'combined'
      If 'combined', all text features are concatenated together to fit the vectorizer.
      Features generated in this way have their names prepended with '__nlp__.'.
      If 'separate', all text features are fit separately with their own copy of the vectorizer.
      Their ngram features are then concatenated together to form the output.
      If 'both', the outputs of 'combined' and 'separate' are concatenated together to form the output.
      It is generally recommended to keep vectorizer_strategy as 'combined' unless the text features are not associated with each-other,
      as fitting separate vectorizers could increase memory usage and model training time.
      Valid values: ['combined', 'separate', 'both']
  max_memory_ratio : float, default 0.15
      Safety measure to avoid out-of-memory errors downstream in model training.
      The number of ngrams generated will be capped to take at most max_memory_ratio proportion of total available memory,
      treating the ngrams as float32 values.
      ngram features will be removed in least frequent to most frequent order.
      Note: For vectorizer_strategy values other than 'combined', the resulting ngrams may use more than this value.
      It is recommended to only increase this value above 0.15 if confident that higher values will not result in out-of-memory errors.
  **kwargs :
      Refer to :class:`AbstractFeatureGenerator` documentation for details on valid key word arguments.
class_docstr_tok: ["Generates","ngram","features","from","text","features",".","Parameters","--","--","--","--","--","vectorizer",":",":","class",":","`","sklearn.feature_extraction.text.CountVectorizer","`","or",":","class",":","`","sklearn.feature_extraction.text.TfidfVectorizer","`",",","default","CountVectorizer","(","min_df=30",",","ngram_range=","(","1",",","3",")",",","max_features=10000",",","dtype=np.uint8",")","#","noqa","sklearn","CountVectorizer","which","is","used","to","generate","the","ngrams","given","the","text","data",".","Can","also","specify","a","TfidfVectorizer",",","but","note","that","memory","usage","will","increase","by","4-8x","relative","to","CountVectorizer",".","vectorizer_strategy",":","str",",","default","'combined'","If","'combined","'",",","all","text","features","are","concatenated","together","to","fit","the","vectorizer",".","Features","generated","in","this","way","have","their","names","prepended","with","'__nlp__.","'",".","If","'separate","'",",","all","text","features","are","fit","separately","with","their","own","copy","of","the","vectorizer",".","Their","ngram","features","are","then","concatenated","together","to","form","the","output",".","If","'both","'",",","the","outputs","of","'combined","'","and","'separate","'","are","concatenated","together","to","form","the","output",".","It","is","generally","recommended","to","keep","vectorizer_strategy","as","'combined","'","unless","the","text","features","are","not","associated","with","each-other",",","as","fitting","separate","vectorizers","could","increase","memory","usage","and","model","training","time",".","Valid","values",":","[","'combined","'",",","'separate","'",",","'both","'","]","max_memory_ratio",":","float",",","default","0.15","Safety","measure","to","avoid","out-of-memory","errors","downstream","in","model","training",".","The","number","of","ngrams","generated","will","be","capped","to","take","at","most","max_memory_ratio","proportion","of","total","available","memory",",","treating","the","ngrams","as","float32","values",".","ngram","features","will","be","removed","in","least","frequent","to","most","frequent","order",".","Note",":","For","vectorizer_strategy","values","other","than","'combined","'",",","the","resulting","ngrams","may","use","more","than","this","value",".","It","is","recommended","to","only","increase","this","value","above","0.15","if","confident","that","higher","values","will","not","result","in","out-of-memory","errors",".","*","*","kwargs",":","Refer","to",":","class",":","`","AbstractFeatureGenerator","`","documentation","for","details","on","valid","key","word","arguments","."]
code: def _adjust_vectorizer_memory_usage(
      self,
      transform_matrix,
      text_data,
      vectorizer_fit,
      downsample_ratio: int = None,
  ):
      @disable_if_lite_mode(ret=downsample_ratio)
      def _adjust_per_memory_constraints(downsample_ratio: int):
          import psutil
  
          # This assumes that the ngrams eventually turn into int32/float32 downstream
          predicted_ngrams_memory_usage_bytes = (
              len(text_data) * 4 * (transform_matrix.shape[1] + 1) + 80
          )
          mem_avail = psutil.virtual_memory().available
          mem_rss = psutil.Process().memory_info().rss
          predicted_rss = mem_rss + predicted_ngrams_memory_usage_bytes
          predicted_percentage = predicted_rss / mem_avail
          if downsample_ratio is None:
              if (
                  self.max_memory_ratio is not None
                  and predicted_percentage > self.max_memory_ratio
              ):
                  self._log(
                      30,
                      "Warning: Due to memory constraints, ngram feature count is being reduced. Allocate more memory to maximize model quality.",
                  )
                  return self.max_memory_ratio / predicted_percentage
  
      downsample_ratio = _adjust_per_memory_constraints(
          downsample_ratio
      )
  
      if downsample_ratio is not None:
          if (downsample_ratio >= 1) or (downsample_ratio <= 0):
              raise ValueError(
                  f"downsample_ratio must be >0 and <1, but downsample_ratio is {downsample_ratio}"
              )
          vocab_size = len(vectorizer_fit.vocabulary_)
          downsampled_vocab_size = int(
              np.floor(vocab_size * downsample_ratio)
          )
          self._log(
              20,
              f"Reducing Vectorizer vocab size from {vocab_size} to {downsampled_vocab_size} to avoid OOM error",
          )
          ngram_freq = get_ngram_freq(
              vectorizer=vectorizer_fit,
              transform_matrix=transform_matrix,
          )
          downscale_vectorizer(
              vectorizer=vectorizer_fit,
              ngram_freq=ngram_freq,
              vocab_size=downsampled_vocab_size,
          )
          # TODO: This doesn't have to be done twice, can update transform matrix based on new vocab instead of calling .transform
          #  If we have this functionality, simply update transform_matrix each time OOM occurs instead of re-calling .transform
          transform_matrix = vectorizer_fit.transform(text_data)
  
      return transform_matrix
  
imports: import copy
  import logging
  import traceback
  import numpy
  import pandas
  from pandas import DataFrame, Series
  from sklearn.feature_selection import SelectKBest, f_classif, f_regression
  from autogluon.common.features.types import S_IMAGE_BYTEARRAY, S_IMAGE_PATH, S_TEXT, S_TEXT_NGRAM
  from autogluon.common.utils.lite import disable_if_lite_mode
  from ..vectorizers import downscale_vectorizer, get_ngram_freq, vectorizer_auto_ml_default
  from .abstract import AbstractFeatureGenerator
label_desc: Use image node_id 7 for calling the TextNgramFeatureGenerator obj's underlying member method code with example usage: obj._adjust_vectorizer_memory_usage(transform_matrix, text_data, vectorizer_fit, downsample_ratio) and returns: transform_matrix, unknown