code: def texts_from_folder(
      datadir,
      classes=None,
      max_features=MAX_FEATURES,
      maxlen=MAXLEN,
      ngram_range=1,
      train_test_names=["train", "test"],
      preprocess_mode="standard",
      encoding=None,  # detected automatically
      lang=None,  # detected automatically
      val_pct=0.1,
      random_state=None,
      verbose=1,
  ):
      """
      ```
      Returns corpus as sequence of word IDs.
      Assumes corpus is in the following folder structure:
      ├── datadir
      │   ├── train
      │   │   ├── class0       # folder containing documents of class 0
      │   │   ├── class1       # folder containing documents of class 1
      │   │   ├── class2       # folder containing documents of class 2
      │   │   └── classN       # folder containing documents of class N
      │   └── test
      │       ├── class0       # folder containing documents of class 0
      │       ├── class1       # folder containing documents of class 1
      │       ├── class2       # folder containing documents of class 2
      │       └── classN       # folder containing documents of class N
  
      Each subfolder should contain documents in plain text format.
      If train and test contain additional subfolders that do not represent
      classes, they can be ignored by explicitly listing the subfolders of
      interest using the classes argument.
      Args:
          datadir (str): path to folder
          classes (list): list of classes (subfolders to consider).
                          This is simply supplied as the categories argument
                          to sklearn's load_files function.
          max_features (int):  maximum number of unigrams to consider
                               Note: This is only used for preprocess_mode='standard'.
          maxlen (int):  maximum length of tokens in document
          ngram_range (int):  If > 1, will include 2=bigrams, 3=trigrams and bigrams
          train_test_names (list):  list of strings represnting the subfolder
                                   name for train and validation sets
                                   if test name is missing, <val_pct> of training
                                   will be used for validation
          preprocess_mode (str):  Either 'standard' (normal tokenization) or one of {'bert', 'distilbert'}
                                  tokenization and preprocessing for use with
                                  BERT/DistilBert text classification model.
          encoding (str):        character encoding to use. Auto-detected if None
          lang (str):            language.  Auto-detected if None.
          val_pct(float):        Onlyl used if train_test_names  has 1 and not 2 names
          random_state(int):      If integer is supplied, train/test split is reproducible.
                                  IF None, train/test split will be random
          verbose (bool):         verbosity
  
      ```
      """
  
      # check train_test_names
      if len(train_test_names) < 1 or len(train_test_names) > 2:
          raise ValueError(
              "train_test_names must have 1 or two elements for train and optionally validation"
          )
  
      # read in training and test corpora
      train_str = train_test_names[0]
      train_b = load_files(
          os.path.join(datadir, train_str),
          shuffle=True,
          categories=classes,
      )
      if len(train_test_names) > 1:
          test_str = train_test_names[1]
          test_b = load_files(
              os.path.join(datadir, test_str),
              shuffle=False,
              categories=classes,
          )
          x_train = train_b.data
          y_train = train_b.target
          x_test = test_b.data
          y_test = test_b.target
      else:
          x_train, x_test, y_train, y_test = train_test_split(
              train_b.data,
              train_b.target,
              test_size=val_pct,
              random_state=random_state,
          )
  
      # decode based on supplied encoding
      if encoding is None:
          encoding = TU.detect_encoding(x_train)
          U.vprint(
              "detected encoding: %s" % (encoding), verbose=verbose
          )
  
      try:
          x_train = [x.decode(encoding) for x in x_train]
          x_test = [x.decode(encoding) for x in x_test]
      except:
          U.vprint(
              "Decoding with %s failed 1st attempt - using %s with skips"
              % (encoding, encoding),
              verbose=verbose,
          )
          x_train = TU.decode_by_line(
              x_train, encoding=encoding, verbose=verbose
          )
          x_test = TU.decode_by_line(
              x_test, encoding=encoding, verbose=verbose
          )
  
      # detect language
      if lang is None:
          lang = TU.detect_lang(x_train)
      check_unsupported_lang(lang, preprocess_mode)
  
      # return preprocessed the texts
      preproc_type = tpp.TEXT_PREPROCESSORS.get(preprocess_mode, None)
      if None:
          raise ValueError("unsupported preprocess_mode")
      preproc = preproc_type(
          maxlen,
          max_features,
          class_names=train_b.target_names,
          lang=lang,
          ngram_range=ngram_range,
      )
      trn = preproc.preprocess_train(x_train, y_train, verbose=verbose)
      val = preproc.preprocess_test(x_test, y_test, verbose=verbose)
      return (trn, val, preproc)
  
file_path: ktrain/ktrain/text/data.py
imports: from ..None import utils
  from ..imports import *
  from .None import preprocessor
  from .None import textutils
label_desc: Use image node_id 1 for calling a global function with example usage: texts_from_folder(datadir, classes, max_features, maxlen, ngram_range, train_test_names, preprocess_mode, encoding, lang, val_pct, random_state, verbose) and returns: trn,  val,  preproc