class_docstr: SimpleQA: Question-Answering on a list of texts
class_docstr_tok: ["SimpleQA",":","Question-Answering","on","a","list","of","texts"]
docstr: ```
  index all plain text documents within a folder.
  The procs, limitmb, and especially multisegment arguments can be used to
  speed up indexing, if it is too slow.  Please see the whoosh documentation
  for more information on these parameters:  https://whoosh.readthedocs.io/en/latest/batch.html
  
  Args:
    folder_path(str): path to folder containing plain text documents (e.g., .txt files)
    index_dir(str): path to index directory (see initialize_index)
    use_text_extraction(bool): If True, the  `textract` package will be used to index text from various
                               file types including PDF, MS Word, and MS PowerPoint (in addition to plain text files).
                               If False, only plain text files will be indexed.
    commit_every(int): commet after adding this many documents
    breakup_docs(bool): break up documents into smaller paragraphs and treat those as the documents.
                        This can potentially improve the speed at which answers are returned by the ask method
                        when documents being searched are longer.
    min_words(int):  minimum words for a document (or paragraph extracted from document when breakup_docs=True) to be included in index.
                     Useful for pruning contexts that are unlikely to contain useful answers
    encoding(str): encoding to use when reading document files from disk
    procs(int): number of processors
    limitmb(int): memory limit in MB for each process
    multisegment(bool): new segments written instead of merging
    verbose(bool): verbosity
  ```
docstr_tok: ["``","`","index","all","plain","text","documents","within","a","folder",".","The","procs",",","limitmb",",","and","especially","multisegment","arguments","can","be","used","to","speed","up","indexing",",","if","it","is","too","slow",".","Please","see","the","whoosh","documentation","for","more","information","on","these","parameters",":","https",":","\/\/whoosh.readthedocs.io\/en\/latest\/batch.html","Args",":","folder_path","(","str",")",":","path","to","folder","containing","plain","text","documents","(","e.g.",",",".txt","files",")","index_dir","(","str",")",":","path","to","index","directory","(","see","initialize_index",")","use_text_extraction","(","bool",")",":","If","True",",","the","`","textract","`","package","will","be","used","to","index","text","from","various","file","types","including","PDF",",","MS","Word",",","and","MS","PowerPoint","(","in","addition","to","plain","text","files",")",".","If","False",",","only","plain","text","files","will","be","indexed",".","commit_every","(","int",")",":","commet","after","adding","this","many","documents","breakup_docs","(","bool",")",":","break","up","documents","into","smaller","paragraphs","and","treat","those","as","the","documents",".","This","can","potentially","improve","the","speed","at","which","answers","are","returned","by","the","ask","method","when","documents","being","searched","are","longer",".","min_words","(","int",")",":","minimum","words","for","a","document","(","or","paragraph","extracted","from","document","when","breakup_docs=True",")","to","be","included","in","index",".","Useful","for","pruning","contexts","that","are","unlikely","to","contain","useful","answers","encoding","(","str",")",":","encoding","to","use","when","reading","document","files","from","disk","procs","(","int",")",":","number","of","processors","limitmb","(","int",")",":","memory","limit","in","MB","for","each","process","multisegment","(","bool",")",":","new","segments","written","instead","of","merging","verbose","(","bool",")",":","verbosity","``","`"]
code: def index_from_folder(
      cls,
      folder_path,
      index_dir,
      use_text_extraction=False,
      commit_every=1024,
      breakup_docs=True,
      min_words=20,
      encoding="utf-8",
      procs=1,
      limitmb=256,
      multisegment=False,
      verbose=1,
  ):
      """
      ```
      index all plain text documents within a folder.
      The procs, limitmb, and especially multisegment arguments can be used to
      speed up indexing, if it is too slow.  Please see the whoosh documentation
      for more information on these parameters:  https://whoosh.readthedocs.io/en/latest/batch.html
  
      Args:
        folder_path(str): path to folder containing plain text documents (e.g., .txt files)
        index_dir(str): path to index directory (see initialize_index)
        use_text_extraction(bool): If True, the  `textract` package will be used to index text from various
                                   file types including PDF, MS Word, and MS PowerPoint (in addition to plain text files).
                                   If False, only plain text files will be indexed.
        commit_every(int): commet after adding this many documents
        breakup_docs(bool): break up documents into smaller paragraphs and treat those as the documents.
                            This can potentially improve the speed at which answers are returned by the ask method
                            when documents being searched are longer.
        min_words(int):  minimum words for a document (or paragraph extracted from document when breakup_docs=True) to be included in index.
                         Useful for pruning contexts that are unlikely to contain useful answers
        encoding(str): encoding to use when reading document files from disk
        procs(int): number of processors
        limitmb(int): memory limit in MB for each process
        multisegment(bool): new segments written instead of merging
        verbose(bool): verbosity
      ```
      """
      if use_text_extraction:
          # TODO:  change this to use TextExtractor
          try:
              import textract
          except ImportError:
              raise Exception(
                  "use_text_extraction=True requires textract:   pip install textract"
              )
  
      if not os.path.isdir(folder_path):
          raise ValueError("folder_path is not a valid folder")
      if folder_path[-1] != os.sep:
          folder_path += os.sep
      ix = index.open_dir(index_dir)
      writer = ix.writer(
          procs=procs, limitmb=limitmb, multisegment=multisegment
      )
      for idx, fpath in enumerate(TU.extract_filenames(folder_path)):
          reference = "%s" % (fpath.join(fpath.split(folder_path)[1:]))
          if TU.is_txt(fpath):
              with open(fpath, "r", encoding=encoding) as f:
                  doc = f.read()
          else:
              if use_text_extraction:
                  try:
                      doc = textract.process(fpath)
                      doc = doc.decode("utf-8", "ignore")
                  except:
                      if verbose:
                          warnings.warn(
                              "Could not extract text from %s" % (fpath)
                          )
                      continue
              else:
                  continue
  
          if breakup_docs:
              small_docs = TU.paragraph_tokenize(
                  doc, join_sentences=True, lang="en"
              )
              refs = [reference] * len(small_docs)
              for i, small_doc in enumerate(small_docs):
                  if len(small_doc.split()) < min_words:
                      continue
                  content = small_doc
                  reference = refs[i]
                  writer.add_document(
                      reference=reference,
                      content=content,
                      rawtext=content,
                  )
          else:
              if len(doc.split()) < min_words:
                  continue
              content = doc
              writer.add_document(
                  reference=reference, content=content, rawtext=content
              )
  
          idx += 1
          if idx % commit_every == 0:
              writer.commit()
              writer = ix.writer(
                  procs=procs,
                  limitmb=limitmb,
                  multisegment=multisegment,
              )
              if verbose:
                  print("%s docs indexed" % (idx))
      writer.commit()
      return
  
imports: from transformers import AutoModelForQuestionAnswering, AutoTokenizer, TFAutoModelForQuestionAnswering
  from whoosh import index, qparser
  from whoosh.fields import *
  from whoosh.qparser import QueryParser
  from ...None import utils
  from ...imports import *
  from ...torch_base import TorchBase
  from ..None import preprocessor
  from ..None import textutils
  from itertools import chain, zip_longest
label_desc: Use image node_id 5 for calling the SimpleQA obj's underlying member method code with example usage: obj.index_from_folder(cls, folder_path, index_dir, use_text_extraction, commit_every, breakup_docs, min_words, encoding, procs, limitmb, multisegment, verbose) without return types