class_docstr: Default tokenizer used by BOWEncoder for parsing and tokenizing.
      
class_docstr_tok: ["Default","tokenizer","used","by","BOWEncoder","for","parsing","and","tokenizing","."]
docstr: Returns the sentence (or prose) as a parsed list of tokens.
  
  Arguments:
      sentence {str} -- Single sentence/prose that needs to be tokenized.
  
  Keyword Arguments:
      keep_ids {bool} -- If True, returned tokens are indexed by their
          original positions in the parsed sentence. If False, the returned
          tokens do not preserve positionality. Has to be set to False for
          training purposes. Set to true at text/execution time, when user
          needs explanability. (default: {False})
  
  Returns:
      list -- List of all tokens extracted from the sentence.
docstr_tok: ["Returns","the","sentence","(","or","prose",")","as","a","parsed","list","of","tokens",".","Arguments",":","sentence","{","str","}","--","Single","sentence\/prose","that","needs","to","be","tokenized",".","Keyword","Arguments",":","keep_ids","{","bool","}","--","If","True",",","returned","tokens","are","indexed","by","their","original","positions","in","the","parsed","sentence",".","If","False",",","the","returned","tokens","do","not","preserve","positionality",".","Has","to","be","set","to","False","for","training","purposes",".","Set","to","true","at","text\/execution","time",",","when","user","needs","explanability",".","(","default",":","{","False","}",")","Returns",":","list","--","List","of","all","tokens","extracted","from","the","sentence","."]
code: def tokenize(self, sentence, keep_ids=False):
      """Returns the sentence (or prose) as a parsed list of tokens.
  
      Arguments:
          sentence {str} -- Single sentence/prose that needs to be tokenized.
  
      Keyword Arguments:
          keep_ids {bool} -- If True, returned tokens are indexed by their
              original positions in the parsed sentence. If False, the returned
              tokens do not preserve positionality. Has to be set to False for
              training purposes. Set to true at text/execution time, when user
              needs explanability. (default: {False})
  
      Returns:
          list -- List of all tokens extracted from the sentence.
      """
      EMPTYTOKEN = "empty_token"
      # Creating our token object, which is used to create documents with linguistic annotations.
      mytokens = self.parser(sentence)
  
      # Lemmatizing each token, removing blank space and converting each token into lowercase.
      mytokens = [
          word.lemma_.lower().strip()
          if word.lemma_ != "-PRON-"
          else word.lower_
          for word in mytokens
      ]
  
      # Removing stop words.
      if keep_ids is True:
          return [
              word
              if word not in self.stop_words
              and word not in self.punctuations
              else EMPTYTOKEN
              for word in mytokens
          ]
      else:
          return [
              word
              for word in mytokens
              if word not in self.stop_words
              and word not in self.punctuations
          ]
  
file_path: interpret-text/python/interpret_text/experimental/common/utils_classical.py
imports: import spacy
  import html
  import string
  import numpy
  import matplotlib.pyplot
  from sklearn.preprocessing import LabelEncoder
  from IPython.core.display import display, HTML
  from sklearn.feature_extraction.text import CountVectorizer
label_desc: Use image node_id 2 for calling the BOWTokenizer obj's underlying member method code with example usage: obj.tokenize(sentence, keep_ids) and returns: unknown, unknown