class_docstr: NER preprocessing base class
class_docstr_tok: ["NER","preprocessing","base","class"]
code: def preprocess(self, sentences, lang=None, custom_tokenizer=None):
      if type(sentences) != list:
          raise ValueError("Param sentences must be a list of strings")
  
      # language detection
      if lang is None:
          lang = TU.detect_lang(sentences)
  
      # set tokenizer
      if custom_tokenizer is not None:
          tokfunc = custom_tokenizer
      elif TU.is_chinese(
          lang, strict=False
      ):  # strict=False: workaround for langdetect bug on short chinese texts
          tokfunc = lambda text: [c for c in text]
      else:
          tokfunc = TU.tokenize
  
      # preprocess
      X = []
      y = []
      for s in sentences:
          tokens = tokfunc(s)
          X.append(tokens)
          y.append([OTHER] * len(tokens))
      from .dataset import NERSequence
  
      nerseq = NERSequence(X, y, p=self.p)
      return nerseq
  
imports: from ...None import utils
  from ...imports import *
  from ...preprocessor import Preprocessor
  from ..None import preprocessor
  from ..None import textutils
label_desc: Use image node_id 6 for calling the NERPreprocessor obj's underlying member method code with example usage: obj.preprocess(sentences, lang, custom_tokenizer) and returns: nerseq