text: import os
  import re
  import json
  import gensim
  import numpy as np
  import string
  from nltk import word_tokenize
  from nltk.corpus import stopwords
  from torchtext.vocab import Vectors
  
  english_stopwords = set(stopwords.words("english"))
  non_alphanum_regex = re.compile(r"\W+")
  
  
  def preprocess(doc, method="nltk", dataset=True):
      if method == "spacy":
          tokens = " ".join(
              [
                  token.lower_
                  for token in doc
                  if token
                  and not (token.lower_ == "null" or token.is_stop or token.is_punct)
              ]
          )
      elif method == "nltk":
          # doc = non_alphanum_regex.sub(' ', doc).lower()
          tokens = " ".join(
              [
                  token
                  for token in word_tokenize(doc.lower())
                  if not (
                      token == "null"
                      or token in english_stopwords
                      or token in string.punctuation
                  )
              ]
          )
      if dataset or tokens != "":
          return tokens
  
  
  def parse_content_line(x, attributes=None, label=True):
      if attributes is None:
          attributes = ["title_left", "title_right"]
      item = json.loads(x)
      elements = [item[attr] if item[attr] is not None else "" for attr in attributes]
      if label:
          ll = int(item["label"])
          elements.append(ll)
      item = np.array(elements)
      return item[np.newaxis, :]
  
  
  def resave_w2v_model(old_path, new_path):
      w2v_model = gensim.models.KeyedVectors.load_word2vec_format(old_path, binary=True)
      w2v_model.save_word2vec_format(fname=new_path)
  
  
  def resave_fasttext_model(old_path, new_path):
      fasttext_model = gensim.models.fasttext.load_facebook_model(old_path)
      fasttext_model.save(fname=new_path)
  
  
  def load_embedding(TEXT, embedding_path):
      _, file_extension = os.path.splitext(embedding_path)
  
      if file_extension == ".bin":
          embedding_name = os.path.basename(embedding_path)
          embedding_dir = os.path.dirname(embedding_path)
          vectors = Vectors(name=embedding_name, cache=embedding_dir)
      elif file_extension == ".txt":
          vectors = Vectors(name=embedding_path)
      else:
          raise NotImplementedError()
  
      TEXT.vocab.set_vectors(vectors.stoi, vectors.vectors, vectors.dim)
  