code: def preprocess():
      """collect and preprocess raw data from acl Imdb dataset"""
      nltk.download("stopwords")
  
      print("preparing raw imdb data")
      data_gz = check_exist_or_download(imdb_dataset_link)
      data_dir = unzip_data(download_dir, data_gz)
  
      # imdb dirs
      # vocab_f = data_dir + '/imdb.vocab'
      train_pos_dir = data_dir + "/train/pos/"
      train_neg_dir = data_dir + "/train/neg/"
      test_pos_dir = data_dir + "/test/pos/"
      test_neg_dir = data_dir + "/test/neg/"
  
      # nltk helpers
      tokenizer = ToktokTokenizer()
      stopword_list = nltk.corpus.stopwords.words("english")
  
      # load pretrained word2vec binary
      print("loading pretrained word2vec")
      google_news_pretrain_fp = check_exist_or_download(
          google_news_pretrain_embeddings_link
      )
      wv = KeyedVectors.load_word2vec_format(
          google_news_pretrain_fp, binary=True
      )
  
      # parse flat files to memory
      data = []
      for data_dir, label in [
          (train_pos_dir, 1),
          (train_neg_dir, 0),
          (test_pos_dir, 1),
          (test_neg_dir, 0),
      ]:
          for filename in os.listdir(data_dir):
              if filename.endswith(".txt"):
                  with open(
                      os.path.join(data_dir, filename),
                      "r",
                      encoding="utf-8",
                  ) as fhdl:
                      data.append((fhdl.read(), label))
  
      # text review cleaning
      print("cleaning text review")
      imdb_data = pd.DataFrame(data, columns=["review", "label"])
      imdb_data["review"] = imdb_data["review"].apply(strip_html)
      imdb_data["review"] = imdb_data["review"].apply(
          remove_between_square_brackets
      )
      imdb_data["review"] = imdb_data["review"].apply(
          remove_special_characters
      )
      imdb_data["review"] = imdb_data["review"].apply(simple_stemmer)
      imdb_data["review"] = imdb_data["review"].apply(
          remove_stopwords, args=(tokenizer, stopword_list)
      )
      imdb_data["token"] = imdb_data["review"].apply(tokenize)
  
      # build  word2index and index2word
      w2i = dict()
      i2w = dict()
  
      # add vocab <pad> as index 0
      w2i["<pad>"] = 0
      i2w[0] = "<pad>"
  
      idx = 1  # start from idx 1
      for index, row in imdb_data["token"].iteritems():
          for w in row:
              if w in wv and w not in w2i:
                  w2i[w] = idx
                  i2w[idx] = w
                  assert idx < 28241
                  idx += 1
      assert len(w2i) == len(i2w)
      print("vocab size: ", len(w2i))
  
      # encode tokens to int
      imdb_data["encoded"] = imdb_data["token"].apply(
          encode_token, args=(wv, w2i)
      )
  
      # select word vector weights for embedding layer from vocab
      embed_weights = []
      for w in w2i.keys():
          val = None
          if w in wv:
              val = wv[w]
          else:
              val = np.zeros(
                  [
                      300,
                  ]
              )
          embed_weights.append(val)
      embed_weights = np.array(embed_weights)
      print(
          "embedding layer lookup weight shape: ", embed_weights.shape
      )
  
      # split into train and test
      train_data = imdb_data[["encoded", "label"]].values
      train, val = train_test_split(
          train_data, test_size=0.33, random_state=42
      )
  
      # save preprocessed for training
      imdb_processed = {
          "train": train,
          "val": val,
          "embed_weights": embed_weights,
          "w2i": w2i,
          "i2w": i2w,
      }
      print("saving preprocessed file to ", preprocessed_imdb_data_fp)
      with open(preprocessed_imdb_data_fp, "wb") as handle:
          pickle.dump(
              imdb_processed, handle, protocol=pickle.HIGHEST_PROTOCOL
          )
  
imports: import re
  import os
  import pickle
  import urllib
  import tarfile
  import numpy
  import pandas
  import nltk
  from nltk.stem import PorterStemmer
  from nltk.tokenize.toktok import ToktokTokenizer
  from gensim.models.keyedvectors import KeyedVectors
  from sklearn.model_selection import train_test_split
  from bs4 import BeautifulSoup
label_desc: Use image node_id 12 for calling a global function with example usage: preprocess() without return types