text: import os
  import sys
  import numpy as np
  import pandas as pd
  from datetime import datetime
  import json
  
  from crossasr.utils import preprocess_text, read_json
  from crossasr.utils import make_dir
  from utils import set_seed
  
  def generate_europarl_corpus():
  
      # download full data from https://www.kaggle.com/djonafegnem/europarl-parallel-corpus-19962011
      data = "corpus/europarl-parallel-corpus-19962011/"
  
      df = pd.DataFrame([""], columns=["English"])
      fpaths = []
      for (_, _, filenames) in os.walk(data):
          print(filenames)
          for f in filenames:
              if ".csv" in f:
                  fpath = data + f
                  fpaths.append(fpath)
      
      fpaths = sorted(fpaths)
      for fpath in fpaths :
          print(fpath)
          d = pd.read_csv(fpath, delimiter=',')
          df = pd.concat([df["English"], d["English"]])
          df = pd.DataFrame(df, columns=["English"])
  
      df = df.rename(columns={'English': 'sentence'})
  
      fpath = "corpus/europarl-full.csv"
      df.to_csv(fpath, index=False)
  
      return fpath
  
  
  def get_sample_data(df, n):
      sample = df[:n]
  
      return pd.DataFrame(sample, columns=["sentence"])
  
  
  def preprocess_data(df, n):
  
      clean_df = df["sentence"].apply(preprocess_text)
  
      # remove empty string
      clean_df = [i for i in clean_df if i and i != ""]
  
      return clean_df[:n]
  
  
  if __name__ == "__main__":
  
      json_config_path = "config.json"
      config = read_json(json_config_path)
  
      print(config)
      seed = config["seed"]
  
      set_seed(seed)
  
      print("start: " + str(datetime.now()))
  
      output_file = generate_europarl_corpus()
  
      print("generate corpus: " + str(datetime.now()))
  
      fpath = output_file
  
      df = pd.read_csv(fpath, delimiter=',')
      print("read data: " + str(datetime.now()))
  
      # drop null
      df = df.dropna()
  
      # drop duplicates
      df = df.drop_duplicates()
  
      # reset index
      df = df.reset_index(drop=True)
  
      print("get sample: " + str(datetime.now()))
      # get sample data
      
      df = df.sample(frac=1, random_state=seed).reset_index(drop=True)
  
      N = 20000
      sample_df = get_sample_data(df, int(2 * N))
  
      print("preprocess data: " + str(datetime.now()))
      
      # text preprocessing
      data = preprocess_data(sample_df, N)
  
      print("write data: " + str(datetime.now()))
  
      # prepare folder to save the data
      directory = config["output_dir"]
      if len(config["corpus_fpath"].split("/")) > 1 :
          directory = os.path.join(directory, "/".join(config["corpus_fpath"].split("/")[:-1]))
      make_dir(directory)
      
      outfile = os.path.join(config["output_dir"], config["corpus_fpath"])
  
      file = open(outfile, "w+")
      for s in data:
          file.write("%s\n" % s)
      file.close()
  