text: #!python
  # Pre-process the data according to a given pattern
  
  from functools import partial
  import numpy as np
  import multiprocessing
  import pandas as pd
  import os
  import Modes
  
  np.set_printoptions(formatter={'float_kind': lambda x: "%.2f" % x}, linewidth=200)
  
  
  def processing(mode, dataFile):
      currentFile = os.path.join(mode.PREPROCESSED_DIR, dataFile)
      if os.path.isfile(currentFile):
          preprocessed_df = pd.read_csv(currentFile, header=None)
      else:
          preprocessed_df = []
      df = pd.read_csv(os.path.join(mode.EXTRACTED_DIR, dataFile), names=mode.COLUMNS, dtype=mode.DTYPE, skiprows=1)
      print(currentFile, len(df) - len(preprocessed_df), "rows to analyze")
      data = pd.DataFrame(columns=range(mode.INPUT_SIZE + mode.OUTPUT_SIZE))
      for i in range(len(preprocessed_df), len(df)):
          if i % mode.SAVE == 0 and i != len(preprocessed_df):  # saving periodically because the process is rather long
              print(currentFile, len(df) - i)
              data = data.astype(int)
              data.to_csv(currentFile, mode='a', header=False, index=False)
              data = pd.DataFrame(columns=range(mode.INPUT_SIZE + mode.OUTPUT_SIZE))
  
          state = df.iloc[i]
          data.loc[len(data)] = mode.row_data(state, with_output=True)
      if len(data):
          data = data.astype(int)
          data.to_csv(currentFile, mode='a', header=False, index=False)
      print(currentFile, 'DONE')
  
  
  def run(mode, cpu):
      if not os.path.isdir(mode.PREPROCESSED_DIR):
          os.makedirs(mode.PREPROCESSED_DIR)
  
      # listing extracted files and sorting
      extracted_files = [f for f in os.listdir(mode.EXTRACTED_DIR)]
      l = list(map(lambda x: int(x.replace('data_', '').replace('.csv', '')), extracted_files))
      l = sorted(range(len(l)), key=lambda k: l[k])
      extracted_files = [extracted_files[k] for k in l]
  
      pool = multiprocessing.Pool(processes=cpu)
      fun = partial(processing, mode)
      pool.map(fun, extracted_files, chunksize=1)
      pool.close()
      pool.join()
  
  if __name__ == '__main__':
      m = Modes.ABR_TJMCS_Mode(['9.1','9.2','9.3','9.4','9.5','9.6','9.7'])
      run(m, max(multiprocessing.cpu_count() - 1, 1))
  