code: # -*- coding: utf-8 -*-
  
  # In this script we find all the history behind an evicted task that has not been finished/killed/failed/lost.
  # We find how many times it has been submitted and what were the events related to this task. 
  # The violatedtasks dictionary is the one that is we are looking for. It has only one entry
  # for each (jobid, taskindex) and the rest is stored in a multidimensional array.
  # This database is a test as it is only using 1/500 of the task_events table. 
  # Also the not_finished_evictedtasks is only for the first(1/500) part of the tasks_events table.
  # Since the script assumed that the machine state changes in the machine table when a task is added, 
  # it is not getting the right results. The problem is that the machines table is only updated when a
  # machine is added so it is just the events of the machines, in order to find the available cpu, 
  # memory and disk of a specific machine at the time that a task is assigned to a machine, 
  # we need to have more complicated calculations.(refer to createDB3) 
  
  # @author: reyhane_askari
  # Universite de Montreal, Dec 2015
  
  from os import chdir, listdir
  from pandas import read_csv
  from os import path
  from random import randint, sample, seed
  from collections import OrderedDict
  from pandas import DataFrame
  import numpy as np 
  import matplotlib.pyplot as plt
  import csv
  import codecs
  
  chdir('/home/askrey/Dropbox/Project_step_by_step/2_find_violations/csvs')
  task_events_csv_colnames = ['time', 'missing', 'job_id', 'task_idx', 'machine_id', 'event_type', 'user', 'sched_cls', 
                              'priority', 'cpu_requested', 'mem_requested', 'disk', 'restriction']         
  evictedtasks = OrderedDict([])
  violatedtasks = OrderedDict([])
  for key, val in csv.reader(open("not_finished_evictedtasks.csv")):
      evictedtasks[key] = val
  
  machines_dictionary = OrderedDict([])
  
  #load machine events table:
  chdir('/home/askrey/Final_project') 
  reader = csv.reader(codecs.open('part-00000-of-00001.csv','rU','utf-8'))
  
  # key of machines_dictionary is the primary fields of the machine events table (time, machine id) 
  # other fields: event type, platform id, CUPs, memory
  
  for row in reader:
    machines_dictionary[(row[0],row[1])] = row[2:]
  
  #for fn in sorted(listdir('task_events')):
  fp = path.join('task_events',sorted(listdir('task_events'))[0])
  task_events_df = read_csv(fp, header = None, index_col = False, names = task_events_csv_colnames, compression = 'gzip')
  
  for index, event in task_events_df.iterrows():
      
      if (event['job_id'], event['task_idx']) in violatedtasks:
        violatedtasks[event['job_id'],event['task_idx']][0].append(event['time'])
        violatedtasks[event['job_id'],event['task_idx']][2].append(event['machine_id'])
        violatedtasks[event['job_id'],event['task_idx']][3].append(event['event_type'])
        violatedtasks[event['job_id'],event['task_idx']][11].append((machines_dictionary[(str(event['time']),str(event['machine_id']))] if (str(event['time']), str(event['machine_id'])) in machines_dictionary else 0))
  
      elif ("("+str(event['job_id'])+ ", "+ str(event['task_idx'])+")") in evictedtasks:
        violatedtasks[event['job_id'],event['task_idx']] = [[event['time']],event['missing'],[event['machine_id']],
  							  [event['event_type']], event['user'], event['sched_cls'], event['priority'], event['cpu_requested'],
  							  event['mem_requested'], event['disk'], event['restriction'], 
  							  [(machines_dictionary[(str(event['time']),str(event['machine_id']))] if (str(event['time']), str(event['machine_id'])) in machines_dictionary else 0 )]]
  
  # moshkel alan in hast k ye jahaE event ha hanooz machine barashoon assign nashode vase hamin hast k machine id nan hast
  
  writer = csv.writer(open('/home/askrey/Dropbox/Databases/testDB5.csv', 'wb'))
  for key, value in violatedtasks.items():
     writer.writerow([key, value])
   # -*- coding: utf-8 -*-
  
  # In this script we use a simple classifer called naive bayes and try to predict the violations. But before that we use
  # some methods to tackle the problem of our skewed dataset. :) 
  
  # 11 May 2016
  # @author: reyhane_askari
  # Universite de Montreal, DIRO
  
  import csv
  import numpy as np
  from sklearn.metrics import roc_curve, auc
  from sklearn.cross_validation import train_test_split
  from sklearn import metrics
  import pandas as pd
  from os import chdir, listdir
  from pandas import read_csv
  from os import path
  from random import randint, sample, seed
  from collections import OrderedDict
  from pandas import DataFrame, Series
  import numpy as np 
  import csv
  import codecs
  
  import matplotlib
  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
  import matplotlib as mpl
  import seaborn as sns
  sns.set()
  import itertools
  from sklearn.decomposition import PCA
  from unbalanced_dataset import UnderSampler, NearMiss, CondensedNearestNeighbour, OneSidedSelection,\
  NeighbourhoodCleaningRule, TomekLinks, ClusterCentroids, OverSampler, SMOTE,\
  SMOTETomek, SMOTEENN, EasyEnsemble, BalanceCascade
  
  almost_black = '#262626'
  
  colnames = ['old_index','job_id', 'task_idx','sched_cls', 'priority', 'cpu_requested',
              'mem_requested', 'disk', 'violation'] 
  
  tain_path = r'/home/askrey/Dropbox/Project_step_by_step/3_create_database/csvs/frull_db_2.csv'
  
  X = pd.read_csv(tain_path, header = None, index_col = False ,names = colnames, skiprows = [0],  usecols = [3,4,5,6,7])
  y = pd.read_csv(tain_path, header = None, index_col = False ,names = colnames, skiprows = [0],  usecols = [8])
  y = y['violation'].values
  # X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=.333, random_state=0)
  main_x = X.values
  main_y = y
  
  verbose = False
  ratio = float(np.count_nonzero(y==1)) / float(np.count_nonzero(y==0))
  # 'SMOTE bordeline 1'
  bsmote1 = SMOTE(ratio=ratio, verbose=verbose, kind='borderline1')
  x, y = bsmote1.fit_transform(main_x, main_y)
  
  ratio = float(np.count_nonzero(y==1)) / float(np.count_nonzero(y==0))
  X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=.333, random_state=0)
  
  from sklearn.ensemble import RandomForestClassifier
  from sklearn.cross_validation import cross_val_score
  
  clf = RandomForestClassifier(n_estimators=10)
  scores = cross_val_score(clf, X_test, y_test)
  
  y_pred = clf.fit(X_train, y_train).predict(X_test)
  y_score = clf.fit(X_train, y_train).predict_proba(X_test)[:,1]
  
  
  mean_accuracy = clf.fit(X_train, y_train).score(X_test,y_test,sample_weight=None)
  fpr, tpr, thresholds = metrics.roc_curve(y_test, y_score)
  
  roc_auc = auc(fpr, tpr)
  
  plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % roc_auc)
  plt.plot([0, 1], [0, 1], 'k--')
  plt.xlim([0.0, 1.0])
  plt.ylim([0.0, 1.05])
  plt.xlabel('False Positive Rate')
  plt.ylabel('True Positive Rate')
  plt.title('Receiver operating characteristic example')
  plt.legend(loc="lower right")
  
  plt.savefig('/home/askrey/Dropbox/Project_step_by_step/5_simple_models/new_scripts/random_forest_SMOTE_bordeline_1.pdf')
  
apis: pandas.read_csv
apis: matplotlib.pyplot.legend pandas.read_csv sklearn.metrics.roc_curve sklearn.metrics.auc matplotlib.pyplot.savefig sklearn.cross_validation.train_test_split matplotlib.pyplot.xlim matplotlib.pyplot.title numpy.count_nonzero sklearn.cross_validation.cross_val_score matplotlib.pyplot.ylabel sklearn.ensemble.RandomForestClassifier matplotlib.pyplot.ylim matplotlib.use matplotlib.pyplot.plot matplotlib.pyplot.xlabel