content: import numpy as np
  import datetime as dtm
  from dateutil import rrule
  import pandas as pd
  import csv
  import matplotlib.pylab as plt
  import sys, os
  #lets first create the csv file
  #
  #change this to actual csv file name
  pingfile="weeklylogs.csv"
  #paramters @plotinterval = 10 minutes
  plotinterval = 10
  #csv file columns
  col_seq=0
  col_pingtime=1
  col_domain=2
  col_state=3
  #
  ########## FUNCTION TO SYNTHESEIZE MISSING DATA POINTS ##########
  #
  def synth_data(synthdf, interval):
      #create a temporary dataframe to hold the syntheseized data
      tmpdf = pd.DataFrame(columns=['seqnum', 'pingdatetime', 'domain', 'statenow'])
      #first check we have a none empty dataframe
      if not synthdf.empty:
          #pick the originating TS data point
          synthdf.sort_values(by='pingdatetime')
          #check if first timestamp starts at 00:00:00; if not add a dumy record
          startseqnum = synthdf.index[0]
          startpingdt = synthdf.iloc[0]['pingdatetime']
          startdomain = synthdf.iloc[0]['domain']
          startstate = synthdf.iloc[0]['statenow']
          #loop through each TS data point to synthetically add new TS points
          #to fill the gap between two consecutive data points
          for i, row in synthdf.iterrows():
              #initiate the synthesiezed data point to the origin
              nextdatapoint = 0
              pingdt_plus_interval = startpingdt
              #stepwise loop to add syntheseized points from relative origin to the next TS data point
              while row['pingdatetime'] > pingdt_plus_interval + dtm.timedelta(minutes = interval) :
                  nextdatapoint += 1
                  pingdt_plus_interval = startpingdt + dtm.timedelta(minutes = nextdatapoint*interval)
                  tmpdf.loc[len(tmpdf.index)] = [startseqnum,pingdt_plus_interval,startdomain,startstate]
              startseqnum = i
              startpingdt = row['pingdatetime']
              startstate = row['statenow']
              
          #after completing through all the TS datapoints check if a none empty dataframe was created
          if not tmpdf.empty:
              tmpdf = pd.concat([tmpdf,synthdf])
              tmpdf = tmpdf.set_index('seqnum')
      #whether null or not return a dataframe with syntheseized TS data
      tmpdf.dropna(thresh=2)
      return tmpdf
  #
  ########## PLOT HISTOGRAM TO FIGURE ##########
  #
  def plot_hist_to_fig(histdf, dname):
      #get date range of the plot to use in suptitile
      begdt = histdf['pingdatetime'].min().date()
      findt = histdf['pingdatetime'].max().date()
      #create a new x-axis index using dataframe index; starting from 1 instead of 0
      histdf['pingdate'] = histdf['pingdatetime'].apply(lambda x: x.date())
      downdf = pd.DataFrame(columns=['xlabel','pingdate', 'downcount'])
      datelist = list(histdf.pingdate.unique())
      for uniquedate in datelist:
          xlabel = str('{:02d}'.format(uniquedate.month))+'-'+str('{:02d}'.format(uniquedate.day))
          downcount = len(histdf[(histdf.statenow == '0') & (histdf.pingdate == uniquedate)])
          totalcount = len(histdf[(histdf.pingdate == uniquedate)])
          downdf.loc[len(downdf.index)] = [xlabel, uniquedate,100*downcount//totalcount]
      downdf = downdf.as_matrix()
      #x-axis values are in the newly generated xvalues column
      xl = np.array(downdf[:,0])
      x = np.array(downdf[:,1])
      #y-axis values (1 or 0) are in the dateframe statenow column
      y = np.array(downdf[:,2])
  
      histfig, ax = plt.subplots()
      ax.bar(x,y,color='red',width=0.5, align="center")
      #to give enough spacing for the suptitle; otherwise overlaps with title
      histfig.subplots_adjust(top=0.87)
  #    plt.figure(figsize=(8,6), dpi=150)
      #beautify the plot and name the labels, titles
      ax.set_title('Percentage of time Server Failed each Day', fontsize=14, fontweight='bold', color='gray')
      histfig.suptitle(dname+'\n'+str(begdt)+' --- '+str(findt), fontsize=10, color='blue')
      ax.set_xlabel('Month-Day', fontsize=12, color='gray')
      ax.set_ylabel('Faile Rate (%)', fontsize=12, color='gray')
      plt.yticks(fontsize=10, color='gray', rotation='horizontal')
      plt.xticks(x, xl, fontsize=10, color='gray', rotation='vertical')
      ax.grid(True)
  
      return histfig
  #
  ########## PLOT DOWN TIMES FREQUENCY TO FIGURE ##########
  #
  def plot_freq_to_fig(plotdf, dname):
      #get date range of the plot to use in suptitile
      begdt = plotdf['pingdatetime'].min().date()
      findt = plotdf['pingdatetime'].max().date()
      failrate = 100-(sum(100*plotdf['statenow'].astype(int))/len(plotdf))
      failrate = failrate.astype(float)
      #create a new x-axis index using dataframe index; starting from 1 instead of 0
      plotdf['xvalues'] = range(1,len(plotdf)+1)
      plotdf = plotdf.as_matrix()
      #x-axis values are in the newly generated xvalues column
      x = np.array(plotdf[:,3].astype(int))
      #y-axis values (1 or 0) are in the dateframe statenow column
      y = np.array(plotdf[:,2].astype(int))
      
      #setup to catputure the plot into a figure
      plotfig = plt.figure(num=None, figsize=(8, 6), dpi=150, facecolor='y', edgecolor='k')
      ax = plotfig.add_subplot(311)
      ax.fill_between(x, 0, y, color='green')
      ax.plot(x,y,color='green',lw=2)
      #to give enough spacing for the suptitle; otherwise overlaps with title
      plotfig.subplots_adjust(top=0.87)
      #beautify the plot and name the labels, titles
      ax.set_title('Frequency of Server Access Failure ('+str(failrate)+'%)', fontsize=14, fontweight='bold', color='gray')
      plotfig.suptitle(dname+'\n'+str(begdt)+' --- '+str(findt), fontsize=10, color='blue')
      ax.set_xlabel('Attempted Machine Accesss Times', fontsize=12, color='gray')
      ax.set_ylabel('Machine State', fontsize=12, color='gray')
      plt.yticks(y, ['UP','DOWN'], fontsize=10, color='gray', rotation='vertical')
      plt.xticks(fontsize=10, color='gray', rotation='horizontal')
      plt.ylim(0,1.1)
      plt.xlim(0,x.max()+10)
      ax.grid(True)
  
      return plotfig
  #
  ############# MAIN ################################
  #
  print("Complile data from file the log files")
  #os.system('./analytics.sh')
  print("Reading data from file "+pingfile)
  with open(pingfile, 'rb') as f:
      data = [i.split(",") for i in f.read().split()]
      df = pd.DataFrame(data, columns=['seqnum', 'pingdatetime', 'domain', 'statenow'])
      for index, row in df.iterrows():
          row[col_pingtime] = dtm.datetime.strptime(row[col_pingtime], '%Y-%m-%d:%H:%M:%S')
          #to avoid duplicate data and to reflect ping time to be on the minute
          row[col_pingtime] = row[col_pingtime].replace(second = 0)
      #format pingdatetime as proper datetime, set it as the indext and then order them
      df['pingdatetime'] = pd.to_datetime(df['pingdatetime'])
      df.sort_values(by='pingdatetime')
      df = df.set_index('seqnum')
      #begin processing for each unique domain
      print(str(len(df.index))+" data rows added to the dataframe, ready for processing ...")
      print ('-----------------------------------------------------')
      for thedomain in df.domain.unique():
          #insert syntheseised data points
          dompingdf = df[df['domain']==thedomain]
          print("Begin data synthesis for "+thedomain+" with data rows = "+str(len(dompingdf.index)))
          amenddf = synth_data(dompingdf,plotinterval)
          if not amenddf.empty:
              #output the syntheseized dataframe to output file
              print(str(len(amenddf.index))+" data rows of syntheseised added to "+thedomain )
              amenddf['pingdatetime'] = pd.to_datetime(amenddf.pingdatetime)
              amenddf = amenddf.sort(['pingdatetime'])
              amenddf.index = range(0,len(amenddf))
              print('writing data to file: ./data/syndata_'+thedomain+'.csv')
              amenddf.to_csv('./data/syndata_'+thedomain+'.csv')
              #plot timeseries with function (need to add if conditions to check if function returns valid fig)
              fig = plot_freq_to_fig(amenddf, thedomain)
              fig.savefig('./plots/freqplot_'+thedomain+'.png', bbox_inches='tight')
              print ('frequency plot created in file: ./plots/freqplot_'+thedomain+'.png')
              fig = plot_hist_to_fig(amenddf, thedomain)
              fig.savefig('./plots/histplot_'+thedomain+'.png', bbox_inches='tight')
              print ('histogram plot created in file: ./plots/histplot_'+thedomain+'.png')
              print ('process complete for '+thedomain)
              print ('-----------------------------------------------------')
          else:
              print ("Warning: no syntheseized data was added to: "+thedomain)
              print ('-----------------------------------------------------')
      print ('End processing data for visualization !!! ')
  