#!/usr/bin/env python
# encoding: utf-8

from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
import os, sys, re
import logging
import argparse
import collections
import datetime
import time

logging.basicConfig(stream=sys.stderr, level=logging.INFO)
logger = logging.getLogger(__name__)


PROGS_OF_INTEREST = set(['Trinity', 'Butterfly', 'samtools', 'inchworm', 'QuantifyGraph',
                         'BubbleUpClustering', 'jellyfish', 'GraphFromFasta', 'bowtie2'])
                         




"""
Formatting from collectl -sZ

# PROCESS SUMMARY (counters are /sec)
# PID  User     PR  PPID THRD S   VSZ   RSS CP  SysT  UsrT Pct  AccuTime  RKB  WKB MajF MinF Command
    1  root     20     0    0 S   32M  552K  2  0.00  0.00   0  00:04.20    0    0    0    0 /sbin/init
    2  root     20     0    0 S     0     0  3  0.00  0.00   0  00:02.56    0    0    0    0 kthreadd
    3  root     RT     2    0 S     0     0  0  0.00  0.00   0  41:48.40    0    0    0    0 migration/0
    4  root     20     2    0 S     0     0  0  0.00  0.00   0  00:36.52    0    0    0    0 ksoftirqd/0

"""


def main():

    parser = argparse.ArgumentParser(description="generate time matrix from collectl dat file",
                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    
    parser.add_argument("--dat", dest="dat_file", type=str, default="", required=True,
                        help="collectl dat file")

    parser.add_argument("--out_prefix", dest="out_prefix", type=str, default="collectl",
                        help="prefix for output files")

    parser.add_argument("--debug", required=False, action="store_true", default=False, help="debug mode")

    args = parser.parse_args()

    if args.debug:
        logger.setLevel(logging.DEBUG)      

    
    cpu_usage_matrix = collections.defaultdict(dict)
    memory_usage_matrix = collections.defaultdict(dict)
    IO_usage_matrix = collections.defaultdict(dict)
    
    times = list()
    prognames = list()

    parse_collectl_dat(args.dat_file,
                       cpu_usage_matrix, memory_usage_matrix, IO_usage_matrix,
                       times, prognames)
    
    output_prefix = args.out_prefix
    cpu_usage_output_filename = output_prefix + ".cpu_usage.matrix"
    mem_usage_output_filename = output_prefix + ".mem_usage.matrix"
    IO_usage_output_filename = output_prefix + ".IO_usage.matrix"
    
    cpu_usage_ofh = open(cpu_usage_output_filename, 'w')
    mem_usage_ofh = open(mem_usage_output_filename, 'w')
    IO_usage_ofh = open(IO_usage_output_filename, 'w')

    # print headers
    header = "time\t" + "\t".join(prognames)

    cpu_usage_ofh.write(header + "\n")
    mem_usage_ofh.write(header + "\n")
    IO_usage_ofh.write(header + "\n")

    
    for timeval in times:
        memvals = list()
        cpuvals = list()
        IOvals = list()

        timeval_in_hr = str(timeval / 60 / 60)
        timeval = str(timeval)

        memvals.append(timeval_in_hr)
        cpuvals.append(timeval_in_hr)
        IOvals.append(timeval_in_hr)
        
        for progname in prognames:
            memory_hash = memory_usage_matrix[timeval]
            memory_val = "0"
            if progname in memory_hash:
                memory_val = str(memory_hash[progname])
            memvals.append(memory_val)

            cpu_val = "0"
            cpu_hash = cpu_usage_matrix[timeval]
            if progname in cpu_hash:
                cpu_val = str(cpu_hash[progname])
            cpuvals.append(cpu_val)
            
            IO_val = "0"
            IO_hash = IO_usage_matrix[timeval]
            if progname in IO_hash:
                IO_val = str(IO_hash[progname])
            IOvals.append(IO_val)
            
        mem_usage_ofh.write("\t".join(memvals) + "\n")
        cpu_usage_ofh.write("\t".join(cpuvals) + "\n")
        IO_usage_ofh.write("\t".join(IOvals) + "\n")
    
    mem_usage_ofh.close()
    cpu_usage_ofh.close()
    IO_usage_ofh.close()
    
    print("Done. See output files: " +
          str([mem_usage_output_filename, cpu_usage_output_filename, IO_usage_output_filename]))
    
        
    
        

    

def parse_collectl_dat(collectl_dat_file, cpu_usage_matrix, memory_usage_matrix, IO_usage_matrix, times, prognames):

    prev_time = 0
    prognames_dict = dict()

    """
    line formatting:
    0       #Date
    1       Time
    2       PID
    3       User
    4       PR
    5       PPID
    6       THRD
    7       S
    8       VSZ
    9       RSS
    10      CP
    11      SysT
    12      UsrT
    13      Pct
    14      AccuTime
    15      RKB
    16      WKB
    17      MajF
    18      MinF
    19      Command
    """

    init_time = -1

    with open(collectl_dat_file) as f:
        for line in f:
            vals = re.split("\s+", line)
            if len(vals) < 20:
                continue
            
            day = vals[0]
            timeval_str = vals[1]
            timestruct = datetime.datetime.strptime("{} {}".format(day, timeval_str), "%Y%m%d %H:%M:%S")
            timeval_numeric = time.mktime(timestruct.timetuple())

            if init_time < 0:
                init_time = timeval_numeric
                timeval_numeric = 1
            else:
                timeval_numeric -= init_time
            

            timeval_key_str = str(timeval_numeric)
            if timeval_numeric > prev_time:
                times.append(timeval_numeric)
                prev_time = timeval_numeric
            
            progname = os.path.basename(vals[19])
            if progname == 'perl':
                progname = os.path.basename(vals[20])
            elif progname == 'java':
                if re.search("Butterfly.jar", line):
                    progname = "Butterfly"
            

            if progname not in PROGS_OF_INTEREST:
                continue


            cpu_usage_timehash = cpu_usage_matrix[timeval_key_str]
            if progname not in cpu_usage_timehash:
                cpu_usage_timehash[progname] = 1
            else:
                cpu_usage_timehash[progname] += 1
            
            
            memory = compute_GB(vals[9])
            progname_memory_hash = memory_usage_matrix[timeval_key_str]
            if progname not in progname_memory_hash:
                progname_memory_hash[progname] = memory
            else:
                progname_memory_hash[progname] += memory
            
            #print("progname: {}, memoryG: {}".format(progname, memory))


            RMB = compute_GB(vals[15]) * (1024**2)
            WMB = compute_GB(vals[16]) * (1024**2)
            sumMB = RMB + WMB
            
            IO_usage_hash = IO_usage_matrix[timeval_key_str]
            if progname not in IO_usage_hash:
                IO_usage_hash[progname] = sumMB
            else:
                IO_usage_hash[progname] += sumMB
            


            prognames_dict[progname] = 1

    progname_list = prognames_dict.keys()

    prognames += progname_list
        
    
    
def compute_GB(memory_val):

    r = re.search("^(\d+)([MKG])$", memory_val)

    if r:
        numG = int(r.group(1))
        metric = r.group(2)

        if metric == 'M':
            numG /= 1024
        elif metric == 'K':
            numG /= 1024**2
    else:
        memory_val = int(memory_val)
        numG = memory_val / (1024**3)

    return numG 
    


 
####################
 
if __name__ == "__main__":
    main()
