 #SNOPSIS

 #runs k-means cluster analysis

 #AUTHOR
 # Isaak Y Tecle (iyt2@cornell.edu)

options(echo = FALSE)

library(randomForest)
library(data.table)
library(genoDataFilter)
library(tibble)
library(dplyr)
library(fpc)
library(cluster)
library(ggfortify)
library(tibble)
library(stringi)
library(phenoAnalysis)

allArgs <- commandArgs()

outputFiles  <- grep("output_files", allArgs, value = TRUE)
outputFiles <- scan(outputFiles, what = "character")

inputFiles  <- grep("input_files", allArgs, value = TRUE)
inputFiles <- scan(inputFiles, what = "character")

kResultFile <- grep("result", outputFiles, value = TRUE)
reportFile  <- grep("report", outputFiles, value = TRUE)
errorFile   <- grep("error", outputFiles, value = TRUE)

combinedDataFile <- grep("combined_cluster_data_file", outputFiles, value = TRUE)

plotPamFile    <- grep("plot_pam", outputFiles, value = TRUE)
plotKmeansFile <- grep("k-means-plot", outputFiles, value = TRUE)
optionsFile    <- grep("options", inputFiles,  value = TRUE)

clusterOptions <- read.table(optionsFile,
                             header = TRUE,
                             sep = "\t",
                             stringsAsFactors = FALSE,
                             na.strings = "")
print(clusterOptions)
clusterOptions <- column_to_rownames(clusterOptions, var = "Params")
userKNumbers   <- as.numeric(clusterOptions["k_numbers", 1])
dataType       <- clusterOptions["data_type", 1]
selectionProp  <- as.numeric(clusterOptions["selection_proportion", 1])
predictedTraits <- clusterOptions["predicted_traits", 1]
predictedTraits <- unlist(strsplit(predictedTraits, ','))

if (is.null(kResultFile)) {
  stop("Clustering output file is missing.")
  q("no", 1, FALSE)
}

clusterData <- c()
genoData    <- c()
genoFiles   <- c()
reportNotes <- c()
genoDataMissing <- c()

extractGenotype <- function(inputFiles) {

    genoFiles <- grep("genotype_data", inputFiles,  value = TRUE)

    genoMetaData <- c()
    filteredGenoFile <- c()

    if (length(genoFiles) > 1) {
        genoData <- combineGenoData(genoFiles)

        genoMetaData   <- genoData$trial
        genoData$trial <- NULL

    } else {
        genoFile <- genoFiles
        genoData <- fread(genoFile,
                          header = TRUE,
                          na.strings = c("NA", " ", "--", "-", "."))

        if (is.null(genoData)) {
            filteredGenoFile <- grep("filtered_genotype_data_",  genoFile, value = TRUE)
            genoData <- fread(filteredGenoFile, header = TRUE)
        }

        genoData <- unique(genoData, by = 'V1')
        genoData <- data.frame(genoData)
        genoData <- column_to_rownames(genoData, 'V1')
    }

    if (is.null(genoData)) {
        stop("There is no genotype dataset.")
        q("no", 1, FALSE)
    } else {

        ##genoDataFilter::filterGenoData
        genoData <- convertToNumeric(genoData)
        genoData <- filterGenoData(genoData, maf=0.01)
        genoData <- roundAlleleDosage(genoData)

        message("No. of geno missing values, ", sum(is.na(genoData)))
        if (sum(is.na(genoData)) > 0) {
            genoDataMissing <- c('yes')
            genoData <- na.roughfix(genoData)
        }

        genoData <- data.frame(genoData)
    }
}

set.seed(235)

clusterDataNotScaled <- c()

if (grepl('genotype', dataType, ignore.case = TRUE)) {
    clusterData <- extractGenotype(inputFiles)

    pca    <- prcomp(clusterData, retx = TRUE)
    pca    <- summary(pca)

    variances <- data.frame(pca$importance)

    varProp        <- variances[3, ]
    varProp        <- data.frame(t(varProp))
    names(varProp) <- c('cumVar')

    selectPcs <- varProp %>% filter(cumVar <= 0.9)
    pcsCnt    <- nrow(selectPcs)

    reportNotes <- paste0('Before clustering this dataset, principal component analysis (PCA) was done on it. ', "\n")
    reportNotes <- paste0(reportNotes, 'Based on the PCA, ', pcsCnt, ' PCs are used to cluster this dataset. ', "\n")
    reportNotes <- paste0(reportNotes, 'They explain 90% of the variance in the original dataset.', "\n")

    scores   <- data.frame(pca$x)
    scores   <- scores[, 1:pcsCnt]
    scores   <- round(scores, 3)

    variances <- variances[2, 1:pcsCnt]
    variances <- round(variances, 4) * 100
    variances <- data.frame(t(variances))

    clusterData <- scores
} else {

    if (grepl('gebv', dataType, ignore.case = TRUE)) {
        gebvsFile <- grep("combined_gebvs", inputFiles,  value = TRUE)
        gebvsData <- data.frame(fread(gebvsFile, header = TRUE))

        clusterNa   <- gebvsData %>% filter_all(any_vars(is.na(.)))
        clusterData <- column_to_rownames(gebvsData, 'V1')
    } else if (grepl('phenotype', dataType, ignore.case = TRUE)) {

        metaFile <- grep("meta", inputFiles,  value = TRUE)

        clusterData <- cleanAveragePhenotypes(inputFiles, metaDataFile = metaFile)

        if (!is.na(predictedTraits) & length(predictedTraits) > 1) {
            clusterData <- rownames_to_column(clusterData, var = 'germplasmName')
            clusterData <- clusterData %>% select(c(germplasmName, predictedTraits))
            clusterData <- column_to_rownames(clusterData, var = 'germplasmName')
        }
    }

    clusterDataNotScaled <- na.omit(clusterData)

    clusterData <- scale(clusterDataNotScaled, center=TRUE, scale=TRUE)
    reportNotes <- paste0(reportNotes, 'Note: Data was standardized before clustering.', "\n")
}

sIndexFile <- grep("selection_index", inputFiles, value = TRUE)
selectedIndexGenotypes <- c()

if (length(sIndexFile) != 0) {
    sIndexData <- data.frame(fread(sIndexFile, header = TRUE))
    selectionProp <- selectionProp * 0.01
    selectedIndexGenotypes <- sIndexData %>% top_frac(selectionProp)

    selectedIndexGenotypes <- column_to_rownames(selectedIndexGenotypes, var = 'V1')

    if (!is.null(selectedIndexGenotypes)) {
        clusterData <- rownames_to_column(clusterData, var = "germplasmName")
        clusterData <- clusterData %>%
            filter(germplasmName %in% rownames(selectedIndexGenotypes))

        clusterData <- column_to_rownames(clusterData, var = 'germplasmName')
    }
}

kMeansOut   <- kmeansruns(clusterData, runs=10)
kCenters    <- kMeansOut$bestk
reportNotes <- paste0(reportNotes, 'Recommended number of clusters (k) for this data set is: ', kCenters, "\n")

if (!is.na(userKNumbers)) {
    if (userKNumbers != 0) {
        kCenters <- as.integer(userKNumbers)
        reportNotes <- paste0(reportNotes, 'However, Clustering was based on ', userKNumbers, "\n")
    }
}

kMeansOut        <- kmeans(clusterData, centers = kCenters, nstart = 10)
kClusters        <- data.frame(kMeansOut$cluster)
kClusters        <- rownames_to_column(kClusters)
names(kClusters) <- c('germplasmName', 'Cluster')

if (!is.null(clusterDataNotScaled)) {
    clusterDataNotScaled <- rownames_to_column(clusterDataNotScaled,
                                               var="germplasmName")

    clusteredData <- inner_join(kClusters, clusterDataNotScaled,
                                by="germplasmName")
} else if (!is.null(selectedIndexGenotypes)) {
    selectedIndexGenotypes <- rownames_to_column(selectedIndexGenotypes,
                                                 var="germplasmName")
    clusteredData <- inner_join(kClusters, selectedIndexGenotypes,
                                by="germplasmName")
} else {
    clusteredData <- kClusters
}

clusteredData <- clusteredData %>%
    mutate_if(is.numeric, funs(round(., 2))) %>%
    arrange(Cluster)

png(plotKmeansFile)
autoplot(kMeansOut, data = clusterData, frame = TRUE,  x = 1, y = 2)
dev.off()

cat(reportNotes, file = reportFile, sep = "\n", append = TRUE)

if (length(genoFiles) > 1) {
    fwrite(genoData,
       file      = combinedDataFile,
       sep       = "\t",
       row.names = TRUE,
       quote     = FALSE,
       )
}

if (length(kResultFile) != 0 ) {
    fwrite(clusteredData,
       file      = kResultFile,
       sep       = "\t",
       row.names = FALSE,
       quote     = FALSE,
       )
}

####
q(save = "no", runLast = FALSE)
####


# cd scripts/
# samtools sort -t CB -@ 20 ../../input/atac_pbmc_5k_nextgem_possorted_bam.bam > ../../input/atac_pbmc_5k_nextgem_possorted_bam.sorted.bam

# python split_bam_file.py

# ./bam_preprocess.sh

library(cicero)
library(data.table)
library(Matrix)
library(proxy)
library(reshape2)
library(BuenColors)
library(umap)

path = '../tmp/count_reads_peaks_output/'
files <- list.files(path,pattern = "\\.txt$")
length(files)

#assuming tab separated values with a header
datalist = lapply(files, function(x)fread(paste0(path,x))$V4)
#assuming the same header/columns for all files
datafr = do.call("cbind", datalist)

df_regions = read.csv("../input/atac_pbmc_5k_nextgem_peaks.bed",
                      sep = '\t',header=FALSE,stringsAsFactors=FALSE)

peaknames = paste(df_regions$V1,df_regions$V2,df_regions$V3,sep = "_")

colnames(datafr) = sapply(strsplit(files,'\\_'),'[', 1)
rownames(datafr) = peaknames

metadata <- read.table('../input/metadata.tsv',
                         header = TRUE,
                         stringsAsFactors=FALSE,quote="",row.names=1)

mat_sparse = as(datafr, "dgTMatrix")
cicero_data = data.frame(cbind(Peak=rownames(datafr)[mat_sparse@i+1],
                           Cell=colnames(datafr)[mat_sparse@j+1],
                           Count=mat_sparse@x),stringsAsFactors = FALSE)
cicero_data$Count = as.numeric(cicero_data$Count)

input_cds <- make_atac_cds(cicero_data, binarize = TRUE)

pData(input_cds)$label = metadata[rownames(pData(input_cds)),'label']

#Ensure there are no peaks included with zero reads
input_cds <- input_cds[Matrix::rowSums(exprs(input_cds)) != 0,]

set.seed(2019)
input_cds <- detectGenes(input_cds)

input_cds <- estimateSizeFactors(input_cds)

input_cds <- reduceDimension(input_cds, max_components = 2, num_dim=15,
                        reduction_method = 'tSNE', norm_method = "none")

tsne_coords <- t(reducedDimA(input_cds))
data("human.hg19.genome")
genome_ref = human.hg19.genome
file_tss='../input/hg19/hg19-tss.bed'
row.names(tsne_coords) <- row.names(pData(input_cds))

cicero_cds <- make_cicero_cds(input_cds, reduced_coordinates = tsne_coords)

conns <- run_cicero(cicero_cds, genome_ref) # Takes a few minutes to run

saveRDS(conns,file="../output/conns_10xpbmc.rds")


two_plots <- function(perf1, perf2, probs, outf1, outf2){
  HLIN  = 61
  perf1 = read.table(perf1, header = T, sep=',')
  d1 = perf1$Delay
  f1 = perf1$CorrectFA
  t1 = perf1$CorrectTP
  perf2 = read.table(perf2, header = T, sep=',')
  d2 = perf2$Delay
  f2 = perf2$CorrectFA
  t2 = perf2$CorrectTP
  maxdelay = 300
  #probs = read.table(probs)$V1
  pdf(outf1)
  ######################## 2nd plot
  # EROR RATE 0.2
  par(mar = c(5, 5, 4, 2) + 0.1)
  plot(d2, t2, ylim = c(0.3, 1.0), xlim =c(10, 150), lwd=3, col = "black", type = "l", cex = 1.,
       xlab = "Delay", ylab = "Fraction", cex.lab = 2.0, cex.axis = 2)

  lines(d2, f2, lwd = 1, type='l', lty=1, ylim = c(0,1), cex = 0.7, col ="grey")
  points(d2, f2, cex = 1, col = "black", pch = 16)
  text(120, 0.9, "FA", cex = 2); text(120, 0.49, "TP", cex = 2)
  text(30, 0.4, expression(paste(lambda, " = 0.2")), cex = 2)
  abline(v = HLIN, lwd = 2)
  dev.off()
  pdf(outf2)
  ######################### 3rd plot
  par(mar = c(5, 5, 4, 2) + 0.1)
  plot(d1, t1, ylim = c(0.3, 1.0), xlim =c(10, 150), lwd=3, col = "black", type = "l", cex = 0.7,
       xlab = "Delay", ylab = "Fraction", cex.lab = 2.0, cex.axis = 2)
  lines(d1, f1, lwd = 1, type='l', lty=1, ylim = c(0,1), cex = 0.7, col = "grey")
  points(d1, f1, cex = 1, col = "black", pch = 16)
  text(30, 0.4, expression(paste(lambda, " = 0.05")), cex = 2)
  text(120, 0.5, "FA", cex = 2); text(120, 0.88, "TP", cex = 2)
  dev.off()
}
#two_plots("D:/_tmp/20150402/sim2/Thr0.05Er0.5Mu10.0Std0.7.txt",
#            "D:/_tmp/20150402/sim2/Thr0.2Er2Mu10.0Std0.7.txt",
#            "D:/_tmp/20150402/sim2/probs.txt",
#          "D:/_tmp/20150402/performance1a.pdf",
#          "D:/_tmp/20150402/performance2a.pdf" )
#Thr0.2Er0.5Mu10.0Std0.7.txt
#Thr0.2Er2Mu10.0Std0.7.txt
two_plots("~/GITLOCAL/recurrency-code/ecml-icdm-15/dat/Simulations/20150402/sim2/Thr0.05Er0.5Mu10.0Std0.7.txt",
          "~/GITLOCAL/recurrency-code/ecml-icdm-15/dat/Simulations/20150402/sim2/Thr0.2Er2Mu10.0Std0.7.txt",
          "~/_tmp/p/probs.txt",
          "~/_tmp/p/performance1sdm.pdf",
          "~/_tmp/p/performance2sdm.pdf" )

conff1 <- function(perf1, perf2, probs, outf){
  HLIN  = 80 #61
  perf1 = read.table(perf1, header = T, sep=',')
  d1 = perf1$Delay
  f1 = perf1$CorrectFA
  t1 = perf1$CorrectTP
  perf2 = read.table(perf2, header = T, sep=',')
  d2 = perf2$Delay
  f2 = perf2$CorrectFA
  t2 = perf2$CorrectTP
  maxdelay = 300
  probs = read.table(probs)$V1
  pdf(outf)
  ##################### 1st plot
  par(mar=c(4.0, 4.5, 2.0, 0.1))
  plot(probs, type='l', xlim = c(0, maxdelay + 50),
       ylim = c(-0.01, 0.5),
       xlab = "Time", main = "PCCF function", ylab = "Probability",
       cex.main = 2, cex.lab = 1.9, cex.axis = 1.5)

  lines(c(rep(NA,200),probs), lty = 2, lwd = 1.0)
  abline(h = 0.1, lwd = 2) # limit
  abline(v = HLIN, lwd = 2)
  lambda1 = 0.5 / 10; lambda2 = 2 / 10
  abline(h = c(lambda1, lambda2), lty = 2, lwd = 1.5)
  text(150, 0.25, expression(paste(lambda, " = 0.2")), cex = 2.0)
  text(150, 0.01, expression(paste(lambda, " = 0.05")), cex = 2.0)
  text(-3, 0.14, "L", cex = 2)
  text(93, 0.23, "D", cex = 2)
  #mtext(side = 1, text = "Time", line = 2)
  #mtext(side = 2, text = "Probability", line = 3.0)
  #mtext(side = 3, text = "Confidence function", line = 0.3)
  dev.off()
}
#conff1("D:/_tmp/20150402/sim2/Thr0.05Er0.5Mu10.0Std0.7.txt",
#          "D:/_tmp/20150402/sim2/Thr0.2Er2Mu10.0Std0.7.txt",
#          "D:/_tmp/20150402/sim2/probs.txt", "D:/_tmp/20150402/PCCF1.pdf")


pdf_file<-"pdf/maps_shp_osm.pdf"
cairo_pdf(bg="grey98", pdf_file,width=12,height=6)

library(maptools)
par(mfcol=c(1,2))
x <- readShapeSpatial("myData/london/greater_london_const_region.shp")
plot(x, axes=TRUE)
y <- readShapeSpatial("myData/london/london.osm-amenities.shp")
plot(y, axes=TRUE, pch=1, col=rgb(100,100,100,60,maxColorValue=255))
dev.off()


# plot mse through epoch
# train mse based on the first value among all the mse for trianing set
# test mse based on average value
rm(list=ls())
options(warn=1)
options(stringsAsFactors=FALSE)
options(digits=15)
require(stringr)
require(magrittr)
# require(R.matlab)
require(ggplot2)
require(reshape2)
# require(xml2)
# require(cowplot)
# require(scales)
# require(lhs)
# require(foreach)
# require(doMC)
# require(readr)
# require(abind)
projectpath="./"
infortab=read.table(file=paste0(projectpath,"submitlist.tab"),sep="\t",header=TRUE)
namevec=infortab[,"names"]
dirres=paste0(projectpath,"result/")
dirlist=1:1#c(1,2,3,4,8)#1:9#1:6
mselist=list(epoch=c(),train=c(),test=c(),names=c())
for(dir in dirlist){
  locdir=paste0(dirres,dir,"/")
  files=list.files(locdir)
  filesoutput=files[str_which(string=files,pattern="testmodel\\.\\d+\\.out")]##might use testmodel\\..+\\.out for different filename (user should ensure)
  lines=readLines(paste0(locdir,filesoutput))
  train_ind=str_which(string=lines,pattern="Train Epoch:")
  test_ind=str_which(string=lines,pattern="Test set:")
  epoch_ind=sapply(seq(length(test_ind)),function(xi){
    if(xi>1){
      range=c(test_ind[xi-1],test_ind[xi])
    }else{
      range=c(0,test_ind[xi])
    }
    train_ind[train_ind<range[2]&train_ind>range[1]][1]
  })
  # trainrecord_ind=epoch_ind[seq(length(epoch_ind))%%2==0]
  lines[epoch_ind] %>% str_extract(string=.,pattern="Train Epoch:\\s+\\w+") %>%
              str_replace_all(string=,pattern="Train Epoch:\\s+",replacement="") %>%
              as.numeric(.) -> epoch_num
  lines[epoch_ind] %>% str_extract(string=.,pattern="Loss\\(per sample\\):\\s+[\\w\\.]+") %>%
              str_replace_all(string=,pattern="Loss\\(per sample\\):\\s+",replacement="") %>%
              as.numeric(.) -> losstrain
  lines[test_ind] %>% str_extract(string=.,pattern="Average loss \\(per sample\\):\\s+[\\w\\.]+") %>%
              str_replace_all(string=,pattern="Average loss \\(per sample\\):\\s+",replacement="") %>%
              as.numeric(.) -> losstest
  mselist[["epoch"]]=c(mselist[["epoch"]],epoch_num)
  mselist[["train"]]=c(mselist[["train"]],losstrain)
  mselist[["test"]]=c(mselist[["test"]],losstest)
  mselist[["names"]]=c(mselist[["names"]],rep(namevec[dir],times=length(epoch_num)))
}
msetab=as.data.frame(mselist)
colnames(msetab)=c("epoch","train","test","names")
msetablong=melt(msetab,id=c("epoch","names"))
msetablong[,"names"]=as.factor(msetablong[,"names"])
# msetablong[,"value"]=log10(msetablong[,"value"])
p<-ggplot(data=msetablong,aes(epoch,value,linetype=variable,colour=names))+
      geom_line(alpha=0.5)+
      scale_y_continuous(trans='log10',limits=c(0.01,max(msetablong[,"value"])))+
      xlab("epoch")+
      ylab("mse")+
      theme_bw()
ggsave(plot=p,file=paste0(dirres,"mse_epoch.pdf"))
##end state(epoch) statistics on mse
endwid=10
names=unique(msetab[,"names"])
summtab=as.data.frame(matrix(NA,nrow=length(names),ncol=4))
colnames(summtab)=c("names","train_mean","test_mean","test_min")
rownames(summtab)=names
for(namegroup in names){
  subtab=msetab[msetab[,"names"]==namegroup,]
  endepoch=max(subtab[,"epoch"])
  subtab2=subtab[subtab[,"epoch"]>=(endepoch-endwid),]
  summtab[namegroup,"names"]=namegroup
  summtab[namegroup,"train_mean"]=mean(subtab2[,"train"])
  summtab[namegroup,"test_mean"]=mean(subtab2[,"test"])
  summtab[namegroup,"test_min"]=mean(sort(subtab[,"test"])[endwid])
}
save(summtab,msetablong,file=paste0(dirres,"Rplot_store.RData"))#,p


#' Custom functions for dismo prediction
#'
#'
#' A collection of functions that enable the usage of mlr model predictions with dismo.
#' @param model MLR trained model.
#' @param data A dataframe containing occurrence data.
#' @importFrom stats predict
#'
#' @return A vector with predictions.
customPredictFun <- function(model, data) {
    v <- predict(model, data, type = "prob")
    v <- as.data.frame(v)
    colnames(v) <- c("absence", "presence")
    return(v$presence)
}

customPredictFunLogreg <- function(model, data) {
    v <- predict(model, data, type = "response")
    return(v)
}

customPredictFunGBM <- function(model, data) {
    v <- predict(model, data, type = "response", n.trees = model$n.trees)
    # scale results
    v <-  (v - min(v))/(max(v) - min(v))
    return(1 - v)
}

customPredictFunMultinom <- function(model, data) {
    v <- predict(model, data, type = "probs")
    return(v)
}

customPredictFunNB <- function(model, data) {
    v <- predict(model, data, type = "raw")
    v <- as.data.frame(v)
    colnames(v) <- c("absence", "presence")
    return(v$presence)
}

customPredictFunXGB <- function(model, data) {
    data <- data.matrix(data)
    v <- predict(model, data)
    return(v)
}

customPredictFunKSVM <- function(model, data) {
    v <- raster::predict(model, data, type = "prob")
    v <- as.data.frame(v)
    colnames(v) <- c("absence", "presence")
    return(v$presence)
}


  mean.weights.by.category = function(p, redo=F) {

    outfilename = file.path( p$annual.results, "crab.sizes.rdata" )

    if (!redo) {
      load( outfilename )
      return( sizes )
    }

    set = snowcrab.db("set.biologicals")
    set = set[,c("trip", "set", "yr", "lon", "lat", "sa")]

    det = snowcrab.db("det.initial")
    det = det[,c("trip", "set", "sex", "cw", "mass", "abdomen", "chela", "mat", "shell", "gonad", "eggcol", "eggPr", "durometer" )]
    det = merge(det, set, by=c("trip","set"), all.x=T, all.y=F, sort=F)
    rm(set)

    p$regions = c(p$regions, "cfaall" ) # add a global category as well
    p$nregions = length(p$regions)

    results = c("mass.mean", "mass.sd", "cw.mean", "cw.sd", "n")
    nresults = length(results)

    sizes = array(data=NA, dim=c(p$nnodes,nresults,p$nfisheryyears,p$nregions),
      dimnames=list(p$nodes, results, p$fisheryyears, p$regions)  )

    det.years = det$yr
    det.locs = det[,c("lon","lat")]

    for (region in p$regions) {
    for (y in p$fisheryyears) {
    for (iv in 1:length(p$nodes)) {

      node = p$nodes[iv]
      nodelabel = p$nodelabels[iv]
      yc = as.character(y)
      ic = filter.class( det, type=nodelabel )
      ir = polygon_inside(det.locs, region=region)
      iy = which( det.years==y )
      i = sort( intersect( intersect(  ir, iy) , ic )  )
      if (length(i)<1) next
      sizes[ node, "mass.mean", yc, region ] = weighted.mean(x=det$mass[i], w=det$sa[i], na.rm=T)
      sizes[ node, "mass.sd", yc, region ] = sd(x=det$mass[i], na.rm=T)
      sizes[ node, "cw.mean", yc, region ] = weighted.mean(x=det$cw[i], w=det$sa[i], na.rm=T)
      sizes[ node, "cw.sd", yc, region ] = sd(x=det$cw[i], na.rm=T)
      sizes[ node, "n", yc, region ] =  length(i)

    }}}

    # fill in from each area
    for (region in p$regions) {
    for (iv in 1:length(p$nodes)) {
      node = p$nodes[iv]
      for (v in results ) {
        i = which(!is.finite( sizes[ node, v, , region ] ))
        sizes[ node, v, i,region ] = apply(sizes[ , v, , region ], 1, mean,na.rm=T) [iv]
      }
    }}

    # fill in using global means where possible
    for (region in p$regions) {
    for (iv in 1:length(p$nodes)) {
      node = p$nodes[iv]
      for (v in results ) {
        i = which(!is.finite( sizes[ node, v, , region ] ))
        sizes[ node, v, i,region ] = apply(sizes[ , v, , 4 ], 1, mean,na.rm=T) [iv]
      }
    }}

    save (sizes, file=outfilename, compress=T)
    return (sizes)
  }


library(R.utils)
library(logging)
library("XML")
library("methods")
library(ggplot2)
library(reshape2)
library (scales)

logReset();
basicConfig('DEBUG');
addHandler(writeToConsole);
addHandler(writeToFile, file=paste("output-",System$getHostname(),'-',Sys.Date(),".log",sep=''), level='DEBUG');
save_to_file = TRUE
#------------------------------------------------------------------#
sargs <- commandArgs(TRUE)
path <- args[1]
athlete <- args[2]
#------------------------------------------------------------------#
files <- list.files(paste(path,'/data/',athlete,sep=''), all.files = TRUE, pattern = ".tsv")

types = c("jo_", "j_", "jo100_1", "jo100_2", "jo300_1", "jo300_2", "jo500_1", "jo500_2", "js_")
jump_list <- list()
for(i in types)
{
  for (j in files)
  {
    if((length(grep(i, j)) != 0)&&(length(grep('_f_', j)) == 0)&&(length(grep('_a', j)) == 0))
    {
      key = i
      value <- gsub(".tsv", "", j)
      jump_list[[ key ]] <- value
    }
  }
}

pos_source <- read.table(paste(path,'/data/',athlete,'/', jump_list["j_"], '_model_BodyKinematics_pos_global.sto',sep=''), skip=18, header = TRUE)
COM_1 = mean(pos_source$center_of_mass_Y[seq(1:20)] )

#------------------------------------------------------------------#
model_xml_file = ""
for (jump in jump_list)
{
  experiment = jump
  #------------------------------------------------------------------#
  model_xml_file = paste(path,'/data/',athlete,'/', experiment, '_gait_with_arms_scaled.osim',sep='')
  #------------------------------------------------------------------#
  logdebug('loading antropometry...');
  antropometry = scan(file = paste(path,'/data/',athlete,'/antrop.txt',sep=''), what = character())
  sex <- antropometry[2]
  weight <- as.numeric(antropometry[4])
  height <- as.numeric(antropometry[6])
  #------------------------------------------------------------------#
  logdebug('loading source data files (pos, vel, acc, body_forces, inverse_dynamics, mot)...');
  pos_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_model_BodyKinematics_pos_global.sto',sep=''), skip=18, header = TRUE)
  vel_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_model_BodyKinematics_vel_global.sto',sep=''), skip=18, header = TRUE)
  acc_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_model_BodyKinematics_acc_global.sto',sep=''), skip=18, header = TRUE)
  body_forces_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_body_forces_at_joints.sto',sep=''), skip=6, header = TRUE)
  inverse_dynamics_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_inverse_dynamics.sto',sep=''), skip=6, header = TRUE)
  mot_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '.mot',sep=''), skip=10, header = TRUE)
  ext_forces_source <- read.table(paste(path,'/data/',athlete,'/', experiment, '_ext_forces.mot',sep=''), skip=10, header = TRUE)
  frames <- length(mot_source$time)
  logdebug(paste('loaded', frames, 'frames'))
  #------------------------------------------------------------------#
  logdebug('determining the phases of the jump...')
  center_of_mass_Y_delta = c(pos_source$center_of_mass_Y, pos_source$center_of_mass_Y[frames]) - c(pos_source$center_of_mass_Y[1], pos_source$center_of_mass_Y)
  center_of_mass_Y_delta = center_of_mass_Y_delta[2:(frames-1)]
  center_of_mass_X_delta = c(pos_source$center_of_mass_X, pos_source$center_of_mass_X[frames]) - c(pos_source$center_of_mass_X[1], pos_source$center_of_mass_X)
  center_of_mass_X_delta = center_of_mass_X_delta[2:(frames-1)]
  center_of_mass_Z_delta = c(pos_source$center_of_mass_Z, pos_source$center_of_mass_Z[frames]) - c(pos_source$center_of_mass_Z[1], pos_source$center_of_mass_Z)
  center_of_mass_Z_delta = center_of_mass_Z_delta[2:(frames-1)]
  #------------------------------------------------------------------#
  logdebug('detecting ground release point...')
  combined_vertical_force = ext_forces_source$ground_force_vy + ext_forces_source$ground_force_vy
  zero_force = combined_vertical_force[length(combined_vertical_force)]
  force_tolerance = 0.1
  force_max_time = min(ext_forces_source$time[combined_vertical_force>=max(combined_vertical_force)])
  force_max_frame = frames-sum(mot_source$time>=force_max_time)-2
  fitting_times = ext_forces_source$time[(combined_vertical_force<=zero_force*(1+ force_tolerance))]
  eject_time = min(fitting_times[fitting_times >= force_max_time])
  eject_frame = frames-sum(mot_source$time>=eject_time)-2
  logdebug(paste("calculated platform eject time (sec): ", eject_time))
  logdebug(paste("calculated platform eject frame: ", eject_frame))
  #------------------------------------------------------------------#
  #SMOTHING
  center_of_mass_Y_delta_smoothed = smooth.spline(center_of_mass_Y_delta, spar=0.4)$y
  center_of_mass_X_delta_smoothed = smooth.spline(center_of_mass_X_delta, spar=0.4)$y
  center_of_mass_Z_delta_smoothed = smooth.spline(center_of_mass_Z_delta, spar=0.4)$y
  center_of_mass_Y_smoothed = smooth.spline(pos_source$center_of_mass_Y, spar=0.4)$y
  #------------------------------------------------------------------#
  logdebug('detecting movement point (COM goes down)...')
  ext_frames = length(ext_forces_source$time)
  movement_tolerance = 1e-3
  movement_force = 50
  firstframe = 3
  frames_array = seq(firstframe, ext_frames)
  if((length(grep("jump_jo_", experiment)) == 0)&&(length(grep("jump_j_", experiment)) == 0))
  firstframe = min(frames_array[abs(ext_forces_source$ground_force_vy)> movement_force])
  frames_array = seq(firstframe, ext_frames)
  counter = 1
  for(delta_move in center_of_mass_Y_delta_smoothed)
  {
    if(abs(delta_move)>movement_tolerance)
      break
    counter = counter + 1
  }
  movement_frame = counter
  if((length(grep("jump_jo_", experiment)) == 0)&&(length(grep("jump_j_", experiment)) == 0)&&(length(grep("jump_js_", experiment)) == 0))
  {movement_frame = firstframe
    movement_time = ext_forces_source$time[movement_frame]
    c=0
    for(i in pos_source$time)
    {
      if(i >= movement_time)
        break
      c=c+1
    }
    movement_frame = c
  }
  movement_time = pos_source$time[movement_frame]
  logdebug(paste("calculated movement start time (sec): ", movement_time))
  logdebug(paste("calculated movement start frame: ", movement_frame))
  #------------------------------------------------------------------#
  logdebug('detecting lowest COM point...')
  lowest_time = min(pos_source$time[center_of_mass_Y_smoothed <= min(center_of_mass_Y_smoothed)])
  lowest_frame = min(seq(1, frames)[center_of_mass_Y_smoothed <= min(center_of_mass_Y_smoothed)])
  logdebug(paste("calculated lowest COM time (sec): ", lowest_time))
  logdebug(paste("calculated lowest COM frame: ", lowest_frame))
  #------------------------------------------------------------------#
  logdebug('detecting highest COM point...')
  highest_frame = min(seq(1,frames)[pos_source$center_of_mass_Y==max(pos_source$center_of_mass_Y[seq(firstframe, frames)])])
  highest_time = mot_source$time[highest_frame]
  logdebug(paste("calculated highest COM time (sec): ", highest_time))
  logdebug(paste("calculated highest COM frame: ", highest_frame))
  #------------------------------------------------------------------#
  logdebug('loading XML model to get segment weights')
  model_xml = xmlParse(file = model_xml_file)
  get_segment_mass <- function(segment)
  {
    #OpenSimDocument -> Model -> BodySet -> objects -> Body name = "tibia_r"
    seg = getNodeSet(model_xml, paste("//Body[@name='",segment,"']/mass",sep=''))
    value = xmlValue(seg[[1]])
    as.numeric(value)
  }
  #------------------------------------------------------------------#
  #support_time
  Separation_time = eject_time - movement_time
  Sep_amort_time = lowest_time - movement_time
  Sep_repul_time = eject_time - lowest_time
  #------------------------------------------------------------------#
  #Inertia
  mass_foot = get_segment_mass("toes_r")+ get_segment_mass("calcn_r") + get_segment_mass("talus_r")
  mass_shank = get_segment_mass("tibia_r")
  mass_thigh = get_segment_mass("femur_r")
  I_foot = ((get_segment_mass("toes_r") + get_segment_mass("calcn_r") + get_segment_mass("talus_r"))/660.49)+((get_segment_mass("toes_r") + get_segment_mass("calcn_r") + get_segment_mass("talus_r"))/4716100)
  I_shank = ((get_segment_mass("tibia_r"))/789.61)+((get_segment_mass("tibia_r"))/278784)
  I_thigh = ((get_segment_mass("femur_r"))/712.89)+((get_segment_mass("femur_r"))/291600)
  #------------------------------------------------------------------#
  #ANGLES
  logdebug('ANGLES calculating')
  angle_ankle_r = mot_source$ankle_angle_r +90
  angle_ankle_r = smooth.spline(angle_ankle_r, spar = 0.5)$y
  angle_knee_r = 180 - abs(mot_source$knee_angle_r)
  angle_knee_r = smooth.spline(angle_knee_r, spar = 0.5)$y
  angle_hip_r = 180 - abs(mot_source$hip_flexion_r)
  angle_hip_r = smooth.spline(angle_hip_r, spar = 0.5)$y

  angles_file = paste(path,'/data/',athlete,'/',experiment,'_angles.csv',sep='')
  columns = c(
    'angle_ankle_r',
    'angle_knee_r',
    'angle_hip_r'
  )
  target_array = matrix(nrow=(frames),ncol=length(columns))
  target_array[,1] = angle_ankle_r
  target_array[,2] = angle_knee_r
  target_array[,3] = angle_hip_r

  write.table(target_array, file = angles_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')
  #------------------------------------------------------------------#
  #ANGULAR_VELOCITES
  logdebug('ANGULAR_VELOCITES calculating')
  angular_velocity_ankle_source = (c(angle_ankle_r, angle_ankle_r[frames]) - c(angle_ankle_r[1], angle_ankle_r)) * 400
  angular_velocity_ankle_source = angular_velocity_ankle_source[1:(frames-1)]
  angular_velocity_knee_source =(c(angle_knee_r,angle_knee_r[frames]) - c(angle_knee_r[1], angle_knee_r)) * 400
  angular_velocity_knee_source = angular_velocity_knee_source[1:(frames-1)]
  angular_velocity_hip_source =(c(angle_hip_r,angle_hip_r[frames]) - c(angle_hip_r[1],angle_hip_r)) * 400
  angular_velocity_hip_source = angular_velocity_hip_source[1:(frames-1)]

  angular_velocity_file = paste(path,'/data/',athlete,'/',experiment,'_angular_velocity.csv',sep='')
  columns = c(
    'angular_velocity_ankle',
    'angular_velocity_knee',
    'angular_velocity_hip'
  )
  target_array = matrix(nrow=(frames-1),ncol=length(columns))
  target_array[,1] = angular_velocity_ankle_source
  target_array[,2] = angular_velocity_knee_source
  target_array[,3] = angular_velocity_hip_source

  write.table(target_array, file = angular_velocity_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')

  #------------------------------------------------------------------#
  #time_source
  time_source <- inverse_dynamics_source$time
  #------------------------------------------------------------------#
  #ENERGY_Ekin
  logdebug('ENERGY calculating')

  Ekin_pelvis = get_segment_mass("pelvis") * ((vel_source$pelvis_X)^2 + (vel_source$pelvis_Y)^2 + (vel_source$pelvis_Z)^2)*0.5
  Ekin_pelvis= smooth.spline(Ekin_pelvis, spar= 0.7)$y
  Ekin_femur = get_segment_mass("femur_r") * ((vel_source$femur_r_X)^2 + (vel_source$femur_r_Y)^2 + (vel_source$femur_r_Z)^2)*0.5
  Ekin_femur= smooth.spline(Ekin_femur, spar= 0.7)$y
  Ekin_tibia  = get_segment_mass("tibia_r") * ((vel_source$tibia_r_X)^2 + (vel_source$tibia_r_Y)^2 + (vel_source$tibia_r_Z)^2)*0.5
  Ekin_tibia= smooth.spline(Ekin_tibia, spar= 0.7)$y
  Ekin_talus  = get_segment_mass("talus_r") * ((vel_source$talus_r_X)^2 + (vel_source$talus_r_Y)^2 + (vel_source$talus_r_Z)^2)*0.5
  Ekin_talus= smooth.spline(Ekin_talus, spar= 0.7)$y
  Ekin_calcn  = get_segment_mass("calcn_r") * ((vel_source$calcn_r_X)^2 + (vel_source$calcn_r_Y)^2 + (vel_source$calcn_r_Z)^2)*0.5
  Ekin_calcn= smooth.spline(Ekin_calcn, spar= 0.7)$y
  Ekin_toes  = get_segment_mass("toes_r") * ((vel_source$toes_r_X)^2 + (vel_source$toes_r_Y)^2 + (vel_source$toes_r_Z)^2)*0.5
  Ekin_toes= smooth.spline(Ekin_toes, spar= 0.7)$y
  Ekin_torso  = get_segment_mass("torso") * ((vel_source$torso_X)^2 + (vel_source$torso_Y)^2 + (vel_source$torso_Z)^2)*0.5
  Ekin_torso= smooth.spline(Ekin_torso, spar= 0.7)$y
  Ekin_humerus  = get_segment_mass("humerus_r") * ((vel_source$humerus_r_X)^2 + (vel_source$humerus_r_Y)^2 + (vel_source$humerus_r_Z)^2)*0.5
  Ekin_humerus= smooth.spline(Ekin_humerus, spar= 0.7)$y
  Ekin_ulna  = get_segment_mass("ulna_r") * ((vel_source$ulna_r_X)^2 + (vel_source$ulna_r_Y)^2 + (vel_source$ulna_r_Z)^2)*0.5
  Ekin_ulna= smooth.spline(Ekin_ulna, spar= 0.7)$y
  Ekin_radius  = get_segment_mass("radius_r") * ((vel_source$radius_r_X)^2 + (vel_source$radius_r_Y)^2 + (vel_source$radius_r_Z)^2)*0.5
  Ekin_radius= smooth.spline(Ekin_radius, spar= 0.7)$y
  Ekin_hand  = get_segment_mass("hand_r") * ((vel_source$hand_r_X)^2 + (vel_source$hand_r_Y)^2 + (vel_source$hand_r_Z)^2)*0.5
  Ekin_hand= smooth.spline(Ekin_hand, spar= 0.7)$y

  Ekin1 = Ekin_pelvis + 2*Ekin_femur + 2*Ekin_tibia + 2*Ekin_talus + 2*Ekin_calcn + 2*Ekin_toes +2*Ekin_humerus + 2*Ekin_ulna +2*Ekin_radius +2*Ekin_hand
  Ekin1 = Ekin1[2:frames]

  Ekin_foot_right = I_foot * (((angular_velocity_ankle_source)*3.14/180)^2)*0.5
  Ekin_shank_right = I_shank * (((angular_velocity_knee_source)*3.14/180)^2)*0.5
  Ekin_thigh_right = I_thigh * (((angular_velocity_hip_source)*3.14/180)^2)*0.5
  Ekin2 = Ekin_foot_right + Ekin_shank_right + Ekin_thigh_right
  Ekin = Ekin1 + (2*Ekin2)
  Ekin_table = Ekin[seq(movement_frame, highest_frame)]
  Ekin_amort_start = Ekin [1]
  Ekin_amort_finish = Ekin [lowest_frame]
  Ekin_amort = Ekin_amort_finish - Ekin_amort_start
  Ekin_amort = round(Ekin_amort,2)
  #------------------------------------------------------------------#
  #EPOT
  Epot_pelvis = get_segment_mass("pelvis") *9.81 * (pos_source$pelvis_Y)^2
  Epot_femur = get_segment_mass("femur_r") * 9.81 * (pos_source$femur_r_Y)^2
  Epot_tibia  = get_segment_mass("tibia_r") * 9.81 *  (pos_source$tibia_r_Y)^2
  Epot_talus  = get_segment_mass("talus_r") * 9.81 *  (pos_source$talus_r_Y)^2
  Epot_calcn  = get_segment_mass("calcn_r") *9.81 *  (pos_source$calcn_r_Y)^2
  Epot_toes  = get_segment_mass("toes_r") * 9.81 * (pos_source$toes_r_Y)^2
  Epot_torso  = get_segment_mass("torso") * 9.81 * (pos_source$torso_Z)^2
  Epot_humerus  = get_segment_mass("humerus_r") * 9.81* (pos_source$humerus_r_Y)^2
  Epot_ulna  = get_segment_mass("ulna_r") * 9.81 * (pos_source$ulna_r_Y)^2
  Epot_radius  = get_segment_mass("radius_r") * 9.81 * (pos_source$radius_r_Y)^2
  Epot_hand  = get_segment_mass("hand_r") * 9.81 * (pos_source$hand_r_Y)^2

  Epot = Epot_pelvis + 2*Epot_femur + 2*Epot_tibia + 2*Epot_talus + 2*Epot_calcn + 2*Epot_toes + 2*Epot_torso + 2*Epot_humerus + 2*Epot_ulna + 2*Epot_radius + 2*Epot_hand
  Epot_table = Epot[seq(movement_frame, highest_frame)]
  #------------------------------------------------------------------#
  #EMECH
  Emech = Ekin + Epot[seq(1, frames-1)]
  Emech_table =Emech[seq(movement_frame, highest_frame)]
  f = highest_frame - movement_frame +1
  enrgy_time = time_source[seq(movement_frame, highest_frame)]
  Emech_max = Emech [eject_frame]
  Emech_min = Emech [lowest_frame]
  Emech_SUM = Emech_max - Emech_min

  #------------------------------------------------------------------#
  energy_file = paste(path,'/data/',athlete,'/',experiment,'_energy.csv',sep='')
  columns = c(
    'time',
    'Ekin',
    'Epot',
    'Emech'

  )
  target_array = matrix(nrow=(f),ncol=length(columns))
  target_array[,1] = enrgy_time
  target_array[,2] = Ekin_table
  target_array[,3] = Epot_table
  target_array[,4] = Emech_table

  write.table(target_array, file = energy_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')

  #------------------------------------------------------------------#

  #MOMENTs
  logdebug('MOMENTs calculating')

  ankle_angle_r_moment = inverse_dynamics_source$ankle_angle_r_moment
  ankle_angle_r_moment =ankle_angle_r_moment[1:frames]
  knee_angle_r_moment = inverse_dynamics_source$knee_angle_r_moment
  knee_angle_r_moment = knee_angle_r_moment[1:frames]
  hip_flexion_r_moment = inverse_dynamics_source$hip_flexion_r_moment
  hip_flexion_r_moment = hip_flexion_r_moment[1:frames]

  Moment_ankle_right_source = (ankle_angle_r_moment)
  Moment_ankle_right_source = smooth.spline(Moment_ankle_right_source, spar = 0.7)$y
  Moment_knee_right_source = (knee_angle_r_moment)
  Moment_knee_right_source = smooth.spline(Moment_knee_right_source, spar = 0.7)$y
  Moment_hip_right_source = -(hip_flexion_r_moment)
  Moment_hip_right_source = smooth.spline(Moment_hip_right_source, spar = 0.8)$y
  Moment_right_sum_source = Moment_ankle_right_source+Moment_knee_right_source+Moment_hip_right_source

  Moment_ankle_right = Moment_ankle_right_source[seq(movement_frame, highest_frame)]
  Moment_knee_right = Moment_knee_right_source[seq(movement_frame, highest_frame)]
  Moment_hip_right = Moment_hip_right_source[seq(movement_frame, highest_frame)]
  Moment_right_sum = Moment_right_sum_source[seq(movement_frame, highest_frame)]

  time_source_moment = time_source[seq(movement_frame, highest_frame)]

  moments_file = paste(path,'/data/',athlete,'/',experiment,'_moments.csv',sep='')
  columns = c(
    'time',
    'Moment_ankle_right',
    'Moment_knee_right',
    'Moment_hip_right',
    'Moment_right_sum'
  )
  target_array = matrix(nrow=(f),ncol=length(columns))
  target_array[,1] = time_source_moment
  target_array[,2] = Moment_ankle_right
  target_array[,3] = Moment_knee_right
  target_array[,4] = Moment_hip_right
  target_array[,5] = Moment_right_sum

  write.table(target_array, file = moments_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')
  #------------------------------------------------------------------#
  #POWERs
  logdebug('POWERs calculating')

  Power_ankle_right_source = (Moment_ankle_right_source[seq(2, frames)]) * ((angular_velocity_ankle_source) *pi/180/weight)
  Power_ankle_right_source = smooth.spline(Power_ankle_right_source, spar = 0.8)$y
  Power_knee_right_source = (Moment_knee_right_source[seq(2, frames)]) * ((angular_velocity_knee_source)*pi/180/weight)
  Power_knee_right_source = smooth.spline(Power_knee_right_source, spar = 0.5)$y
  Power_hip_right_source = (Moment_hip_right_source[seq(2, frames)]) * ((angular_velocity_hip_source)*pi/180/weight)
  Power_hip_right_source = smooth.spline(Power_hip_right_source, spar = 0.5)$y
  Power_sum_source = ((Power_knee_right_source) + (Power_ankle_right_source) + (Power_hip_right_source))
  time_source_power = time_source [seq(movement_frame, highest_frame)]

  Power_ankle_right = Power_ankle_right_source[seq(movement_frame, highest_frame)]
  Power_knee_right = Power_knee_right_source[seq(movement_frame, highest_frame)]
  Power_hip_right = Power_hip_right_source[seq(movement_frame, highest_frame)]
  Power_sum = Power_sum_source[seq(movement_frame, highest_frame)]

  powers_file = paste(path,'/data/',athlete,'/',experiment,'_powers.csv',sep='')
  columns = c(
    'time',
    'Power_ankle_right',
    'Power_knee_right',
    'Power_hip_right',
    'Power_sum'
  )
  target_array = matrix(nrow=(f),ncol=length(columns))
  target_array[,1] = time_source_power
  target_array[,2] = Power_ankle_right
  target_array[,3] = Power_knee_right
  target_array[,4] = Power_hip_right
  target_array[,5] = Power_sum

  write.table(target_array, file = powers_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')
  Power_ankle_right_max= max(Power_ankle_right)
  Power_knee_right_max= max(Power_knee_right)
  Power_hip_right_max= max(Power_hip_right)
  Power_sum_max= max(Power_sum[seq(1,f-1)])
  #------------------------------------------------------------------#
  #Work

  Work_ankle = (Moment_ankle_right_source[seq(1, frames-1)])*angular_velocity_ankle_source*0.0025*pi/180
  Work_ankle = smooth.spline(Work_ankle,spar = 0.8)$y
  Work_knee = (Moment_knee_right_source[seq(1, frames-1)])*angular_velocity_knee_source*0.0025*pi/180
  Work_knee = smooth.spline(Work_knee,spar = 0.4)$y
  Work_hip = (Moment_hip_right_source[seq(1, frames-1)])*angular_velocity_hip_source*0.0025*pi/180
  Work_hip = smooth.spline(Work_hip,spar = 0.7)$y
  Work = 2*(Work_ankle+Work_knee+Work_hip)
  Work = smooth.spline(Work,spar = 0.7)$y

  Work_ankle = Work_ankle[seq(movement_frame, highest_frame)]
  Work_knee = Work_knee[seq(movement_frame, highest_frame)]
  Work_hip = Work_hip[seq(movement_frame, highest_frame)]
  Work = Work[seq(movement_frame, highest_frame)]

  time_source_work = time_source[seq(movement_frame, highest_frame)]

  W = Work [seq(lowest_frame, eject_frame)]
  W_SUM = sum (W)

  sum_file = paste(path,'/data/',athlete,'/',experiment,'_sum_file.csv',sep='')
  columns = c(
    'Energy_summ',
    'Work_summ'
  )
  target_array = matrix(nrow=(1),ncol=length(columns))
  target_array[,1] = Emech_SUM
  target_array[,2] = W_SUM

  write.table(target_array, file = sum_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')

  work_file = paste(path,'/data/',athlete,'/',experiment,'_works.csv',sep='')
  columns = c(
    'time',
    'Work_ankle',
    'Work_knee',
    'Work_hip',
    'Work'
  )
  target_array = matrix(nrow=(f),ncol=length(columns))
  target_array[,1] = time_source_work
  target_array[,2] = Work_ankle
  target_array[,3] = Work_knee
  target_array[,4] = Work_hip
  target_array[,5] = Work

  write.table(target_array, file = work_file, append = FALSE, col.names = columns, row.names=FALSE, dec=',', sep=';')
  #------------------------------------------------------------------#
  #STIFFNESS
  logdebug('STIFFNESS calculating')
  amort_com_max = max(pos_source$center_of_mass_Y[seq(1, eject_frame)])
  amort_com_min = min(pos_source$center_of_mass_Y[seq(1, eject_frame)])
  max_sum_of_left_and_right_force_Y = max(ext_forces_source$ground_force_vy + ext_forces_source$X1_ground_force_vy)
  max_avg_of_left_and_right_force_Y = max((ext_forces_source$ground_force_vy + ext_forces_source$X1_ground_force_vy)/2)
  amort_stiffness_K_sum = max_sum_of_left_and_right_force_Y / (amort_com_max - amort_com_min)
  amort_stiffness_K_avg = max_avg_of_left_and_right_force_Y / (amort_com_max - amort_com_min)

  max_moment_ankle = max(abs(Moment_ankle_right_source[seq(1, eject_frame)]))
  max_moment_knee = max(abs(Moment_knee_right_source[seq(1,  eject_frame)]))
  max_moment_hip = max(abs(Moment_hip_right_source[seq(1, eject_frame)]))
  max_angular_velociry_ankle = max(abs(angular_velocity_ankle_source[seq(1, eject_frame)]))
  max_angular_velociry_knee = max(abs(angular_velocity_knee_source[seq(1, eject_frame)]))
  max_angular_velociry_hip = max(abs(angular_velocity_hip_source[seq(1, eject_frame)]))

  Stiffness_a = max_moment_ankle/max_angular_velociry_ankle * 100
  Stiffness_k = max_moment_knee/max_angular_velociry_knee * 100
  Stiffness_h = max_moment_hip/max_angular_velociry_hip * 100

  Stiffness_sum = Stiffness_a+Stiffness_k+Stiffness_h

  Stiffness_ankle = Stiffness_a*100/Stiffness_sum
  Stiffness_knee = Stiffness_k*100/Stiffness_sum
  Stiffness_hip = Stiffness_h*100/Stiffness_sum
  #------------------------------------------------------------------#
  #START_FORCE
  P_athlet = weight * 9.8
  start_force_source <- ext_forces_source$ground_force_vy
  l = length(start_force_source)
  start_force_source = smooth.spline(start_force_source,spar = 0.7)$y
  max_start_force = max(start_force_source)
  c=0
  for(i in start_force_source)
  {
    if(i>= max_start_force)
      break
    c= c+1
  }
  start_force_source = start_force_source [seq(c, l)]
  count = 0
  for (i in start_force_source)
  {
    if(i <= P_athlet)
      break
    count = count+1
  }
  count = count + c
  start_force_time = ext_forces_source[count,1]
  phase_split_time = ext_forces_source[c,1]
  start_force_time = start_force_time - phase_split_time
  start_force = P_athlet/start_force_time*0.001
  #------------------------------------------------------------------#
  #COM

  COM_2 = pos_source$center_of_mass_Y[highest_frame]
  COM = COM_2 - COM_1
  COM = round(COM, 2)

  COM_Y <- pos_source$center_of_mass_Y
  if (save_to_file)
  {
    png_file = paste(path,'/data/',athlete,'/',experiment, '_plot_2_jump_phases.png', sep = "")
    png(png_file, width=1600, height=1000, units="px", res=106)
  }
  plot(COM_Y, ann=FALSE, type="l", col="red", lwd=2)
  lines(x=c(-10000,10000), y=c(0,0), col='black')
  title(main=paste("Jump Phases Breakdown (Using smoothed Center of Mass + Force Reactions)", sep=''))
  title(ylab="Delta (m)")
  title(xlab=paste("Frames", sep=""))
  legend("topleft", c("center_of_mass VERTICAL delta smoothed", "center_of_mass TRANSVERSAL delta smoothed","center_of_mass LONGITUDINAL delta smoothed",
                      paste("Frame ", movement_frame, " (" , movement_time, " sec): start of movement (Phase 1 begins)", sep=""), paste("Frame ", lowest_frame, " (" , lowest_time, " sec): center of mass it at lowest point (Phase 1 ends, 2 begins)",sep=''), paste("Frame ", eject_frame, " (" , eject_time, " sec): ejecting from the platform (Phase 2 ends, Phase 3 begins)", sep=''), paste("Frame ", highest_frame, " (" , highest_time, " sec): center of mass is at highest point (Phase 3 ends)", sep='')),
         col=c('red', "darkgreen","blue", "orange", "brown", "purple", "darkgreen"), lty=c(1,1,1,2,2,2,2), lwd=c(2,1,1,2,2,2,2));
  top = max(COM_Y)
  bottom = min(COM_Y)
  lines(x=c(eject_frame, eject_frame), y=c(bottom, top), lty=2, lwd=2, col="purple")
  lines(x=c(movement_frame, movement_frame), y=c(bottom, top), lty=2, lwd=2, col="orange")
  lines(x=c(lowest_frame, lowest_frame), y=c(bottom, top), lty=2, lwd=2, col="brown")
  lines(x=c(highest_frame, highest_frame), y=c(bottom, top), lty=2, lwd=2, col="darkgreen")
  if(save_to_file) dev.off();

  #------------------------------------------------------------------#
  #EFF
  Ekin_sum = mean(Ekin[seq(eject_frame+1, frames-1)])
  Epot_sum = mean(Epot[seq(1, eject_frame)])
  Eff = (Ekin_sum)/(Epot_sum) *100
  #------------------------------------------------------------------#
  #TABLE
  summary_file = paste(path,'/data/',athlete,'/',experiment,'_table.csv',sep='')
  target_array = matrix(nrow=2,ncol=15)
  target_array[1,1] = 'COM'
  target_array[1,2] = 'Separation_time'
  target_array[1,3] = 'Amort_time'
  target_array[1,4] = 'Push_off_time'
  target_array[1,5] = 'Effectiv'
  target_array[1,6] = 'Stiffness'
  target_array[1,7] = 'start_force'
  target_array[1,8] = 'Stiffness_ankle'
  target_array[1,9] = 'Stiffness_knee'
  target_array[1,10] = 'Stiffness_hip'
  target_array[1,11] = 'Power_ankle_right_max'
  target_array[1,12] = 'Power_knee_right_max'
  target_array[1,13] = 'Power_hip_right_max'
  target_array[1,14] = 'Power_sum_max'
  target_array[1,15] = 'Ekin_amort'

  target_array[2,1] = COM
  target_array[2,2] = Separation_time
  target_array[2,3] = Sep_amort_time
  target_array[2,4] = Sep_repul_time
  target_array[2,5] = Eff
  target_array[2,6] = amort_stiffness_K_avg
  target_array[2,7] = start_force
  target_array[2,8] = Stiffness_ankle
  target_array[2,9] = Stiffness_knee
  target_array[2,10] = Stiffness_hip
  target_array[2,11] = Power_ankle_right_max
  target_array[2,12] = Power_knee_right_max
  target_array[2,13] = Power_hip_right_max
  target_array[2,14] = Power_sum_max
  target_array[2,15] = Ekin_amort

  write.table(target_array, file = summary_file, append = FALSE, col.names = FALSE, row.names=FALSE, dec=',', sep=';')

}

#------------------------------------------------------------------#
#GRAPHICS

tables <- list.files(paste(path,'/data/',athlete, sep=''), all.files = TRUE, pattern = ".csv")
types = c("jo_", "j_", "jo100_1", "jo100_2", "jo300_1", "jo300_2", "jo500_1", "jo500_2", "js_")
table_list <- list()
for(i in types)
{
  for (j in tables)
  {
    if((length(grep(i, j)) != 0)&&(length(grep('_table', j)) != 0))
    {
      key = i
      value <- gsub(".csv", "", j)
      table_list[[ key ]] <- value
    }
  }
}
#------------------------------------------------------------------#
#EFFECTIVNOST
Eff <- list()
n=1
for (table in table_list)
{
  normalized_file = table
  file = paste(path,"/data/",athlete,"/",normalized_file,".csv", sep = '')
  Eff_jump_source <- read.csv(file=file, sep=";")
  Eff_jump = Eff_jump_source$Eff
  Eff[n] <- Eff_jump
  n = n+1
}
x <- c(1, 2, 3, 4, 5, 6, 7, 8, 9)
y <- c(Eff)
if (save_to_file)
{
  png_file = paste(path,'/data/',athlete,'/','Jumps_effectiv.png', sep = '')
  png(png_file, width=1600, height=1000, units="px", res=106)
}
plot (x, y, type = "l",col="red")
axis(1, at=1:8, lab=c("Jump_jo","Jump_j","Jump_jo100","Jump_j100","Jump_jo300", "Jump_j300", "Jump_jo500", "Jump_j500", "Jump_js"))
title(main=paste("Jump_effectiv", sep=''))
title(ylab="Effectiv")
title(xlab=paste("Jumps", sep=""))
if (save_to_file) dev.off();
#------------------------------------------------------------------#
#JOINT_STIFFNESS
J_Stiff_ankle <- list()
J_Stiff_knee <- list()
J_Stiff_hip <- list()
t=1
target_array = matrix(nrow=9,ncol=4)
target_array[1,1] = 'jump_jo'
target_array[2,1] = 'jump_j'
target_array[3,1] = 'jump_jo100'
target_array[4,1] = 'jump_j100'
target_array[5,1] = 'jump_jo300'
target_array[6,1] = 'jump_j300'
target_array[7,1] = 'jump_jo500'
target_array[8,1] = 'jump_j500'
target_array[9,1] = 'jump_js'
for (table in table_list)
{
  normalized_file = table
  file = paste(path,"/data/",athlete,"/",normalized_file,".csv", sep = '')
  J_Stiff_source <- read.csv(file=file, sep=";")
  J_Stiffness_ankle = J_Stiff_source$Stiffness_ankle
  J_Stiffness_knee = J_Stiff_source$Stiffness_knee
  J_Stiffness_hip = J_Stiff_source$Stiffness_hip
  J_Stiff_ankle [t] <- J_Stiffness_ankle
  J_Stiff_knee [t] <- J_Stiffness_knee
  J_Stiff_hip [t] <- J_Stiffness_hip
  summary_file = paste(path,'/data/',athlete,'/', 'j_stiff.csv',sep='')
  target_array[t,2] = J_Stiff_ankle[[t]]
  target_array[t,3] = J_Stiff_knee[[t]]
  target_array[t,4] = J_Stiff_hip[[t]]
  t=t+1
}
write.table(target_array, file = summary_file, append = FALSE, col.names = FALSE, row.names=FALSE, dec=',', sep=';')

if (save_to_file)
{
  png_file = paste(path,'/data/',athlete,'/','Joint_Stiffness.png', sep = '')
  png(png_file, width=1600, height=1000, units="px", res=106)
}
Stiff_data <- read.table(paste(path,'/data/',athlete,'/','j_stiff.csv',sep=''), sep=';')
names.arg=c("jo", "j", "jo100", "j100", "jo300", "j300", "jo500", "j500")
x = c(1, 2, 3, 4, 5, 6 ,7, 8)
Ankle_Stiffness = c(Stiff_data$V2)
Knee_Stiffness = c(Stiff_data$V3)
Hip_Stiffness = c(Stiff_data$V4)
to_plot <- data.frame(x=names.arg,Hip_Stiffness=Hip_Stiffness, Knee_Stiffness=Knee_Stiffness, Ankle_Stiffness=Ankle_Stiffness)
melted<-melt(to_plot, id="x")
ggplot(melted,aes(x=x,y=value,fill=variable)) + geom_bar(stat="identity",position = "fill", alpha=.3) + scale_y_continuous(labels = percent_format())+ theme_bw()
if (save_to_file) dev.off();
#------------------------------------------------------------------#
#Report

l=2
COM_source <- list()
Separation_time_source <- list()
Sep_amort_time_source <- list()
Sep_repul_time_source <- list()
summ_stiff_source <- list()
eff_source <- list()
start_f_source <- list()
power_sum_source <- list()

target_array = matrix(nrow=10, ncol=9)
target_array[1,1] = ' '
target_array[2,1] = 'jump_jo'
target_array[3,1] = 'jump_j'
target_array[4,1] = 'jump_jo100_1'
target_array[5,1] = 'jump_jo100_2'
target_array[6,1] = 'jump_jo300_1'
target_array[7,1] = 'jump_jo300_2'
target_array[8,1] = 'jump_jo500_1'
target_array[9,1] = 'jump_jo500_2'
target_array[10,1] = 'jump_js'

target_array[1,2] = 'COM'
target_array[1,3] = 'Separation_time'
target_array[1,4] = 'Amort_time'
target_array[1,5] = 'Push_off_time'
target_array[1,6] = 'Effectiv'
target_array[1,7] = 'Stiffness'
target_array[1,8] = 'Start_force'
target_array[1,9] = 'Power_sum_max'
#target_array[1,10] = 'Ekin_amort'
#target_array[1,11] = 'KPD'

for (table in table_list)
{
  normalized_file = table
  file = paste(path,"/data/",athlete,"/",normalized_file,".csv", sep = '')
  table_source <- read.csv(file=file, sep=";")
  summ_stiff_source <- table_source$Stiffness
  eff_source <- table_source$Eff
  COM_source <- table_source$COM
  Separation_time_source <- table_source$Separation_time
  Sep_amort_time_source <- table_source$Amort_time
  Sep_repul_time_source <- table_source$Push_off_time
  start_f_source <- table_source$start_force
  power_sum_source <- table_source$Power_sum_max

  summ_stiff_source [l] <- summ_stiff_source
  eff_source [l] <- eff_source
  COM_source [l] <- COM_source
  Separation_time_source [l] <- Separation_time_source
  Sep_amort_time_source [l] <- Sep_amort_time_source
  Sep_repul_time_source [l] <- Sep_repul_time_source
  start_f_source [l] <- start_f_source
  power_sum_source [l] <- power_sum_source

  summary_file = paste(path,'/data/',athlete,'/', 'Report.csv',sep='')

  target_array[l,2] = COM_source[[l]]
  target_array[l,3] = Separation_time_source[[l]]
  target_array[l,4] = Sep_amort_time_source[[l]]
  target_array[l,5] = Sep_repul_time_source[[l]]
  target_array[l,6] = eff_source[[l]]
  target_array[l,7] = summ_stiff_source[[l]]
  target_array[l,8] = start_f_source[[l]]
  target_array[l,9] = power_sum_source[[l]]
  l=l+1
}
write.table(target_array, file = summary_file, append = FALSE, col.names = FALSE, row.names=FALSE, dec=',', sep=';')
#------------------------------------------------------------------#
df<-read.table(paste(path,'/data/',athlete,'/Report.csv',sep=''), header = TRUE, sep=";", dec=',')
df <- data.frame(df)
df$COM <- gsub("\\.", ",", df$COM)
df$Separation_time <- gsub("\\.", ",", df$Separation_time)
df$Amort_time <- gsub("\\.", ",", df$Amort_time)
df$Push_off_time <- gsub("\\.", ",", df$Push_off_time)
df$Effectiv <- gsub("\\.", ",", df$Effectiv)
df$Stiffness <- gsub("\\.", ",", df$Stiffness)
df$Start_force <- gsub("\\.", ",", df$Start_force)
df$Power_sum_max <- gsub("\\.", ",", df$Power_sum_max)
write.table(df, file = summary_file, append = FALSE, col.names = TRUE, row.names=FALSE, dec=',', sep=';')
#------------------------------------------------------------------#
my.file.rename <- function(from, to) {
  todir <- dirname(to)
  if (!isTRUE(file.info(todir)$isdir)) dir.create(todir, recursive=TRUE)
  file.rename(from = from,  to = to)}

old_way = paste(path,'/data/',athlete, sep="")

folder_files <- list.files(paste(path,'/data/',athlete,sep=''), all.files = TRUE)
types = c("_energy.csv","_moments.csv","_powers.csv","_works.csv", "Report.csv", "_sum_file.csv")
for(i in types)
{
  for (j in folder_files)
  {
    if((length(grep(i, j)) != 0))
    {
      old_way_file = paste(old_way, '/', j, sep = "")
      new_way = paste(old_way,'/', athlete, '/', j, sep = "")
      my.file.rename(from = old_way_file, to = new_way)
    }
  }
}
#------------------------------------------------------------------#


library("ggpubr")
library(tidyverse)

Final <- read.csv("C:/Users/amira/Downloads/Adolescent - Sheet1.csv")

ggplot(data = Final) +
  geom_boxplot(mapping = aes(x = Sex, y = LowDistraction1))


#===================================================================================================#

#Preprocessing

#===================================================================================================#
selDataMatrix <- lumiMethyNorm(
    fileName = "Matrix_signal_intensities.txt",
    inputtype = "signal",
    sample_names = c("Hep_D0", "Hep_D7", "Hep_D14", "Hep_D21", "Hep_D28")
)

#===================================================================================================

# correlation matrix (Fig. 1f)

#===================================================================================================
cor.exp <- cor(selDataMatrix)

hc <- hclust(dist(cor.exp))
breaks1 <- seq(0.95, 1, length = 500) #the range should be optimized
mycols1 <- colorRamp2(breaks1, colorRampPalette(brewer.pal(9, "GnBu"))(500)) # color for motif enrichment

ht1 <- Heatmap(cor.exp,
                col = mycols1,
                name = "Correlation Coefficient", #title of legend,
                row_names_gp = gpar(fontsize = 12), # Text size for row names
                column_names_gp = gpar(fontsize = 12), # Text size for column names
                column_names_rot = 90,
                heatmap_legend_param = list(title = "R",
                    title_gp = gpar(fontsize = 12),
                    labels_gp = gpar(fontsize = 6),
                    grid_height = unit(0.8, "cm"),
                    grid_width = unit(0.8, "cm"),
                    title_position = c("topcenter")
                    ),
                row_dend_width = unit(2, "cm"), #row dendrogram width
                column_dend_height = unit(2, "cm"), #column dendrogram height
                row_dend_side = "right",
                column_dend_side = "bottom",
                cluster_rows = hc,
                cluster_columns = hc,
                cell_fun = function(j, i, x, y, width, height, fill){
                    grid.rect(x = x, y = y, width = width, height = height, gp = gpar(col = "black", fill = NA))
                }
            )
ht1
pdf("Hep_correlationplot.pdf")
draw(ht1)
dev.off()

#===================================================================================================

# DMR b2b plot (Fig. 1g)

#===================================================================================================
#Count DMP
comp <- data.frame(D0_D7=c(1,2), D7_D14=c(2,3), D14_D21=c(3,4), D21_D28=c(4,5)) # comparisons

M_ndiffprobes <- sapply(comp, function(x){
    sum(selDataMatrix[,as.numeric(x[1])] - selDataMatrix[,x[2]] <= -2)
})
D_ndiffprobes <- sapply(comp, function(x){
    sum(selDataMatrix[,as.numeric(x[1])] - selDataMatrix[,x[2]] >= 2)
})

nprobes_df <- data.frame(Time=c("D0-D7", "D7-D14", "D14-D21", "D21-D28"), Methylated=M_ndiffprobes, Demethylated=D_ndiffprobes)

#plot
nprobes_b2b_df <- tidyr::pivot_longer(nprobes_df, col = -Time, , names_to = "MetDemet", values_to = "nprobes")

the_order <- rev(as.character(unlist(nprobes_b2b_df %>%
    filter (MetDemet == "Methylated") %>%
    select(Time)
)))

theme <- theme(
    panel.background = element_blank(),
    panel.border = element_blank(),
    panel.grid.major = element_blank(),
    panel.grid.minor = element_blank(),
    strip.background = element_blank(),
    plot.title = element_text(size = 10,hjust = 0.5),
    axis.text.x = element_blank(),
    axis.text.y = element_blank(),
    axis.title.y = element_blank(),
    axis.ticks = element_blank(),
    legend.key = element_blank(),
    legend.position = "none",
    plot.margin = unit(c(1,1,1,1),"line"),
    panel.grid.major.x = element_line(linetype = "dotted", size = 0.3, color = "#3A3F4A")
)

g <- ggplot(nprobes_b2b_df, aes(x=Time))
g <- g + scale_x_discrete(limits = the_order)

#Methylated plot (left)
g.M <- g + geom_bar(
    data=nprobes_b2b_df[nprobes_b2b_df[["MetDemet"]]=="Methylated",],
    aes(y=nprobes), fill=rgb(0.8, 0, 0.2, alpha=0.9),
    stat = "identity"
)
g.M <- g.M + scale_y_reverse(limits=c(max(nprobes_b2b_df[["nprobes"]])*1.2,0))
g.M <- g.M + ggtitle('Methylated')
g.M <- g.M + ylab("Number of probes")
g.M <- g.M + coord_flip()
g.M <- g.M + theme
g.M <- g.M + theme(
    axis.text.x = element_text(size = 10),
    axis.ticks.x = element_line(size=0.5),
    axis.line.x = element_line(size=0.5)
)

#Demethylated plot (Right)
g.D <- g + geom_bar(
    data=nprobes_b2b_df[nprobes_b2b_df[["MetDemet"]]=="Demethylated",],
    aes(y=nprobes), fill=rgb(0, 0.7, 0, alpha=0.9),
    stat = "identity"
)
g.D <- g.D + scale_y_continuous(limits=c(0,max(nprobes_b2b_df[["nprobes"]])*1.2))
g.D <- g.D + ggtitle('Demethylated')
g.D <- g.D + ylab("Number of probes")
g.D <- g.D + coord_flip()
g.D <- g.D + theme
g.D <- g.D + theme(
    axis.text.x = element_text(size = 10),
    axis.ticks.x = element_line(size=0.5),
    axis.line.x = element_line(size=0.5)
)

#labels (Center)
g.time <- g + geom_bar(
    data=nprobes_b2b_df[nprobes_b2b_df[["MetDemet"]]=="Demethylated",],
    aes(y=0, fill = 'white'),
    alpha=0, stat = "identity"
)
g.time <- g.time + geom_text( aes( y = 0,  label = as.character(Time)), size = 3)
g.time <- g.time + ggtitle("Time")
g.time <- g.time + coord_flip()
g.time <- g.time + ylab("")
g.time <- g.time + theme
g.time <- g.time + theme(panel.grid.major.x = element_blank())

g_all <- grid.arrange(
    g.M, g.time, g.D,
    ncol=3,
    widths=c(4.25/10,1.5/10,4.25/10)
)

ggsave(g_all, file="nDMP_b2bplot.pdf", width=7, height=7)

#===================================================================================================

# Functional analysis of DMRs (Supplementary Figure 3)

    #GO analysis using eGREAT and then summaryze using rrvgo

#===================================================================================================
COMP <- data.frame(Hep_00_07 = c(1, 2), Hep_07_14 = c(2, 3), Hep_14_21 = c(3, 4), Hep_21_28 = c(4, 5))
topn <- 5

for(i in c("Methyl, Demethyl")){
    #Construction of Entrez ID list
    enrich_reducedTerms_BP.df <- NULL
    for(j in seq(length(COMP))){
        job <- DMPrGREAT(
            data = selDataMatrix,
            out_prefix = paste0(i, "_", names(COMP)[j]),
            ControlColnum = COMP[1, j],
            TreatmentColnum = COMP[2, j],
            MethylDemethyl = i,
        )

        #extraction of enriched ensemble gene table
        tb = getEnrichmentTables(job)

        #Calculation of the similarity matrix
        simMatrix <- calculateSimMatrix(
            tb$"GO Biological Process"$ID,
            orgdb = "org.Hs.eg.db",
            ont = "BP",
            method = "Rel"
        )
        scores <- setNames(-log10(tb$"GO Biological Process"$Hyper_Adjp_BH), tb$"GO Biological Process"$ID)
        reducedTerms <- reduceSimMatrix(
            simMatrix = simMatrix,
            scores = scores,
            threshold = threshold,
            orgdb = "org.Hs.eg.db"
        )
        reducedTerms2 <- left_join(reducedTerms, tb$"GO Biological Process", by = c("go" = "ID"))

        #summarize by cluster
        ndmp <- length(DmpId(selDataMatrix, COMP[1, j], COMP[2, j], MethylDemethyl = i))

        reducedTerms2 %>%
            group_by(parentTerm) %>%
            summarise(total_Region_Hits = sum(Hyper_Foreground_Region_Hits), total_score = sum(score)) %>%
            mutate(Hit_ratio = total_Region_Hits / ndmp) %>%
            mutate(sample = names(COMP)[j]) -> parents_summry

        parents_summry_top <- parents_summry[order(parents_summry$total_score, decreasing = TRUE)[seq(topn)],]
        enrich_reducedTerms_BP.df <- rbind(enrich_reducedTerms_BP.df, parents_summry_top)
    }

    enrich_reducedTerms_BP.df$Reduced_GO <- factor(
        enrich_reducedTerms_BP.df$parentTerm,
        levels = rev(unique(enrich_reducedTerms_BP.df$parentTerm))
    )

    #plot
    g <- ggplot(data = enrich_reducedTerms_BP.df, aes(x = sample, y = Reduced_GO, color = total_score, size = Hit_ratio))
    g <- g + geom_point()
    g <- g + scale_color_gradient(low = "blue", high = "red")
    g <- g + theme_light()
    g <- g + theme(axis.text.x = element_text(size = 15, angle = 90),
        axis.text.y = element_text(size = 15),
        axis.title.x = element_blank(),
        axis.title.y = element_text(size = 15),
        legend.title = element_text(size = 15),
        legend.text = element_text(size = 15)
        )
    ggsave(g, file = paste0("rGREATrrvgo_", i, "__GO.BP_summary.pdf"), width = 14, height = 10.5)
}

#===================================================================================================

# Heatmap for methylation regulating TF (Fig. 2b, d)

#===================================================================================================
#Motif overrepresentation analysis
motifDBList <- IMAGE_PWMlist
comp <- c("Hep_00_07", "Hep_07_14", "Hep_14_21", "Hep_21_28")

for(i in c("Methyl", "Demethyl")){
    for(j in 1:4){
        outdir <- paste0("data/Methyl850_HEP/motif_analysis/", comp[j])
        dir.create(outdir, recursive = TRUE)

        MotScr(
            infile = selDataMatrix,
            motifDBList = motifDBList,
            outname = paste0(outdir, "/", ),
            ControlColnum = j,
            TreatmentColnum = j + 1,
            MethylDemethyl = i
            )
    }
}

#Heatmap
for(i in c("Methyl", "Demethyl")){
    #load the data
    data_list <- lapply(comp, function(x){
        read.table(paste0("data/Methyl850_HEP/motif_analysis/", x, "/", i, "_mot_analysis_result.txt"),
        sep = "\t",
        header = T,
        stringsAsFactors = FALSE)
    })
    names(data_list) <- comp

    #Processing of motif names
    motif_names <- sapply(sapply(data_list[[1]][, 1], function(x){ strsplit(as.character(x), "\\.") }), function(x){ x[1] })

    #P-value matrix
    pvals <- as.data.frame(lapply(data_list, function(x){ x[, 7] }))
    rownames(pvals) <- motif_names
    colnames(pvals) <- paste0(comp, "_p")

    #Conversion of pvalue = 0 to 1.00e-300
    pvals.matrix <- as.matrix(pvals)
    pvals.matrix[pvals.matrix == 0] <- 1.00e-300
    pvals <- as.data.frame(pvals.matrix)

    # "significant" matrix
    sig_matrix <- sapply(data_list, function(x){ x[, 15] })
    rownames(sig_matrix) <- motif_names
    colnames(sig_matrix) <- paste0(comp, "_sig")

    #Conbine "Significant" matrix and p-value matrix as.data frame
    pval_sig <- data.frame(motif = rownames(pvals), as.data.frame(pvals), as.data.frame(sig_matrix))

    #Selection of rows which has at least one pvalue <= 10e-50 and at least one "significant"
    pcutoff <- 1

    pval_sig %>%
        select(ends_with("_p")) %>%
        apply(., 1, min) < pcutoff -> pval_index
    pval_sig %>%
        select(ends_with("_sig")) %>%
        apply(., 1, function(x){ any(x == "significant") }) -> sig_index

    sel_pval_sig <- filter(pval_sig, pval_index & sig_index)

    #non-significant p-value to NA
    sel_pval_sig %>%
        select(ends_with("_p")) -> sel_pvals
    sel_pval_sig %>%
        select(ends_with("_sig")) ->  sel_sigs
    sel_pvals[sel_sigs != "significant" | is.na(sel_sigs)] <- "NA"

    sel_pval_sig_na <- cbind(sel_pvals, sel_sigs)

    #load CAGE expression table
    load("data/CAGE_HEP/gene_expMatrix.RData")

    E_TC_names <- c("Hep_00", "Hep_07", "Hep_14", "Hep_21", "Hep_28")
    colnames(mean.gene_expMatrix) <- paste0(E_TC_names, "_exp")
    colnames(sd.gene_expMatrix) <- paste0(E_TC_names, "_sd")
    mean_sd.gene_expMatrix <- cbind(mean.gene_expMatrix, sd.gene_expMatrix)

    #integrate motif enrichment and CAGE expression data
    gene_names <- as.vector(t(as.data.frame(strsplit(as.character(rownames(sel_pval_sig_na)), "_")))[, 1])

    #selection of mean.gene.expMatrix
    sel_mean_sd.gene_expMatrix <- t(sapply(gene_names, function(x){
        if(any(rownames(mean_sd.gene_expMatrix) %in% x)){
            mean_sd.gene_expMatrix[rownames(mean_sd.gene_expMatrix) %in% x,]
        } else{
        return(rep(NA, ncol(mean_sd.gene_expMatrix)))
        }
    }))

    sel_all_df <- cbind(sel_pval_sig_na, sel_mean_sd.gene_expMatrix)
    sel_all_df <- mutate(sel_all_df, motif = rownames(sel_all_df))

    #remove low expression motifs
    TPMcutoff <- 50

    highExp_sel_all_df <- NULL
    for(i in seq(length(comp))){
        Msig_index <- sel_all_df[, paste0(comp[i], "_sig")] == "significant"
        TPM_index <- as.vector(apply(sel_all_df[, paste0(E_TC_names[c(i, i + 1)], "_exp")], 1, max) >= TPMcutoff)
        highExp_sel_all_df <- rbind(highExp_sel_all_df, sel_all_df[which(Msig_index & TPM_index),])
    }
    highExp_sel_all_df <- highExp_sel_all_df %>% distinct(motif, .keep_all = TRUE)

    #plot heatmap
    #Extract pvals for heatmap
    highExp_sel_pvals <- select(highExp_sel_all_df, ends_with("_p"))
    rownames(highExp_sel_pvals) <- highExp_sel_all_df[, "motif"]
    colnames(highExp_sel_pvals) <- gsub("_p", "", colnames(highExp_sel_pvals))

    #log10 conversion
    log_highExp_sel_pvals <- apply(highExp_sel_pvals, c(1, 2), function(x){-log(as.numeric(x), 10) })

    #heatmap
    df1 <- log_highExp_sel_pvals
    breaks1 <- seq(0, 150, length = 500)
    mycols1 <- colorRamp2(breaks1, colorRampPalette(brewer.pal(9, "GnBu"))(500)) # color for motif enrichment
    hc.rows <- hclust(dist(as.matrix(replace(df1, is.na(df1), 0))))

    ## cluster color
    if(i == "Methyl"){
        clust_num <- 4
    } else{
        clust_num <- 5
    }

    clusters <- cutree(hc.rows, clust_num) #cluster annotation
    clust_col <- brewer.pal(clust_num, "Pastel1") #cluster colors
    names(clust_col) <- seq(clust_num)
    ha_row = rowAnnotation(df = data.frame(cluster = clusters), col = list(cluster = clust_col), width = unit(1, "cm"))

    ht1 <- Heatmap(df1,
        width = unit(6, "cm"), #With of heatmap
        name = "-10g10P", #title of legend
        column_title = "diff probe nubers and Motif enrichment",
        column_title_gp = gpar(fontsize = 10),
        row_title = NA,
        show_row_names = TRUE,
        row_names_gp = gpar(fontsize = 5), # Text size for row names
        column_names_gp = gpar(fontsize = 8), # Text size for column names
        col = mycols1,
        cluster_rows = hc.rows,
        row_dend_width = unit(4, "cm"), #row dendrogram width
        row_dend_reorder = FALSE,
        cluster_columns = FALSE,
        rect_gp = gpar(col = "white", lty = 1, lwd = 0.5),
        na_col = "grey50",
        cell_fun = function(j, i, x, y, width, height, fill){
            grid.rect(x = x, y = y, width = width, height = height, gp = gpar(col = "black", fill = NA))
        },
        right_annotation = ha_row
    )

    pdf(paste0(i , "_motif_enirch_heatmap.pdf"))
    draw(ht1)
    dev.off()

    save(list = ls(), file = paste0("data/Methyl_850_HEP/", i, "_motif_heatmap.RData"))
}
#===================================================================================================

# exprssion line plot (Fig. 2c and 2e; Supplementary Figure 4)

#===================================================================================================
for(i in c("Methyl", "Demethyl")){
    #load tmp data
    load(paste0("data/Methyl850_HEP/", i ,"_motif_heatmap.RData"))

    ## add acluster metadata
    highExp_sel_all_df2 <- cbind(highExp_sel_all_df, cluster = clusters)

    # motif cluster expression and sd
    highExp_sel_exp_sd <- select(highExp_sel_all_df2, motif, cluster, ends_with("_exp"), ends_with("_sd"))

    if(i == "Methyl"){
        cluster_bum <- 4
    } else{
        cluster_num <- 5
    }

    y.df_list <- NULL
    for(j in seq(clust_num)){
        highExp_sel_exp_sd %>%
            filter(cluster == j) %>%
            arrange(motif) -> cluster_gene_exp

        #extract gene names
        gene <- sapply(strsplit(as.character(unlist(select(cluster_gene_exp, motif))), "_"), function(x){ x[1] })

        #remove dupicated genes
        cluster_gene_exp2 <- cluster_gene_exp[which(!duplicated(gene)),]
        gene2 <- gene[which(!duplicated(gene))]

        #extract only expression data and formatting for plot
        df2 <- as.data.frame(cbind(Gene = gene2, select(cluster_gene_exp2, ends_with("_exp"))))
        colnames(df2)[-1] <- c(0, 7, 14, 21, 28)

        y <- tidyr::pivot_longer(df2, cols = -Gene, names_to = "Day", values_to = "TPM")

        #extract only sd data and formatting for plot
        df2.sd <- as.data.frame(cbind(Gene = gene2, select(cluster_gene_exp2, ends_with("_sd"))))
        colnames(df2.sd)[-1] <- c(0, 7, 14, 21, 28)
        sd.y <- tidyr::pivot_longer(df2.sd, cols = -Gene, names_to = "Day", values_to = "SD")

        #integrate TPM and SD
        y.df <- cbind(y, select(sd.y, SD))
        y.df_list <- c(y.df_list, list(y.df))
    }

        #plot
    theme <- theme(panel.background = element_blank(), # initialization
        panel.border = element_rect(fill = NA),
        panel.grid.major = element_blank(),
        panel.grid.minor = element_blank(),
        strip.background = element_blank(),
        plot.title = element_text(hjust = 0.5),
        axis.text = element_text(colour = "black", size = 9),
        axis.title = element_text(colour = "black", size = 9),
        axis.ticks = element_line(colour = "black"),
        legend.text = element_text(colour = "black", size = 9),
        legend.title = element_blank(),
        legend.key = element_rect(fill = "white"),
        legend.position = "bottom",
        plot.margin = unit(c(1, 1, 1, 1), "line")
    )

    g_list <- NULL
    for(k in seq(length(y.df_list))){
        df3 <- y.df_list[[k]]
        cluster_name <- paste0("Cluster ", k)

        ## Color setting
        max.tpm <- df3 %>% group_by(Gene) %>% summarise(max = max(TPM))
        max.tpm <- floor(max.tpm[, "max"])
        TPM_ranged_col <- colorRampPalette(brewer.pal(9, "YlGn"))(max(max.tpm))
        mycols <- sapply(max.tpm, function(x){ TPM_ranged_col[x] })

        #For legend of max TPM (color of each gene)
        dat = data.frame(max_TPM = seq(max(max.tpm)), y = 1, color = TPM_ranged_col)
        gc <- ggplot(dat, aes(max_TPM, y, colour = max_TPM)) +
                geom_point(size = 0) +
                scale_colour_gradientn(colors = TPM_ranged_col) +
                theme(legend.text = element_text(colour = "black", size = 9))

        tpm_color_legend <- get_legend(gc)

        ## ggplot2
        g <- ggplot()
        g <- g + geom_line(df3, mapping = aes(x = as.numeric(Day), y = TPM, color = Gene, group = Gene), size = 0.6) # draw lines
        g <- g + geom_point(df3, mapping = aes(x = as.numeric(Day), y = TPM, color = Gene, group = Gene), size = 0.9) #draw points
        g <- g + geom_errorbar(df3, mapping = aes(x = as.numeric(Day), ymin = TPM - SD, ymax = TPM + SD, color = Gene, group = Gene), width = 0.2, size = 0.3) #error bar
        g <- g + ggtitle(cluster_name) #title
        g <- g + xlab("Day") #y axis label
        g <- g + scale_color_manual(values = as.vector(mycols)) #color of each data
        g <- g + scale_x_continuous(breaks = c(0, 7, 14, 21, 28))
        g <- g + theme_bw(10) #basic theme
        g <- g + theme #custam theme
        g <- g + guides(color = guide_legend("Gene"), fill = guide_legend("Gene")) #lege

        g_legend <- get_legend(g)

        g2 <- plot_grid(g + theme(legend.position="none"), tpm_color_legend, nrow=1, ncol=2, rel_widths = c(9, 1))
        g3 <- plot_grid(g2, g_legend, nrow=2, ncol=1, rel_heights = c(7.5, 2.5))

        g_list <- c(g_list, list(g3))
    }

    g4 <- plot_grid(plotlist = g_list, nrow = 2, align = "hv")

    ggsave(g4, file = paste0(i, "_TF_exp.pdf"), width = 21, height = 14)
}