# 4. faza: Analiza podatkov

tabela3[-dv2,] %>% group_by(Drzava)

# komu v povprečju najvec: tabela5$Stevilo_nastopov in tabela

pod9 <- tabela5[-which(tabela5$Stevilo_nastopov + tabela5$Prvi_nastop < 1993),]

# komu smo dali max točke in kdo je zmagal SLO
df1 <- tabela4 %>% group_by(Leto) %>% filter(Tocke == max(Tocke))

pod_zmagovalci2 <- pod_zmagovalci[-which(grepl(1994, pod_zmagovalci$LETO)),]
pod_zmagovalci2 <- pod_zmagovalci2[-which(grepl(2000, pod_zmagovalci2$LETO)),]
pod_zmagovalci2 <- subset(pod_zmagovalci2, LETO > 1992)

df1 <- df1 %>% merge(pod_zmagovalci2, by.x="Leto", by.y="LETO") %>%
  select(-c(NASTOPAJOCI, NASLOV)) %>%
  rename("Zmagovalna" = DRZAVA, "Zmag_tocke" = TOCKE)

# število glasovanj za zmagovalce
sum(df1$Drzava == df1$Zmagovalna)
sort(df1[which(df1$Drzava != df1$Zmagovalna),]$Drzava)

# jugoslavija
df2 <- tabela3 %>% group_by(Leto) %>% filter(Tocke == max(Tocke))

pod_zmagovalci3 <- subset(pod_zmagovalci, LETO < 1993 & LETO > 1960)
pod_zmagovalci3 <- pod_zmagovalci3[-which(grepl(1977, pod_zmagovalci3$LETO)),]
pod_zmagovalci3 <- pod_zmagovalci3[-which(grepl(1978, pod_zmagovalci3$LETO)),]
pod_zmagovalci3 <- pod_zmagovalci3[-which(grepl(1979, pod_zmagovalci3$LETO)),]
pod_zmagovalci3 <- pod_zmagovalci3[-which(grepl(1980, pod_zmagovalci3$LETO)),]
pod_zmagovalci3 <- pod_zmagovalci3[-which(grepl(1985, pod_zmagovalci3$LETO)),]

df2 <- df2 %>% merge(pod_zmagovalci3, by.x="Leto", by.y="LETO") %>%
  select(-c(NASTOPAJOCI, NASLOV)) %>%
  rename("Zmagovalna" = DRZAVA, "Zmag_tocke" = TOCKE)

# število glasovanj za zmagovalce
sum(df2$Drzava == df2$Zmagovalna)

# ali lahko napovemo komu da slovenija max točk?
summary(lm(Vsota ~ Drzava, data = tabela9))

summary(lm(Zmag_tocke ~ Tocke, data=df1))

summary(model6 <- lm(Tocke ~ Drzava, data=subset(tabela4, Drzava != "Slovenia" &
                                                   Drzava != "Morocco" &
                                                   Drzava != "Luxembourg" &
                                                   Drzava != "Yugoslavia" &
                                                   Drzava != "Czech Republic" &
                                                   Drzava != "Slovakia" &
                                                   Drzava != "Monaco" &
                                                   Drzava != "Australia" &
                                                   Drzava != "Andorra" &
                                                   Drzava != "Serbia & Montenegro")))

# za koga glasujemo, ko ne glasujemo pravilno
df1$Drzava[which(df1$Drzava != df1$Zmagovalna)]

# jezik, finale, vrsta zasedbe (Ž,M,SKUPINA) pesmi
graf2

summary(lm(UVRSTITEV ~ TOCKE, data = tabela1))
summary(lm(UVRSTITEV ~ as.numeric(TOCKE), data = df3))

# slovenija uvrstitev, finale, jezik, zasedba
summary(model1 <- lm(Finale ~ Jezik, data=df3))

summary(model2 <- lm(UVRSTITEV ~ Jezik, data=df3))

summary(model3 <- lm(Finale ~ Zasedba, data=df3))

summary(model4 <- lm(UVRSTITEV ~ Zasedba, data=df3))

summary(model5 <- lm(UVRSTITEV ~ Jezik + Zasedba, data=df3))


fn.info = file.path("../generated_data/CHI/CHI_sample_info_2_CD38hi.txt")
df.info = fread(fn.info) %>%
  dplyr::filter(time==0, Response %in% c("low","high"))

sigs = c("IFN26", "LI.M165")
sig.names = c("IFN-I-DCact", "LI.M165 (DC activation module)") %>% setNames(sigs)
clustering = 1
clust.name = "C9"

dn.fig = "figures/sig_test_boxplots"
dir.create(dn.fig, showWarnings = F, recursive = T)

for(s in sigs) {
  df.mat = fread(glue::glue("results/sig_scores/scores_{s}_{clustering}.txt"))

  # add Responders
  df = df.mat %>% left_join(df.info %>% dplyr::select(subject, Response), by="subject") %>%
    dplyr::mutate(Response = factor(Response, levels=c("low","high"))) %>%
    dplyr::select(subject, Response, C9)

  X = df %>% pull(clust.name)
  Y = df$Response
  tt = t.test(X[Y=="high"], X[Y=="low"], data=df, alternative="greater")
  wt = wilcox.test(X[Y=="high"], X[Y=="low"], data=df, alternative="greater")

  df.txt = data.frame(x=-Inf, y=Inf, label=glue::glue("p = {format(wt$p.value, digits=2)}"))

  cm = pals::glasbey(13)[c(1:3,5:9,13,11)]
  yrng = c(-1,1.55)
  dotsz = 2
  ggplot(df, aes(Response, C9, group=Response)) +
    coord_flip(ylim = c(-0.55, 0.65)) +
    geom_boxplot(width=0.6, fill=cm[10], position = position_dodge2(preserve = "total"), lwd = 0.3, alpha = 0.5, outlier.shape = NA, show.legend = F) +
    geom_dotplot(aes(fill = Response), binaxis = "y", stackdir = "center", dotsize = dotsz, binwidth=0.03,
                 position=position_dodge(), show.legend = F) +
    geom_text(aes(x, y, label=label), col="red", hjust=1, vjust=-0.5, data=df.txt, inherit.aes = F) +
    scale_fill_manual(values = c("white", "black")) +
    ylab(glue::glue("{sig.names[s]} score")) +
    theme_classic()

  fn.fig = glue::glue("{dn.fig}/{s}_C9_score_boxplot")
  ggsave(paste0(fn.fig, ".png"), w=2.8, h=1.5)
  ggsave(paste0(fn.fig, ".pdf"), w=2.8, h=1.5, useDingbats=F)
}


radarchartspecifypointsize <- function (df, axistype = 0, seg = 4, pty = 16, pcol = 1:8, plty = 1:6,
    plwd = 1, pdensity = NULL, pangle = 45, pfcol = NA, cglty = 3,
    cglwd = 1, cglcol = "navy", axislabcol = "blue", title = "",
    maxmin = TRUE, na.itp = TRUE, centerzero = FALSE, vlabels = NULL,
    vlcex = NULL, caxislabels = NULL, calcex = NULL, paxislabels = NULL,
    palcex = NULL, pointsize = 2, ...)
{
    if (!is.data.frame(df)) {
        cat("The data must be given as dataframe.\n")
        return()
    }
    if ((n <- length(df)) < 3) {
        cat("The number of variables must be 3 or more.\n")
        return()
    }
    if (maxmin == FALSE) {
        dfmax <- apply(df, 2, max)
        dfmin <- apply(df, 2, min)
        df <- rbind(dfmax, dfmin, df)
    }
    plot(c(-1.2, 1.2), c(-1.2, 1.2), type = "n", frame.plot = FALSE,
        axes = FALSE, xlab = "", ylab = "", main = title, asp = 1,
        ...)
    theta <- seq(90, 450, length = n + 1) * pi/180
    theta <- theta[1:n]
    xx <- cos(theta)
    yy <- sin(theta)
    CGap <- ifelse(centerzero, 0, 1)
    for (i in 0:seg) {
        polygon(xx * (i + CGap)/(seg + CGap), yy * (i + CGap)/(seg +
            CGap), lty = cglty, lwd = cglwd, border = cglcol)
        if (axistype == 1 | axistype == 3)
            CAXISLABELS <- paste(i/seg * 100, "(%)")
        if (axistype == 4 | axistype == 5)
            CAXISLABELS <- sprintf("%3.2f", i/seg)
        if (!is.null(caxislabels) & (i < length(caxislabels)))
            CAXISLABELS <- caxislabels[i + 1]
        if (axistype == 1 | axistype == 3 | axistype == 4 | axistype ==
            5) {
            if (is.null(calcex))
                text(-0.05, (i + CGap)/(seg + CGap), CAXISLABELS,
                  col = axislabcol)
            else text(-0.05, (i + CGap)/(seg + CGap), CAXISLABELS,
                col = axislabcol, cex = calcex)
        }
    }
    if (centerzero) {
        arrows(0, 0, xx * 1, yy * 1, lwd = cglwd, lty = cglty,
            length = 0, col = cglcol)
    }
    else {
        arrows(xx/(seg + CGap), yy/(seg + CGap), xx * 1, yy *
            1, lwd = cglwd, lty = cglty, length = 0, col = cglcol)
    }
    PAXISLABELS <- df[1, 1:n]
    if (!is.null(paxislabels))
        PAXISLABELS <- paxislabels
    if (axistype == 2 | axistype == 3 | axistype == 5) {
        if (is.null(palcex))
            text(xx[1:n], yy[1:n], PAXISLABELS, col = axislabcol)
        else text(xx[1:n], yy[1:n], PAXISLABELS, col = axislabcol,
            cex = palcex)
    }
    VLABELS <- colnames(df)
    if (!is.null(vlabels))
        VLABELS <- vlabels
    if (is.null(vlcex))
        text(xx * 1.2, yy * 1.2, VLABELS)
    else text(xx * 1.2, yy * 1.2, VLABELS, cex = vlcex)
    series <- length(df[[1]])
    SX <- series - 2
    if (length(pty) < SX) {
        ptys <- rep(pty, SX)
    }
    else {
        ptys <- pty
    }
    if (length(pcol) < SX) {
        pcols <- rep(pcol, SX)
    }
    else {
        pcols <- pcol
    }
    if (length(plty) < SX) {
        pltys <- rep(plty, SX)
    }
    else {
        pltys <- plty
    }
    if (length(plwd) < SX) {
        plwds <- rep(plwd, SX)
    }
    else {
        plwds <- plwd
    }
    if (length(pdensity) < SX) {
        pdensities <- rep(pdensity, SX)
    }
    else {
        pdensities <- pdensity
    }
    if (length(pangle) < SX) {
        pangles <- rep(pangle, SX)
    }
    else {
        pangles <- pangle
    }
    if (length(pfcol) < SX) {
        pfcols <- rep(pfcol, SX)
    }
    else {
        pfcols <- pfcol
    }
    for (i in 3:series) {
        xxs <- xx
        yys <- yy
        scale <- CGap/(seg + CGap) + (df[i, ] - df[2, ])/(df[1,
            ] - df[2, ]) * seg/(seg + CGap)
        if (sum(!is.na(df[i, ])) < 3) {
            cat(sprintf("[DATA NOT ENOUGH] at %d\n%g\n", i, df[i,
                ]))
        }
        else {
            for (j in 1:n) {
                if (is.na(df[i, j])) {
                  if (na.itp) {
                    left <- ifelse(j > 1, j - 1, n)
                    while (is.na(df[i, left])) {
                      left <- ifelse(left > 1, left - 1, n)
                    }
                    right <- ifelse(j < n, j + 1, 1)
                    while (is.na(df[i, right])) {
                      right <- ifelse(right < n, right + 1, 1)
                    }
                    xxleft <- xx[left] * CGap/(seg + CGap) +
                      xx[left] * (df[i, left] - df[2, left])/(df[1,
                        left] - df[2, left]) * seg/(seg + CGap)
                    yyleft <- yy[left] * CGap/(seg + CGap) +
                      yy[left] * (df[i, left] - df[2, left])/(df[1,
                        left] - df[2, left]) * seg/(seg + CGap)
                    xxright <- xx[right] * CGap/(seg + CGap) +
                      xx[right] * (df[i, right] - df[2, right])/(df[1,
                        right] - df[2, right]) * seg/(seg + CGap)
                    yyright <- yy[right] * CGap/(seg + CGap) +
                      yy[right] * (df[i, right] - df[2, right])/(df[1,
                        right] - df[2, right]) * seg/(seg + CGap)
                    if (xxleft > xxright) {
                      xxtmp <- xxleft
                      yytmp <- yyleft
                      xxleft <- xxright
                      yyleft <- yyright
                      xxright <- xxtmp
                      yyright <- yytmp
                    }
                    xxs[j] <- xx[j] * (yyleft * xxright - yyright *
                      xxleft)/(yy[j] * (xxright - xxleft) - xx[j] *
                      (yyright - yyleft))
                    yys[j] <- (yy[j]/xx[j]) * xxs[j]
                  }
                  else {
                    xxs[j] <- 0
                    yys[j] <- 0
                  }
                }
                else {
                  xxs[j] <- xx[j] * CGap/(seg + CGap) + xx[j] *
                    (df[i, j] - df[2, j])/(df[1, j] - df[2, j]) *
                    seg/(seg + CGap)
                  yys[j] <- yy[j] * CGap/(seg + CGap) + yy[j] *
                    (df[i, j] - df[2, j])/(df[1, j] - df[2, j]) *
                    seg/(seg + CGap)
                }
            }
            if (is.null(pdensities)) {
                polygon(xxs, yys, lty = pltys[i - 2], lwd = plwds[i -
                  2], border = pcols[i - 2], col = pfcols[i -
                  2])
            }
            else {
                polygon(xxs, yys, lty = pltys[i - 2], lwd = plwds[i -
                  2], border = pcols[i - 2], density = pdensities[i -
                  2], angle = pangles[i - 2], col = pfcols[i -
                  2])
            }
            points(xx * scale, yy * scale, pch = ptys[i - 2],
                col = pcols[i - 2],cex= pointsize)
        }
    }
}


#' Convert given edu distribution into eapr
#'
#' This function returns EAPR for given education distribution
#' @param prop distribution data 'edu' (sorted) 'value'
#' @return eapr
#' @keywords read
#' @export
#' @examples

eapr <- function(prop) {
  # prop <<- prop
  # stop("Hello")
  # print("Hello")
  x <- rev(cumsum(rev(prop$value)))
  n <- length(x)
  eapr.x <- prop[-nrow(prop), ]
  eapr.x$value <- x[2:n]/x[1:(n-1)]
  return(eapr.x)
}


# -----------------------------------------------------------------------------
# PROJECT:
#    Evaluating the structure of the communities of the estuary
#    and gulf of St.Lawrence
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
# STEP:
#   2. Evaluation of analysis accuracy + tables and figures
# -----------------------------------------------------------------------------

# -----------------------------------------------------------------------------
# FILES:
#           Script <- file = 'Script/2-2-Tanimoto_accuracy.r'
#           RData <- file = 'RData/Tanimoto_accuracy.RData'
#           Figures <- file = ''
#           Tables <- file = ''
# -----------------------------------------------------------------------------

load("./RData/Tanimoto_analysis.RData")

Tanimoto_accuracy <- tanimoto_accuracy(Tanimoto_analysis = Tanimoto_analysis)

# Creating an empty plot
eplot <- function(x, y) {
  plot(x = x, y = y, bty = "n",ann = FALSE,xaxt = "n",yaxt = "n",type = "n",bg = "grey", ylim = c(-0.09,1.09), xlim = c(-0.09,1.09))
}

pdf("./Article/results4.pdf",width=7,height=7)
# Plots
par(mfrow=c(2,2))
# Graph
for(j in 8:11) {
    eplot(x = accuracy[, 'wt'], y = accuracy[, j])
    par(pch = 21,  xaxs = "i", yaxs = "i", family = "serif")
    # foodwebs <- to.verify
    # col <- c('blue','green','black','red','yellow','darkgrey','orange','brown','grey','green','darkgreen','darkblue')
    col <- gray.colors(11, start = 0, end = 0.8, gamma = 2.2, alpha = NULL)
    names <- c('TSS','Score y', 'Score -y', 'Accuracy score')
    # sample(colours(), length(foodwebs))
    # cols <- c("#FF000088","#00FF0088","#0000FF88")
    # cols2 <- c("#FF0000","#00FF00","#0000FF")

    # Axes
    # rect(0, 0, 1, 1, col = "#eeeeee", border = NA)
    axis(side = 1, at = seq(0, 1, by = 0.2), labels = seq(0, 1, by = 0.2), las = 1, pos = 0)
    axis(side = 2, at = seq(0, 1, by = 0.2), labels = seq(0, 1, by = 0.2), las = 1, pos = -0.09)
    axis(side = 3, at = seq(0, 1, by = 0.2), labels = seq(0, 1, by = 0.2), las = 1, pos = 1)
    axis(side = 4, at = seq(0, 1, by = 0.2), labels = seq(0, 1, by = 0.2), las = 1, pos = 1.09)
    # abline(v = seq(0,6,by = 2), col = "white", lty = 2)
    # abline(h = seq(1,2,by = 1), col = "white", lty = 2)

    #
    mtext(text = names[j-7], side = 2, line = 2, at = 0.5, font = 2, cex = 1)
    mtext(text = "Similarity weight", side = 1, line = 2, at = 0.5, font = 2, cex = 1)

    for(i in 1:12) {
        x <- as.numeric(names(Tanimoto_analysis[[1]]))
        y <- numeric()
        for(k in 1:6) {
            y <- c(y,mean(as.numeric(accuracy[which(accuracy[, 'K'] == i & accuracy[, 'wt'] == names(Tanimoto_analysis[[1]])[k]), j][-5])))
            # mean(as.numeric(accuracy[which(accuracy[, 'K'] == i & accuracy[, 'wt'] == names(Tanimoto_analysis[[1]])[k]), j][-5]))
        }

        points(x = x, y = y, bg = col[i], cex = 1.25, pch = 18, col = col[i])
        lines(x = x, y = y, col = col[i], lwd = 0.5)

        # points(x = accuracy[which(accuracy[, 'K'] == i), 'wt'], y = accuracy[which(accuracy[, 'K'] == i), j], bg = col[i], cex = 1.25, pch = 18, col = col[i])
        # lines(x = accuracy[which(accuracy[, 'K'] == i), 'wt'], y = accuracy[which(accuracy[, 'K'] == i), j], col = col[i], lwd = 0.5)
    }

    # boxplot(formula = as.numeric(accuracy[, j]) ~ accuracy[, 'wt'],
    #         data = accuracy,
    #         boxwex = 0.075,
    #         axes = FALSE,
    #         add = TRUE,
    #         at = seq(0, 1, by = 0.1))
}
dev.off()


################################################################################
# The contents of this file are Teradata Public Content
# and have been released to the Public Domain.
# Licensed under BSD; see "license.txt" file for more information.
# Copyright (c) 2021 by Teradata
################################################################################
#
# R And Python Analytics with SCRIPT Table Operator
# Orange Book supplementary material
# Alexander Kolovos - October 2021 - v.2.1
#
# Example 1: Scoring (R version)
# File     : ex1rSco.r
#
# Score input rows from database by using model information from
# an existing R model object.
# Use case:
# Predict the propensity of a financial services customer base
# to open a credit card account.
#
# Script performs identical task as ex1rScoNonIter.r. Reads in data in chunks.
# Script accounts for the general scenario that an AMP might have no data.
#
# Requires the randomForest add-on package.
#
# Required input:
# - ex1tblSco table data from file "ex1dataSco.csv"
# - scoring model saved in R model object "ex1rMod.rds"
#
# Output:
# - cust_id    : The customer ID
# - Prob0      : Probability that a customer does not open an account
# - Prob1      : Probability that a customer opens an account
# - cc_acct_ind: [Actual outcome]
#
################################################################################

# Load dependency package
# We use the suppressPackageStartupMessages() function to prevent unwanted
# output that may interfere with the output expected by the database.
suppressPackageStartupMessages(library(randomForest))

DELIMITER ='\t'
stdin <- file(description="stdin",open="r")

inputDF <- data.frame();

# A list with the column names of the input data
cn <- c("cust_id", "tot_income", "tot_age", "tot_cust_years", "tot_children",
        "female_ind", "single_ind", "married_ind", "separated_ind",
        "ca_resident_ind", "ny_resident_ind", "tx_resident_ind",
        "il_resident_ind", "az_resident_ind", "oh_resident_ind", "ck_acct_ind",
        "cc_acct_ind", "sv_acct_ind", "ck_avg_bal", "sv_avg_bal", "cc_avg_bal",
        "ck_avg_tran_amt", "sv_avg_tran_amt", "cc_avg_tran_amt", "q1_trans_cnt",
        "q2_trans_cnt", "q3_trans_cnt", "q4_trans_cnt")

# Know your data: You must know in advance the number and data types of the
# incoming columns from the SQL Engine database!
ct <- c("integer", "double", "integer", "integer", "integer", "integer",
        "integer", "integer", "integer", "integer", "integer", "integer",
        "integer", "integer", "integer", "integer", "factor", "integer",
        "double", "double", "double", "double", "double", "double", "integer",
        "integer", "integer", "integer")

# Read the scoring model information
ScoringModel <- readRDS("./myDB/ex1rMod.rds")

### Ingest and process the input data rows, nRowsIn at a pass
###
nRowsIn <- 500

# Use tryCatch to produce an error if something goes wrong in the try block
tryCatch({
    # Read streamed input data in chunks of 1000 K rows. Count the passes
    nPass = 0

    while (1==1) {

        nPass = nPass + 1

        # Read input rows directly with read.table(). The nrows argument
        # specifies to ingest up to nRowsIn rows in current pass.
        inputDF <- try(read.table(stdin, sep=DELIMITER, flush=TRUE,
                                  header=FALSE, quote="", na.strings="",
                                  colClasses=ct, col.names=cn, nrows=nRowsIn),
                       silent=TRUE)

        # If AMP has no data, stream ended, or issue occured, quit gracefully.
        #
        # Note: The connection is left open for the iterative process. We use
        #       as exit criterion the iteration where inputDF is found empty.
        if (nrow(inputDF) == 0 || class(inputDF) == "try-error") {
            inputDF <- NULL
            quit()
        }

        Predicted <- suppressWarnings( predict(ScoringModel, newdata=inputDF,
                                               type="vote") );

        # Build export data frame:
        Scores <- data.frame(inputDF$cust_id, Predicted, inputDF$cc_acct_ind)

        # No more need for inputDF contents. Create it anew for next pass.
        remove('inputDF')
        inputDF <- data.frame();

        # Export results to the SQL Engine database through standard output
        write.table(Scores, file=stdout(), col.names=FALSE, row.names=FALSE,
                    quote=FALSE, sep=DELIMITER, na="")
    }
},
error = function(c) {
    msg <- conditionMessage(c)
    # Load error message
    write(paste("Script Failure: ", msg, sep=" "), stderr())
    quit(status=99)
})

# Close the open connection
close(stdin)


bottom.contact.maxdepth = function( sm, O, bcmethods, bcp ) {
  # using previously computed estimates of touchdown (bc0) and lift-off (bc1) and associated SD's
  # find maximum depths in the neighbourhoods

  res = list( bc0=NA, bc1=NA)

  names( sm) = c("Z", "timestamp", "ts" ) # Z is used to abstract the variable name such that any variable can be passed
  N = nrow(sm)

  # must be careful as timestamp is being converted to text and will lose tzone ... need to reset to correct timezone:
  bcm0 = paste(bcmethods, "0", sep="")
  bcm1 = paste(bcmethods, "1", sep="")

  tt0 =   tt1 =  t( as.data.frame(O[ bcm1 ]) )

  tzone = lubridate::tz(sm$timestamp[1])
  # reconvert to UTC as time zone is lost with the transpose
  tmp0 = lubridate::ymd_hms( t( as.data.frame(O[ bcm0 ]) ), tz=tzone )
  tmp1 = lubridate::ymd_hms( t( as.data.frame(O[ bcm1 ]) ), tz=tzone )

  bottom0.mean =  mean(tmp0, na.rm=TRUE)
  bottom1.mean =  mean(tmp1, na.rm=TRUE)

  bottom0.sd = max( 10, sd( tmp0, na.rm=TRUE  ), na.rm=TRUE ) # in seconds
  bottom1.sd = max( 10, sd( tmp1, na.rm=TRUE  ), na.rm=TRUE )

  bc0.range = bottom0.mean + bcp$maxdepth.sd.multiplier * bottom0.sd * c(-1, 1)
  bc1.range = bottom1.mean + bcp$maxdepth.sd.multiplier * bottom1.sd * c(-1, 1)

  bc0 = which( sm$timestamp > bc0.range[1] & sm$timestamp < bc0.range[2] )
  bc1 = which( sm$timestamp > bc1.range[1] & sm$timestamp < bc1.range[2] )
  zx0 = abs( sm$Z[bc0] - max( sm$Z[bc0] , na.rm=TRUE ) )
  zx1 = abs( sm$Z[bc1] - max( sm$Z[bc1] , na.rm=TRUE ) )

  zx0i = which( zx0 < bcp$eps.depth )
  zx1i = which( zx1 < bcp$eps.depth )

  if (length( zx0i) > 0 ) {
    z0 = min ( zx0i )
  } else {
    z0 = which.max( sm$Z[bc0] )
  }

  if (length( zx1i) > 0 ) {
    z1 = max ( zx1i )
  } else {
    z1 = which.max( sm$Z[bc1] )
  }

  bc0i = bc0[ z0 ] # choose left-most (earliest)
  bc1i = bc1[ z1 ] # choose right most (latest)

  res =  list( bc0=sm$timestamp[bc0i], bc1=sm$timestamp[bc1i] )

  return(res)
}


library(rrBLUP)
set.seed(42)
args = commandArgs(trailingOnly=TRUE)
X_file <- args[1] # geno matrix
Y_file <- args[2] # pheno matrix
test_file <- args[3] # individuals in test set

Y <- read.csv(Y_file, row.names=1)
X <- read.csv(X_file, row.names=1)
Test <- read.table(test_file, head=F,sep='\t',stringsAsFactors=F)

X_test <- X[Test[,1],]
Y_test <- Y[Test[,1],]
X <- X[!rownames(X) %in% Test[,1],, drop=False]
Y <- Y[!rownames(Y) %in% Test[,1],, drop=False]
write.csv(X,'geno_training.csv',row.names=T,quote=F)
write.csv(Y,'pheno_training.csv',row.names=T,quote=F)
write.csv(X_test,'geno_test.csv',row.names=T,quote=F)
write.csv(Y_test,'pheno_test.csv',row.names=T,quote=F)


library(ComplexHeatmap)
library(circlize)
library(cluster)
library(svglite)

setwd('D:/1998c/Documents/Research/202002scRNA/package_release/scETM/scripts/heatmap_fig/AD')

mat <- read.table('perm_md_onesided_celltype.csv', row.names = 1,sep=',',check.names = F,
                   header = TRUE, stringsAsFactors = FALSE)
mat = t(mat)
aster <- read.table("perm_p_onesided_celltype.csv",row.names = 1,sep=',',
                    header = TRUE, stringsAsFactors = FALSE)

aster = t(aster)
hmap <- Heatmap(
  mat,
  cell_fun = function(j, i, x, y, w, h, fill) {
    if (aster[i, j] < 0.05) {
      grid.text("*", x, y)}},
  name='cell type',
  column_dend_side = "bottom",
  show_row_names = TRUE,
  show_column_names = TRUE,
  cluster_rows = FALSE,
  cluster_columns = FALSE,
  row_names_gp = gpar(fontsize = 7),
  column_names_gp = gpar(fontsize = 10),
  raster_quality = 5,
  rect_gp = gpar(col = "grey", lwd = 0.3))

mat2 <- read.table('perm_md_onesided_condition.csv', row.names = 1, sep=',', check.names = F,
                  header = TRUE, stringsAsFactors = FALSE)
mat2 = t(mat2)

aster2 <- read.table("perm_p_onesided_condition.csv", row.names = 1, sep=',',
                     header = TRUE, stringsAsFactors = FALSE)
aster2 = t(aster2)
hmap2 <- Heatmap(
  mat2,
  cell_fun = function(j, i, x, y, w, h, fill) {
    if(aster2[i, j] < 0.05) {
      grid.text("*", x, y)}},
  name='condition',
  column_dend_side = "bottom",
  show_row_names = TRUE,
  show_column_names = TRUE,
  cluster_rows = FALSE,
  cluster_columns = FALSE,
  row_names_gp = gpar(fontsize = 7),
  column_names_gp = gpar(fontsize = 10),
  raster_quality = 5,
  rect_gp = gpar(col = "grey", lwd = 0.3))

pdf(file = "panel_c_permtest.pdf",width=3,height=6)
draw(hmap+hmap2, heatmap_legend_side="right", annotation_legend_side="right")
dev.off()

svglite("panel_c_permtest.svg",width=3,height=6)
draw(hmap+hmap2, heatmap_legend_side="left", annotation_legend_side="right")
dev.off()


val = 1

while (val <= 5) {
    print(val)
    val = val + 1
}


##The function to calculate the likelihood values for each of the integrated measurements.

IntegratedLikelihoodCalculation2<-function (WD, OD, CropName)
{

##Step 1. Get the information of integrated measurements from the processed evaluation file or File A.
eval(parse(text=paste('Evaluation<-read.table("',OD,
'/EvaluateFrame_2.txt", header=TRUE,comment.char="")',sep = '')));
#print(Evaluation);
Dimension<-dim(Evaluation);

Treatments<-Evaluation[,"RUN"]
TreatmentNumberIndex<-which.max(Evaluation[,"RUN"]);
TreatmentNumber<-Treatments[TreatmentNumberIndex]; ##Get the number of treatments in the experiment.
RunNumber<-(Dimension[1]/TreatmentNumber); ##Get the number of model runs.

## Step 2. Get the information about measurement variance from a given Excel file.
#library(xlsReadWrite);

if (CropName=="PT" || CropName=="SC" || CropName=="CS" || CropName=="TN" || CropName=="TR" || CropName=="WH")
{
#  eval(parse(text=paste('VAR<-read.xls("',WD,
#  '/MeasurementVariance.xls", sheet = "',CropName,'", rowNames = T, colNames=T)',sep = '')));

  eval(parse(text=paste('VAR<-read.csv("',WD,
  '/MeasurementVariance_', CropName, '.csv", header = T)',sep = '')));
} else
{
#  eval(parse(text=paste('VAR<-read.xls("',WD,
#  '/MeasurementVariance.xls", sheet = "Sheet 1", rowNames = T, colNames=T)',sep = '')));

  eval(parse(text=paste('VAR<-read.csv("',WD,
  '/MeasurementVariance_All.csv", header = T)',sep = '')));
}

newRowNames <- VAR[ , 1];
VAR <- VAR[ , -1];
rownames(VAR) <- newRowNames;
##Read the variance information including standard devaiton, Variance, and CV from a given Excel file.

##Step 3. Calclate the likelihood value for the integrated measurements, such as
 #anthesis date (ADAP), maturity date (MDAP), first pod date (PD1TS),
 #Pod/Ear/Panicle weight at maturity (kg [dm]/ha) (PWAM),
 #Yield at harvest maturity (kg [dm]/ha) (HWAM), Tops weight at maturity (kg [dm]/ha) (CWAM),
 #Leaf area index, maximum (LAIX), Leaf number per stem at maturity (L#SM).

MeasurementNames<-c();
RowNames<-rownames(VAR);

NumberOfMeasurement<-dim(VAR)[1];

if (CropName=="BA" || CropName=="RI" || CropName=="WH")
{
VAR["PD1T","Flag"]<-0;
VAR["PWAM","Flag"]<-0;
#Evaluate file of barley doesn't have this two outputs.
}

for (i in 1:NumberOfMeasurement)
{
if (VAR[i,"Flag"]==2)
{
MeasurementNames<-c(MeasurementNames,RowNames[i]);
}
}

eval(parse(text=paste('EvaluateFile<-readLines("',OD,'/Evaluate_output.txt",n=-1)',sep = '')));
IsHWAH<-grep("HWAH",EvaluateFile);
#Read the Evaluate file to see if it contains the output called "HWAH", while the default name is "HWAM".

if(length(IsHWAH)!=0)
{
Address<-which(MeasurementNames == "HWAM");
MeasurementNames[Address]<-"HWAH";
}

##Integrated measurement names used in the second round of GLUE.
MeasurementNumber<-length(MeasurementNames); ##Number of integrated measurements.

IntegratedLikelihoodMatrix<-matrix(Evaluation[,1],nrow=length(Evaluation[,1]),ncol=1,byrow=T);
#The first column of Evaluation is the number of runs in each model run with an individual random parameter sets.
#Since more than one experiment could be run simultaneously, the total treatment number should be the sum of all
#treatments in all experiments. Thus, the value of "Run" is used as number of "Total Treatment".

IntegratedLikelihoodMatrix<-rbind("Treatment",IntegratedLikelihoodMatrix);

##Calculate the likelihood values for each of the measurements.
for (i in 1:MeasurementNumber)
{

Simulation<-Evaluation[,paste(MeasurementNames[i],"S", sep="")]; ##Read the simulated values.
Measurement<-Evaluation[,paste(MeasurementNames[i],"M", sep="")]; ##Read the measured values.
Simulation<-ifelse(Simulation==-99, NA, Simulation);
Measurement<-ifelse(Measurement==-99, NA, Measurement);##Change the unknown measurements "-99" to NA.

if (is.na(VAR[MeasurementNames[i],"CV"]))
{
Variance<-VAR[MeasurementNames[i],"Variance"];
} else
{
CV<-VAR[MeasurementNames[i],"CV"];
Variance<-(Measurement*CV)^2;
}

eval(parse(text=paste('source("',WD,'/Calculation.r")',sep = '')));
Likelihood<-Calculation(Simulation, Measurement, Variance);
##Call the function "Calculation" to calculate the likelihood values.

Likelihood<-c(MeasurementNames[i],Likelihood); ##Add a tile for the likelihood values.

eval(parse(text = paste('Likelihood',MeasurementNames[i],'<-matrix(Likelihood,nrow=length(Likelihood),ncol=1,byrow=T)',
sep = ''))); ##Save the likehood values for each of the measurements as a single column matrix.

eval(parse(text = paste('IntegratedLikelihoodMatrix<-cbind(IntegratedLikelihoodMatrix,Likelihood',
MeasurementNames[i],')',sep="")));
##Save the likehood values for all of the measurements as a matrix, with the treatment number as the first column.

}

IntegratedLikelihoodMatrix<-ifelse(is.na(IntegratedLikelihoodMatrix)==TRUE, 1, IntegratedLikelihoodMatrix);
#Change the NA values to 1 before calculating the combined likelihood value.

#library ('MASS');
#eval(parse(text=paste('write.matrix(IntegratedLikelihoodMatrix,file ="',OD,
#'/IntegratedLikelihoodMatrix_Frame_2.txt")',sep = '')));

eval(parse(text=paste('write.table(as.matrix(IntegratedLikelihoodMatrix), file ="',OD,
'/IntegratedLikelihoodMatrix_Frame_2.txt", row.names = F, col.names = F, append = F)',sep = '')));

eval(parse(text=paste('IntegratedLikelihoodMatrixTable<-read.table("',OD,
'/IntegratedLikelihoodMatrix_Frame_2.txt", header=TRUE,comment.char="")',sep = '')));

##Step 4. Likelihood combination
eval(parse(text=paste('source ("',WD,'/Combination.r")',sep = '')));
IntegratedCombinedLikelihood<-Combination(IntegratedLikelihoodMatrixTable);
#Calculate the combined likelihood values for the integrated measurements.

names(IntegratedCombinedLikelihood)<-"IntegratedCombinedLikelihood" #Set the name for the combined likelihood values.

IntegratedLikelihoodMatrixTable<-cbind(IntegratedLikelihoodMatrixTable,IntegratedCombinedLikelihood)#Add the combined likelihood 1 to the likelihood matrix.

##Step 5. Distribute the icombined likelihood values of the integrated measurements to each treatment.
for (i in 1:TreatmentNumber)
{

Selection<-i;
RowIndex <- which(Selection==IntegratedLikelihoodMatrixTable$Treatment);
# Get the address of the rows that contain information of treatment i.
ColumnIndex<-c("Treatment",MeasurementNames,"IntegratedCombinedLikelihood");

eval(parse(text = paste('IntegratedCombinedLikelihoodTreatment_2_',i,
'<-IntegratedLikelihoodMatrixTable[RowIndex,ColumnIndex]',sep="")));

#library ('MASS');
#eval(parse(text = paste('write.matrix(IntegratedCombinedLikelihoodTreatment_2_',i,
#',file ="',OD,'/IntegratedLikelihoodTreatment_2_',i,'.txt")',sep="")));

eval(parse(text = paste('write.table(as.matrix(IntegratedCombinedLikelihoodTreatment_2_',i,
'), file ="',OD,'/IntegratedLikelihoodTreatment_2_',i,'.txt", row.names = F, col.names = T, append = F)',sep="")));

}

return(TreatmentNumber);
rm(list = ls());

}


library(GenomicFeatures)
library(ggplot2)
library(GenomicRanges)
library(bumphunter)
library(data.table)
library(clusterProfiler)
require(org.Hs.eg.db)

load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMVs_hypoDMRs_methylSeekR.rda")
load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMVs_hypoDMRs_methylSeekR_prenatal.rda")
load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/DMR/DMR_objects.rda")

rm(DMVs,DMVs_pren)

dmvs = c(DMVs.100kb,DMVs_100kb.pren)

## How many are identified?

num = data.frame(num = unlist(elementNROWS(dmvs)), id = names(dmvs))
num$celltype = ifelse(num$id %in% pd$Data.ID, pd[match(num$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(num, quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_number.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_number_byAge_postnatal.pdf")

w = num[which(num$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = pd[match(w$id, pd$Data.ID), "Age"])
cor.test(x = w[which(w$celltype=="Neuron"),"Age"], y = w[which(w$celltype=="Neuron"),"num"])
#t = -0.82869, df = 22, p-value = 0.4162, cor -0.1739822

cor.test(x = w[which(w$celltype=="Glia"),"Age"], y = w[which(w$celltype=="Glia"),"num"])
#t = -1.1274, df = 6, p-value = 0.3026, cor -0.4180941

ggplot(w, aes(x = Age, y = num, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") + theme_classic() +
  ylab("Number") + xlab("Age (Years)") +
  ggtitle("Number DMVs Identified by Age") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(num, aes(x = celltype, y = num)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Count") + xlab("") + ylim(0,5500) +
  ggtitle("Number of DMVs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

num = read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_number.csv")
range(num$num[which(num$celltype=="Prenatal")]) # 1098 5305
range(num$num[which(num$celltype=="Neuron")]) # 1113 1410
range(num$num[which(num$celltype=="Glia")]) # 915 1144

t.test(num[num$celltype=="Prenatal","num"],num[num$celltype!="Prenatal","num"])
#t = 6.0393, df = 19.216, p-value = 7.875e-06
#  mean of x mean of y
# 2856.850  1164.469

## how long are these regions?

widths = do.call(rbind, Map(cbind, lapply(dmvs, function(y) data.frame(mean = mean(width(y)), median = median(width(y)),
                                                                       sd = sd(width(y)), min = min(width(y)), max = max(width(y)))), id = as.list(names(dmvs))))
widths$celltype = ifelse(widths$id %in% pd$Data.ID, pd[match(widths$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(widths, quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_widths.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_widths.pdf")

w = widths[which(widths$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = pd[match(w$id, pd$Data.ID), "Age"])
cor.test(x = w[which(w$celltype=="Neuron"),"Age"], y = w[which(w$celltype=="Neuron"),"median"])
#t = 0.25929, df = 22, p-value = 0.7978, cor 0.05519736

cor.test(x = w[which(w$celltype=="Glia"),"Age"], y = w[which(w$celltype=="Glia"),"median"])
#t = -0.99175, df = 6, p-value = 0.3596, cor -0.3752859

ggplot(w, aes(x = Age, y = median, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") + theme_classic() +
  ylab("Bases") + xlab("Age (Years)") +
  ggtitle("Median DMV Widths by Age") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(widths, aes(x = celltype, y = median)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Bases") + xlab("") +
  ggtitle("Median DMV Widths") + ylim(0,8500) +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
ggplot(widths, aes(x = celltype, y = mean)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Bases") + xlab("") +
  ggtitle("Mean DMV Widths") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

t.test(widths[widths$celltype=="Prenatal","median"],widths[widths$celltype!="Prenatal","median"])
#t = 6.7297, df = 20.132, p-value = 1.459e-06
#  mean of x mean of y
# 7239.625  6665.750
t.test(widths[widths$celltype=="Prenatal","median"],widths[widths$celltype=="Neuron","median"])
#t = 6.8666, df = 19.939, p-value = 1.153e-06
#  mean of x mean of y
#7239.625  6655.500
t.test(widths[widths$celltype=="Glia","median"],widths[widths$celltype=="Neuron","median"])
#t = 0.92004, df = 8.387, p-value = 0.3833
#  mean of x mean of y
#6696.5    6655.5
t.test(widths[widths$celltype=="Glia","median"],widths[widths$celltype=="Prenatal","median"])
#t = -5.7655, df = 25.452, p-value = 4.887e-06
#  mean of x mean of y
#6696.500  7239.625

## What proportion of the genome do they cover?

load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/PMDs_methylSeekR.rda")
load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/PMDs_methylSeekR_prenatal.rda")
sLengths = c(lapply(PMDsegments.CG, reduce), lapply(PMDsegments.CGpren, reduce))
rm(PMDsegments.CGpren, PMDsegments.CG)
percGenome = data.frame(percent = unlist(mapply(function(p, t) round((sum(width(p))/sum(as.numeric(width(t)))*100),2), dmvs, sLengths)),
                        id = names(dmvs))
percGenome$celltype = ifelse(percGenome$id %in% pd$Data.ID, pd[match(percGenome$id, pd$Data.ID),"Cell.Type"], "Prenatal")

write.csv(percGenome, quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_genomeCoverage.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_percentGenome.pdf",
    height = 3.5, width = 4.5)

w = percGenome[which(percGenome$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = pd[match(w$id, pd$Data.ID), "Age"])
cor.test(x = w[which(w$celltype=="Neuron"),"Age"], y = w[which(w$celltype=="Neuron"),"percent"])
#t = -1.2851, df = 22, p-value = 0.2121, cor -0.264245

cor.test(x = w[which(w$celltype=="Glia"),"Age"], y = w[which(w$celltype=="Glia"),"percent"])
#t = -1.321, df = 6, p-value = 0.2346, cor -0.4746662

ggplot(w, aes(x = Age, y = percent, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) +
  scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") +  theme_classic() +
  ylab("Percent") + xlab("Age (Years)") +
  ggtitle("Percent of Genome that is DMV by Age") +
  theme(title = element_text(size = 20),
        text = element_text(size = 20),
        legend.title=element_blank(),
        legend.position="bottom")
ggplot(percGenome, aes(x = celltype, y = percent)) + geom_boxplot() +
  labs(fill="") + ylim(0,2) +
  ylab("Percent") + xlab("") +
  ggtitle("Percent Genome: DMV") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

genome = data.table(read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_genomeCoverage.csv"))
genome[,mean(percent), by = c("celltype")]
#   celltype        V1
#1:     Glia 0.2925000
#2:   Neuron 0.3408333
#3: Prenatal 0.9515000

t.test(genome[genome$celltype=="Prenatal","percent"],genome[genome$celltype!="Prenatal","percent"])
#t = 5.9192, df = 19.101, p-value = 1.044e-05
#  mean of x mean of y
#0.95150   0.32875
(0.95150-0.32875)/0.32875 # 1.894297

t.test(genome[genome$celltype=="Prenatal","percent"],genome[genome$celltype=="Neuron","percent"])
#t = 5.8072, df = 19.064, p-value = 1.338e-05
#  mean of x mean of y
#0.9515000 0.3408333
t.test(genome[genome$celltype=="Glia","percent"],genome[genome$celltype=="Neuron","percent"])
#t = -4.676, df = 10.093, p-value = 0.0008513
#  mean of x mean of y
#0.2925000 0.3408333
t.test(genome[genome$celltype=="Prenatal","percent"],genome[genome$celltype=="Glia","percent"])
#t = 6.2471, df = 19.302, p-value = 4.966e-06
#  mean of x mean of y
#0.9515    0.2925

### How much of the dmvs overlap?

# Master list of all bases in each setting
dmaster = reduce(do.call(getMethod(c, "GenomicRanges"), GRangesList(dmvs))) # 43995 total
c(mean=mean(width(dmaster)), median=median(width(dmaster)), sd=sd(width(dmaster)), min=min(width(dmaster)), max=max(width(dmaster)))/1000
#         mean    median        sd       min       max
#kb:  13.33677   9.27300  15.81653   5.00000 451.64300

## regions that are shared by samples

all = list(All = Reduce(intersect, dmvs),
           Prenatal = Reduce(intersect, DMVs_100kb.pren),
           Postnatal = Reduce(intersect, DMVs.100kb),
           Neurons = Reduce(intersect, DMVs.100kb[which(names(DMVs.100kb) %in% pd[pd$Cell.Type=="Neuron","Data.ID"])]),
           Glia = Reduce(intersect, DMVs.100kb[which(names(DMVs.100kb) %in% pd[pd$Cell.Type=="Glia","Data.ID"])]))
elementNROWS(all)
#      All  Prenatal Postnatal   Neurons      Glia
#      318       559       399       588       554

round((unlist(lapply(all, function(x) sum(width(x))))/sum(width(dmaster))*100),2)
#      All  Prenatal Postnatal   Neurons      Glia
#     1.49      2.88      1.97      2.85      2.79
# 1.5-3% of total bases in the DMV state are shared by all or most

shared = mapply(function(all,ind) lapply( ind, function(x) round(all / x *100,2)), lapply(all, function(x) (sum(width(x))/sum(as.numeric(width(sLengths[[1]])))*100)),
                lapply( list(dmvs,DMVs_100kb.pren,DMVs.100kb,
                             DMVs.100kb[which(names(DMVs.100kb) %in% pd[pd$Cell.Type=="Neuron","Data.ID"])],
                             DMVs.100kb[which(names(DMVs.100kb) %in% pd[pd$Cell.Type=="Glia","Data.ID"])]),
                        function(y) lapply(y, function(x) sum(width(x))/sum(as.numeric(width(sLengths[[1]])))*100)), SIMPLIFY = F)
shared = do.call(rbind, Map(data.frame, percent = lapply(shared, function(x) unlist(x, recursive=F)), id = lapply(shared, names), group = as.list(names(shared))))

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/percent_shared_dmvs_perSample_byBaseCoverage.pdf")
ggplot(shared, aes(x = group, y = percent)) + geom_boxplot() +
  labs(fill="") + theme_classic() +
  ylim(0,100) +
  ylab("Percent") + xlab("") +
  ggtitle("Percent Bases Shared Per Sample: DMV") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

data.table(shared)[,mean(percent), by = c("group")]
#       group       V1
#1:       All 20.18327
#2:  Prenatal 22.17000
#3: Postnatal 34.00156
#4:   Neurons 47.20042
#5:      Glia 54.08875

## How much of prenatal DMVs is represented in each postnatal sample?

pren = reduce(do.call(getMethod(c, "GenomicRanges"), GRangesList(dmvs[-which(names(dmvs) %in% pd$Data.ID)])))

sharedP = lapply(dmvs, function(x) Reduce(intersect, list(x, pren)))

prenperc = mapply(function(pren,ind) round(sum(width(reduce(pren))) / sum(width(reduce(ind))) *100,2),
                   sharedP, dmvs, SIMPLIFY = F)

prenperc = data.frame(perc = unlist(prenperc))
prenperc$Person = rownames(prenperc)
prenperc$Age = pd[match(prenperc$Person, pd$Data.ID),"Age"]
prenperc$CellType = ifelse(prenperc$Person %in% pd$Data.ID, pd[match(prenperc$Person, pd$Data.ID),"Cell.Type"], "Prenatal")

write.table(prenperc, quote = F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_sharedWithPrenatal.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/percent_shared_withPrenatal_DMV_perSample_byBaseCoverage.pdf", height = 4)
ggplot(prenperc[prenperc$CellType!="Prenatal",], aes(x = CellType, y = perc)) + geom_boxplot() +
  labs(fill="") + theme_classic() +
  ylim(0,100) +
  ylab("Percent") + xlab("") +
  ggtitle("Percent Bases Shared with Prenatal") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
ggplot(prenperc[prenperc$CellType!="Prenatal",], aes(x = Age, y = perc, colour = CellType)) +
  geom_path() + geom_point() + ylim(0,100) +
  theme_classic() + scale_colour_brewer(8, palette="Dark2") +
  ylab("Percent") + xlab("Age") +
  ggtitle("Percent Bases Shared with Prenatal") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

cor.test(x = prenperc[prenperc$CellType=="Neuron",]$Age, y = prenperc[prenperc$CellType=="Neuron",]$perc)
#t = -3.2655, df = 22, p-value = 0.00354, cor -0.5713691
cor.test(x = prenperc[prenperc$CellType=="Glia",,]$Age, y = prenperc[prenperc$CellType=="Glia",,]$perc)
#t = -2.1369, df = 6, p-value = 0.07648, cor -0.6573872

data.table(prenperc)[,mean(perc),by="CellType"]
#   CellType        V1
#1:     Glia  97.29125
#2:   Neuron  95.51083
#3: Prenatal 100.00000

## Find overlaps with DMRs, features, shared groups

txdb = loadDb("/dcl01/lieber/ajaffe/Amanda/annotation_objects/gencode.v25lift37.annotation.sqlite")
islands = read.table("/dcl01/lieber/ajaffe/Amanda/annotation_objects/cpgIslandExt.hg19.txt", sep="\t", header = T)
rpmsk = read.table("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/RepeatMasker_genomewide_HG19.txt", header=T)
features = c(genes = makeGRangesFromDataFrame(geneMap, keep.extra.columns = T),
             rpmskgr = makeGRangesFromDataFrame(rpmsk, seqnames.field="genoName",start.field="genoStart",end.field="genoEnd",keep.extra.columns=TRUE),
             islands = makeGRangesFromDataFrame(islands, keep.extra.columns = T, start.field = "chromStart", end.field = "chromEnd"),
             promoters = promoters(txdb, upstream=2000, downstream=200))
lapply(features, head)

DMR = lapply(DMR, function(x) x[which(x$sig=="FWER < 0.05"),])
DMRgr = lapply(DMR, function(x) makeGRangesFromDataFrame(x, keep.extra.columns = T))

load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/bumphunting/rda/limma_Neuron_CpGs_minCov_3_ageInfo_dmrs.Rdata")
dmrs = split(dmrs, dmrs$k6cluster_label)
names(dmrs) = c("Gr1","Gr2","Gr3","Gr4","Gr5","Gr6")

oo = lapply(dmvs, function(x) lapply(c(DMRgr[names(DMRgr) %in% c("CellType","Age")], as.list(dmrs), all, features), function(d) findOverlaps(x,d)))

dmvDMR = lapply(dmvs, as.data.frame)
dmvDMR = lapply(dmvDMR, function(x) data.frame(x, rnum = 1:nrow(x)))
dmvDMR = mapply(function(d,oo) data.frame(d, CT = ifelse(d$rnum %in% queryHits(oo$CellType), "CT", "no"),
                                          Age = ifelse(d$rnum %in% queryHits(oo$Age), "Age", "no"),
                                          Gr1 = ifelse(d$rnum %in% queryHits(oo$Gr1), "Gr1", "no"),
                                          Gr2 = ifelse(d$rnum %in% queryHits(oo$Gr2), "Gr2", "no"),
                                          Gr3 = ifelse(d$rnum %in% queryHits(oo$Gr3), "Gr3", "no"),
                                          Gr4 = ifelse(d$rnum %in% queryHits(oo$Gr4), "Gr4", "no"),
                                          Gr5 = ifelse(d$rnum %in% queryHits(oo$Gr5), "Gr5", "no"),
                                          Gr6 = ifelse(d$rnum %in% queryHits(oo$Gr6), "Gr6", "no"),
                                          All = ifelse(d$rnum %in% queryHits(oo$All), "All", "no"),
                                          Prenatal = ifelse(d$rnum %in% queryHits(oo$Prenatal), "Prenatal", "no"),
                                          Postnatal = ifelse(d$rnum %in% queryHits(oo$Postnatal), "Postnatal", "no"),
                                          Neurons = ifelse(d$rnum %in% queryHits(oo$Neurons), "Neurons", "no"),
                                          Glia = ifelse(d$rnum %in% queryHits(oo$Glia), "Glia", "no"),
                                          genes = ifelse(d$rnum %in% queryHits(oo$genes), "Gene", NA),
                                          islands = ifelse(d$rnum %in% queryHits(oo$islands), "CpG-Island", "non-Island"),
                                          promoters = ifelse(d$rnum %in% queryHits(oo$promoters), "promoter", "no")), dmvDMR, oo, SIMPLIFY = F)
dmvDMR = lapply(dmvDMR, function(d) data.frame(d, tog = paste(d$CT, d$Age, d$Gr1,d$Gr2,d$Gr3,d$Gr4,d$Gr5,d$Gr6, d$All, d$Prenatal,
                                                              d$Postnatal, d$Neurons, d$Glia, sep = ":"),
                                               dmr = paste(d$CT, d$Age, d$Gr1,d$Gr2,d$Gr3,d$Gr4,d$Gr5,d$Gr6, sep = ":"),
                                               regionID = paste0(d$seqnames,":",d$start,"-", d$end)))
for (i in 1:length(dmvDMR)) {
  if (length(unique(queryHits(oo[[i]][["rpmskgr"]])))==nrow(dmvDMR[[i]])) {
    dmvDMR[[i]] = data.frame(dmvDMR[[i]][queryHits(oo[[i]][["rpmskgr"]]),],
                             repeats = as.character(features$rpmskgr$repClass)[subjectHits(oo[[i]][["rpmskgr"]])]) } else {
                               dmvDMR[[i]] = rbind(data.frame(dmvDMR[[i]][queryHits(oo[[i]][["rpmskgr"]]),],
                                                              repeats = as.character(features$rpmskgr$repClass)[subjectHits(oo[[i]][["rpmskgr"]])]),
                                                   data.frame(dmvDMR[[i]][-unique(queryHits(oo[[i]][["rpmskgr"]])),], repeats = "No repeats"))
                             }
}

dmvDMRgr = lapply(dmvDMR, function(x) makeGRangesFromDataFrame(x, keep.extra.columns = T))
oo = lapply(dmvDMRgr, function(x) findOverlaps(x, features$genes))
dmvDMR = mapply(function(p, ov) rbind(data.frame(p[queryHits(ov),], nearestSymbol = features$genes$Symbol[subjectHits(ov)],
                                                 nearestID = names(features$genes)[subjectHits(ov)], EntrezID = features$genes$EntrezID[subjectHits(ov)]),
                                      data.frame(p[-unique(queryHits(ov)),], nearestSymbol = "NoGeneOverlap", nearestID = "NoGeneOverlap", EntrezID = "NoGeneOverlap")),
                dmvDMR, oo, SIMPLIFY = F)

postnatalpd = pd
save(dmvDMR, postnatalpd, geneMap, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/dmvs_annotated.rda")

### Annotate to genomic features

load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/dmvs_annotated.rda")

## how many fall within CpG islands?

CpGIslands = lapply(dmvDMR, function(y) round(length(unique(y[which(y$islands=="CpG-Island"),,]$regionID))/length(unique(y$regionID))*100,2))
CpGIslands = data.frame(islands = unlist(CpGIslands), id = names(CpGIslands))
CpGIslands$celltype = ifelse(CpGIslands$id %in% pd$Data.ID, pd[match(CpGIslands$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(CpGIslands, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_CpG_Island_Overlap.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_overlap_with_CpG-Islands.pdf")
ggplot(CpGIslands, aes(x = celltype, y = islands)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Percent") + ylim(0,100) +
  xlab("") +
  ggtitle("Percent Overlapping CpG Islands") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

cgi = data.table(read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_CpG_Island_Overlap.csv"))
cgi[,mean(islands), by = "celltype"]
#   celltype       V1
#1:     Glia 93.37250
#2:   Neuron 90.88583
#3: Prenatal 86.89900
mean(cgi$islands) # 89.735

# How about Repetitive Elements?

dmvDMRdt = lapply(dmvDMR, data.table)
repeats = lapply(dmvDMRdt, function(y) y[,length(unique(regionID)), by = "repeats"])
repeats = lapply(repeats, function(y) data.frame(y, perc = round(y$V1/sum(y$V1)*100,2)))
repeats = do.call(rbind, Map(cbind, repeats, id = as.list(names(repeats))))
repeats$repeats = factor(repeats$repeats, levels = c("SINE","LINE","Simple_repeat","DNA","LTR","Low_complexity","No repeats","Unknown","snRNA",
                                                     "Other","DNA?","Satellite","RC","scRNA","tRNA","SINE?","srpRNA","RNA","rRNA","LINE?","LTR?","Unknown?"))
repeats$celltype = ifelse(repeats$id %in% pd$Data.ID, pd[match(repeats$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(repeats, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_repeats_Overlap.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_overlap_with_repeats.pdf")
x = repeats[which(repeats$repeats %in% c("SINE","LINE","Simple_repeat","DNA","LTR","Low_complexity","No repeats")),]
x$repeats = droplevels(x$repeats)
ggplot(x, aes(x = celltype, y = perc, fill = repeats)) +
  theme_classic() + geom_bar(position = "fill",stat = "identity") +
  labs(fill="") +
  ylab("Percent") +
  xlab("") +
  ggtitle("Percent Overlapping Repeats: DMV") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

repeats = data.table(read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_repeats_Overlap.csv"))
repeats[repeats %in% c("No repeats","SINE","Low_complexity","Simple_repeat","LINE"),mean(perc), by = c("repeats","celltype")]
#           repeats celltype       V1
# 1: Low_complexity     Glia 27.05000
# 2:  Simple_repeat     Glia 25.42625
# 3:           SINE     Glia 19.77125
# 4:           LINE     Glia 14.25000
# 5:     No repeats     Glia  0.13125
# 6:  Simple_repeat   Neuron 25.06875
# 7: Low_complexity   Neuron 26.29000
# 8:           SINE   Neuron 20.25458
# 9:           LINE   Neuron 14.61542
#10:     No repeats   Neuron  0.08250
#11:           LINE Prenatal 17.10400
#12: Low_complexity Prenatal 21.68050
#13:  Simple_repeat Prenatal 20.32200
#14:           SINE Prenatal 21.17250
#15:     No repeats Prenatal  0.04850

## Overlap with DMRs

dmr = lapply(dmvDMRdt, function(y) y[,length(unique(regionID)), by = "dmr"])
dmr = lapply(dmr, function(y) data.frame(y, perc = round(y$V1/sum(y$V1)*100,2)))
models = lapply(dmr, function(y) data.frame(CT = sum(data.frame(y)[grep("CT", y$dmr),"perc"]),
                                            Age = sum(data.frame(y)[grep("Age", y$dmr),"perc"]),
                                            Gr1 = sum(data.frame(y)[grep("Gr1", y$dmr),"perc"]),
                                            Gr2 = sum(data.frame(y)[grep("Gr2", y$dmr),"perc"]),
                                            Gr3 = sum(data.frame(y)[grep("Gr3", y$dmr),"perc"]),
                                            Gr4 = sum(data.frame(y)[grep("Gr4", y$dmr),"perc"]),
                                            Gr5 = sum(data.frame(y)[grep("Gr5", y$dmr),"perc"]),
                                            Gr6 = sum(data.frame(y)[grep("Gr6", y$dmr),"perc"])))
dmr = do.call(rbind, Map(cbind, dmr, id = as.list(names(dmr))))
models = do.call(rbind, Map(cbind, models, id = as.list(names(models))))

dmr$celltype = ifelse(dmr$id %in% pd$Data.ID, pd[match(dmr$id, pd$Data.ID),"Cell.Type"], "Prenatal")
models$celltype = ifelse(models$id %in% pd$Data.ID, pd[match(models$id, pd$Data.ID),"Cell.Type"], "Prenatal")

dmr$NO = ifelse(dmr$dmr=="no:no:no:no:no:no:no:no", "No overlap", "Overlap")

write.csv(dmr, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_dmr_Overlap.csv")
write.csv(models, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_dmr_Overlap_byModel.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_overlap_with_dmr.pdf", width = 14, height = 6)
ggplot(dmr, aes(x = celltype, y = perc, fill = NO)) +
  theme_classic() + geom_bar(position = "fill",stat = "identity") +
  labs(fill="") +
  ylab("Proportion") +
  xlab("") +
  ggtitle("Proportion Overlapping DMRs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(dmr[which(dmr$dmr=="no:no:no:no:no:no:no:no"),], aes(x = celltype, y = perc)) +
  theme_classic() + geom_boxplot() +
  labs(fill="") +
  ylab("Percent") +
  xlab("") +
  ggtitle("Percent Not Overlapping DMRs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(models, aes(x = celltype, y = CT)) +
  theme_classic() + geom_boxplot() +
  labs(fill="") +
  ylab("Percent") +
  xlab("") +
  ggtitle("Percent Overlapping Cell Type DMRs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(models, aes(x = celltype, y = Age)) +
  theme_classic() + geom_boxplot() +
  labs(fill="") +
  ylab("Percent") +
  xlab("") +
  ggtitle("Percent Overlapping Age DMRs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")

x = melt(models[,!colnames(models) %in% c("CT","Age")])
x$variable = gsub("Gr1","1:G-N+", x$variable)
x$variable = gsub("Gr2","2:G0N+", x$variable)
x$variable = gsub("Gr3","3:G0N-", x$variable)
x$variable = gsub("Gr4","4:G+N0", x$variable)
x$variable = gsub("Gr5","5:G+N-", x$variable)
x$variable = gsub("Gr6","6:G-N0", x$variable)

ggplot(x, aes(x = celltype, y = value, fill = variable)) +
  theme_classic() + geom_boxplot() +
  labs(fill="") + facet_grid(. ~ variable) +
  ylab("Percent") +
  xlab("") + theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  scale_fill_brewer(8, palette="Dark2") +
  ggtitle("Percent Overlapping DMRs") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

dmr = data.table(read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_dmr_Overlap.csv"))
models = data.table(read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_dmr_Overlap_byModel.csv"))
models[,list(Gr1=mean(Gr1),Gr2=mean(Gr2),Gr3=mean(Gr3),Gr4=mean(Gr4),Gr5=mean(Gr5),Gr6=mean(Gr6)),by="celltype"]
#   celltype     Gr1       Gr2      Gr3      Gr4     Gr5     Gr6
#1:     Glia 2.71625 1.6100000 0.297500 1.258750 0.31750 1.72125
#2:   Neuron 1.13625 0.7054167 1.562083 1.418333 0.83375 0.32125
#3: Prenatal 1.54000 1.5705000 0.215000 1.360000 0.42350 0.51550

# Are DMRs overlapping DMVs at the DMV periphery, or uniformly distributed?

dmvDMR = Map(cbind, lapply(dmvDMR, function(x) data.frame(x[,1:5], dmr = x$dmr, regionID = x$regionID)), id = as.list(names(dmvDMR)))
dmvDMR = lapply(dmvDMR, unique)
tiles = lapply(dmvDMR, function(x) tile(makeGRangesFromDataFrame(x[which(x$dmr!="no:no:no:no:no:no:no:no"),]), n = 100))
tiles = lapply(tiles, as.list)
ti = list(Prenatal = unlist(tiles[-which(names(tiles) %in% postnatalpd$Data.ID)], recursive = F),
          Neuron = unlist(tiles[which(names(tiles) %in% postnatalpd[which(postnatalpd$Cell.Type=="Neuron"),"Data.ID"])], recursive = F),
          Glia = unlist(tiles[which(names(tiles) %in% postnatalpd[which(postnatalpd$Cell.Type=="Glia"),"Data.ID"])], recursive = F))
DMR = lapply(DMR, function(x) x[which(x$sig=="FWER < 0.05"),])
DMR = lapply(DMR, function(x) reduce(makeGRangesFromDataFrame(x)))
CTov = lapply(ti, function(x) lapply(x, function(y) findOverlaps(y,DMR$CellType)))
Aov = lapply(ti, function(x) lapply(x, function(y) findOverlaps(y,DMR$Age)))
Iov = lapply(ti, function(x) lapply(x, function(y) findOverlaps(y,DMR$Interaction)))

CToverlaps = Ageoverlaps = Intoverlaps = list(list(),list(),list())
for (i in 1:length(ti)) {
  for (j in 1:length(ti[[i]])) {
    if (length(queryHits(CTov[[i]][[j]]))>0) {
      CToverlaps[[i]][[j]] = data.frame(region = queryHits(CTov[[i]][[j]])) } else {
        CToverlaps[[i]][[j]] = data.frame(region = 0) }
    if (length(queryHits(Aov[[i]][[j]]))>0) {
      Ageoverlaps[[i]][[j]] = data.frame(region = queryHits(Aov[[i]][[j]])) } else {
        Ageoverlaps[[i]][[j]] = data.frame(region = 0) }
    if (length(queryHits(Iov[[i]][[j]]))>0) {
      Intoverlaps[[i]][[j]] = data.frame(region = queryHits(Iov[[i]][[j]])) } else {
        Intoverlaps[[i]][[j]] = data.frame(region = 0) }
  }
}
CToverlaps = do.call(rbind, Map(cbind, lapply(CToverlaps, function(x) do.call(rbind, x)), celltype = as.list(names(ti))))
Ageoverlaps = do.call(rbind, Map(cbind, lapply(Ageoverlaps, function(x) do.call(rbind, x)), celltype = as.list(names(ti))))
Intoverlaps = do.call(rbind, Map(cbind, lapply(Intoverlaps, function(x) do.call(rbind, x)), celltype = as.list(names(ti))))
overlaps = rbind(cbind(CToverlaps, model = "Cell Type"), cbind(Ageoverlaps, model = "Age"),
                 cbind(Intoverlaps, model = "Interaction"))

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMR_position_in_DMV.pdf", width = 10)
ggplot(overlaps, aes(region, colour = model)) + geom_density() + theme_classic() +
  labs(fill="") + facet_grid(. ~ celltype) +
  ylab("Density") + xlab("% of DMV") +
  ggtitle("DMR Position in DMV") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.position="bottom", legend.title=element_blank())
dev.off()

## association of existing DMVs with genes

oo = lapply(dmvs, function(x) findOverlaps(makeGRangesFromDataFrame(geneMap), makeGRangesFromDataFrame(x)))
dmvGene = mapply(function(p, ov) if (nrow(p)>length(unique(subjectHits(ov)))) {
  rbind(data.frame(p[subjectHits(ov),], nearestSymbol = geneMap[queryHits(ov),"Symbol"],
                   nearestID = geneMap[queryHits(ov),"gencodeID"],
                   EntrezID = geneMap[queryHits(ov),"EntrezID"]),
        data.frame(p[-unique(subjectHits(ov)),], nearestSymbol = "NoGeneOverlap",
                   nearestID = "NoGeneOverlap", EntrezID = "NoGeneOverlap")) } else {
                     data.frame(p[subjectHits(ov),], nearestSymbol = geneMap[queryHits(ov),"Symbol"],
                                nearestID = geneMap[queryHits(ov),"gencodeID"], EntrezID = geneMap[queryHits(ov),"EntrezID"])
                   },lapply(dmvs, as.data.frame), oo, SIMPLIFY = F)
save(dmvGene, postnatalpd, geneMap, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMVs_geneOverlap.rda")

dmvDMRdt = lapply(dmvGene, data.table)
dmvDMRdt = Map(cbind, dmvDMRdt, regionID = lapply(dmvDMRdt, function(x) paste0(x$seqnames,":",x$start,"-",x$end)))
geneoverlap = lapply(dmvDMRdt, function(y) round(length(unique(as.character(data.frame(y)[which(y$nearestID!="NoGeneOverlap"),"regionID"])))/
                                                 length(unique(as.character(y$regionID)))*100,2))
geneoverlap = data.frame(geneoverlap = unlist(geneoverlap), id = names(geneoverlap))
geneoverlap$celltype = ifelse(geneoverlap$id %in% pd$Data.ID, pd[match(geneoverlap$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(geneoverlap, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_Gene_Overlap.csv")

numgenes = lapply(dmvDMRdt, function(y) y[,length(unique(nearestID)), by = "regionID"])
numgenes = do.call(rbind, Map(cbind, lapply(numgenes, function(y)
           data.frame(mean = mean(y$V1), median = median(y$V1), sd = sd(y$V1), min = min(y$V1), max = max(y$V1))), id = as.list(names(numgenes))))
numgenes$celltype = ifelse(numgenes$id %in% pd$Data.ID, pd[match(numgenes$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(numgenes, quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_numgenes.csv")

geneoverlap = read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_Gene_Overlap.csv")
mean(geneoverlap$geneoverlap)

w = numgenes[which(numgenes$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = postnatalpd[match(w$id, postnatalpd$Data.ID), "Age"])
cor.test(x = w[which(w$celltype=="Neuron"),"Age"], y = w[which(w$celltype=="Neuron"),"mean"])
#t = -0.86357, df = 22, p-value = 0.3971, cor -0.1810702
cor.test(x = w[which(w$celltype=="Glia"),"Age"], y = w[which(w$celltype=="Glia"),"mean"])
#t = -1.0309, df = 6, p-value = 0.3423, cor -0.3879235

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_overlap_with_Genes.pdf")
ggplot(geneoverlap, aes(x = celltype, y = geneoverlap)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Percent") + ylim(0,100) +
  xlab("") +
  ggtitle("Percent Overlapping Genes") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(w, aes(x = Age, y = mean, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") + theme_classic() +
  ylab("Number") + xlab("Age (Years)") +
  ggtitle("Median DMV Gene Number by Age") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(numgenes, aes(x = celltype, y = median)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Number") + xlab("") +
  ggtitle("Median DMV Gene Number") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
ggplot(numgenes, aes(x = celltype, y = mean)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Number") + xlab("") +
  ggtitle("Mean DMV Gene Number") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

library(GenomicFeatures)
txdb = loadDb("/dcl01/lieber/ajaffe/Amanda/annotation_objects/gencode.v25lift37.annotation.sqlite")
promoters = promoters(txdb, upstream=2000, downstream=200)
promoters = reduce(promoters)
oo = lapply(dmvGene, function(x) findOverlaps(promoters, makeGRangesFromDataFrame(x)))
for (i in 1:length(dmvGene)) {
  dmvGene[[i]][unique(subjectHits(oo[[i]])),"promoters"] = "promoters"
  dmvGene[[i]][-unique(subjectHits(oo[[i]])),"promoters"] = "no"
}

dmvGenedt = lapply(dmvGene, data.table)
dmvGenedt = Map(cbind, dmvGenedt, regionID = lapply(dmvGenedt, function(x) paste0(x$seqnames,":",x$start,"-",x$end)))
promoverlap = lapply(dmvGenedt, function(y) round(length(unique(as.character(data.frame(y)[which(y$promoters=="promoters"),"regionID"])))/
                                                 length(unique(as.character(y$regionID)))*100,2))
promoverlap = data.frame(promoverlap = unlist(promoverlap), id = names(promoverlap))
promoverlap$celltype = ifelse(promoverlap$id %in% pd$Data.ID, pd[match(promoverlap$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(promoverlap, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_prom_Overlap.csv")

numproms = lapply(dmvGenedt, function(y) y[promoters=="promoters",length(unique(nearestID)), by = "regionID"])
numproms = do.call(rbind, Map(cbind, lapply(numproms, function(y)
  data.frame(mean = mean(y$V1), median = median(y$V1), sd = sd(y$V1), min = min(y$V1), max = max(y$V1))), id = as.list(names(numproms))))
numproms$celltype = ifelse(numproms$id %in% pd$Data.ID, pd[match(numproms$id, pd$Data.ID),"Cell.Type"], "Prenatal")
write.csv(numproms, quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_numproms.csv")

mean(promoverlap$promoverlap) # 92.38346

w = numproms[which(numproms$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = postnatalpd[match(w$id, postnatalpd$Data.ID), "Age"])
cor.test(x = w[which(w$celltype=="Neuron"),"Age"], y = w[which(w$celltype=="Neuron"),"mean"])
#t = -1.2137, df = 22, p-value = 0.2377, cor -0.2505089
cor.test(x = w[which(w$celltype=="Glia"),"Age"], y = w[which(w$celltype=="Glia"),"mean"])
#t = -1.1737, df = 6, p-value = 0.285, cor -0.4321072

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_overlap_with_proms.pdf")
ggplot(promoverlap, aes(x = celltype, y = promoverlap)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Percent") + ylim(0,100) +
  xlab("") +
  ggtitle("Percent Overlapping Promoters") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(w, aes(x = Age, y = mean, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") + theme_classic() +
  ylab("Number") + xlab("Age (Years)") +
  ggtitle("Median DMV Promoter Number by Age") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
ggplot(numproms, aes(x = celltype, y = median)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Number") + xlab("") +
  ggtitle("Median DMV Promoter Number") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
ggplot(numproms, aes(x = celltype, y = mean)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Number") + xlab("") +
  ggtitle("Mean DMV Promoter Number") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20))
dev.off()

## transcription factor genes in our DMVs?

library(PWMEnrich.Hsapiens.background)
data(MotifDb.Hsap)
tfs = unlist(lapply(MotifDb.Hsap, function(x) x$name))
tfs = tfs[-grep("UW",tfs)]

not = tfs[!tfs %in% geneMap$Symbol]
not = na.omit(not[-grep("::", not, fixed=T)])

nomatch = c("SMCR7L","DUX4","MYF","MZF1_1-4","MZF1_5-13","RORA_1","RORA_2","MIZF","EWSR1-FLI1","ZNF238",
            "ZNF306","POU5F1P1","BHLHB2","BHLHB3","CART1","RAXL1","TRP53","TRP73","ZNF435","HNF1B")
nomatch[which(nomatch %in% geneMap$Symbol)]
nomatch[which(nomatch %in% good)]

tfs = unique(c(tfs[tfs %in% geneMap$Symbol], c("HINFP1","RBM8B"), unique(unlist(strsplit(not[grep("::", not, fixed=T)], "::", fixed=T)))))
tfs = geneMap[which(geneMap$Symbol %in% tfs),] # 830 genes

oo = lapply(dmvs, function(p) findOverlaps(p,makeGRangesFromDataFrame(tfs)))
hits = lapply(oo, function(x) data.frame(tfs = length(unique(subjectHits(x))), dmvs = length(unique(queryHits(x)))))
hits = do.call(rbind, Map(cbind, hits, id = as.list(names(dmvs)), num.dmvs = as.list(elementNROWS(dmvs))))
hits$celltype = ifelse(hits$id %in% pd$Data.ID, pd[match(hits$id, pd$Data.ID),"Cell.Type"], "Prenatal")
hits$DMV.perc = round(hits$dmvs / hits$num.dmvs * 100,1)
hits$TF.perc = round(hits$tfs / nrow(tfs) * 100,1)
write.csv(hits, quote=F,file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_tfsgenes_Overlap.csv")

hits  = read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_tfsgenes_Overlap.csv")
hits = data.table(hits)
hits[,median(tfs),by="celltype"]
#   celltype    V1
#1:     Glia 176.5
#2:   Neuron 183.0
#3: Prenatal 304.0

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_tfsgenes_Overlap.pdf")
ggplot(reshape2::melt(hits)[grep("perc", reshape2::melt(hits)$variable),], aes(x = celltype, y = value)) + geom_boxplot() +
  theme_classic() + facet_grid( ~ variable) +
  labs(fill="") +
  ylab("Percent") +
  xlab("") + theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  ggtitle("Percent Genes and DMVs Overlapping") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

## Enrichment in transcription factor genes

geneuniverse = na.omit(unique(geneMap$gencodeID))
sets = na.omit(unique(tfs$gencodeID))
inDMV = lapply(dmvGene, function(x) na.omit(unique(x$nearestID)))
outDMV = lapply(inDMV, function(x) geneuniverse[!(geneuniverse %in% x)])

DMVenrich = mapply(function(yes, no) {
  DMV_OVERLAP = c( sum( yes %in% sets),sum(!(yes %in% sets)))
  NOT_DMV_OVERLAP= c(sum(no %in% sets), sum(!(no %in% sets)))
  enrich_table = cbind(DMV_OVERLAP, NOT_DMV_OVERLAP)
  res = fisher.test(enrich_table)
  dat=c(res$p.value, res$estimate)
  names(dat) <- c("P.Value","Odds.Ratio")
  return(dat)
}, inDMV, outDMV, SIMPLIFY =F)
DMVenrich = do.call(rbind, Map(cbind, lapply(DMVenrich, function(y) data.frame(P.Value = y["P.Value"], Odds.Ratio = y["Odds.Ratio"])),
                               id = as.list(names(DMVenrich))))
DMVenrich$celltype = ifelse(DMVenrich$id %in% pd$Data.ID, pd[match(DMVenrich$id, pd$Data.ID),"Cell.Type"], "Prenatal")
DMVenrich$FDR = p.adjust(DMVenrich$P.Value, method = "fdr")
DMVenrich$celltype = factor(DMVenrich$celltype, levels = c("Prenatal", "Neuron", "Glia"))
write.csv(DMVenrich,file="/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_tfsgenes_enrichment.csv",quote=F)

DMVenrich = read.csv("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_tfsgenes_enrichment.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_tfsgenes_Enrichment.pdf", height = 4, width = 6)
ggplot(DMVenrich[DMVenrich$FDR<=0.05,], aes(x = celltype, y = Odds.Ratio)) + geom_boxplot() +
  theme_classic() +
  labs(fill="") +
  ylab("Odds Ratio") + ylim(0,15) +
  geom_hline(yintercept=1, linetype="dotted") +
  xlab("") + theme(axis.text.x = element_text(angle = 45, hjust = 1)) +
  ggtitle("Fisher Test for TF Genes in DMVs (FDR<0.05)") +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
dev.off()

## Identify the regions that are getting smaller or larger over age

load("/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/dmvs_annotated.rda")
dmvDMRdt = Map(cbind, lapply(dmvDMR, data.table), tog2 = lapply(dmvDMR, function(d) paste(d$All, d$Prenatal, d$Postnatal, d$Neurons, d$Glia, sep = ":")))
lapply(dmvDMRdt, function(x) x[,length(unique(regionID)),by="tog2"])

subset = do.call(rbind, Map(cbind, lapply(dmvDMRdt, function(x) data.frame(regionID = x$regionID, width = x$width, tog2 = x$tog2)), id = as.list(names(dmvDMRdt))))
subset$celltype = ifelse(subset$id %in% pd$Data.ID, pd[match(subset$id, pd$Data.ID),"Cell.Type"], "Prenatal")
subset = unique(subset)
w = subset[which(subset$celltype %in% c("Neuron", "Glia")),]
w = cbind(w, Age = postnatalpd[match(w$id, postnatalpd$Data.ID), "Age"])
w = split(w, w$tog2)
elementNROWS(w)
#All:Prenatal:Postnatal:Neurons:Glia            9200
#no:no:no:Neurons:no                            3423
#no:no:no:no:Glia                               1746
#no:no:no:no:no                                15341
#no:no:Postnatal:Neurons:Glia                   2756
#no:Prenatal:no:Neurons:no                      1772
#no:Prenatal:no:no:Glia                         1630
#no:Prenatal:no:no:no                           1387
#no:Prenatal:Postnatal:Neurons:Glia                7
#no:Prenatal:no:Neurons:Glia                       1

w = lapply(w, data.table)
mean = lapply(w, function(x) data.frame(x[,mean(width), by = c("id","celltype","Age")]))
med = lapply(w, function(x) data.frame(x[,median(width), by = c("id","celltype","Age")]))

nwidth = lapply(mean[1:9], function(z) cor.test(z[which(z$celltype=="Neuron"),"Age"], z[which(z$celltype=="Neuron"),"V1"]))
gwidth = lapply(mean[1:8], function(z) cor.test(z[which(z$celltype=="Glia"),"Age"], z[which(z$celltype=="Glia"),"V1"]))
mnwidth = lapply(med[1:9], function(z) cor.test(z[which(z$celltype=="Neuron"),"Age"], z[which(z$celltype=="Neuron"),"V1"]))
mgwidth = lapply(med[1:8], function(z) cor.test(z[which(z$celltype=="Glia"),"Age"], z[which(z$celltype=="Glia"),"V1"]))

write.csv(rbind(data.frame(Category = names(nwidth), tstat = unlist(lapply(nwidth, function(x) x$statistic)),
                           parameter = unlist(lapply(nwidth, function(x) x$parameter)), p.value = unlist(lapply(nwidth, function(x) x$p.value)),
                           estimate = unlist(lapply(nwidth, function(x) x$estimate)), CellType = "Neurons", Comparison = "Mean"),
                data.frame(Category = names(gwidth), tstat = unlist(lapply(gwidth, function(x) x$statistic)),
                           parameter = unlist(lapply(gwidth, function(x) x$parameter)), p.value = unlist(lapply(gwidth, function(x) x$p.value)),
                           estimate = unlist(lapply(gwidth, function(x) x$estimate)), CellType = "Glia", Comparison = "Mean"),
                data.frame(Category = names(mnwidth), tstat = unlist(lapply(mnwidth, function(x) x$statistic)),
                           parameter = unlist(lapply(mnwidth, function(x) x$parameter)), p.value = unlist(lapply(mnwidth, function(x) x$p.value)),
                           estimate = unlist(lapply(mnwidth, function(x) x$estimate)), CellType = "Neurons", Comparison = "Median"),
                data.frame(Category = names(mgwidth), tstat = unlist(lapply(mgwidth, function(x) x$statistic)),
                           parameter = unlist(lapply(mgwidth, function(x) x$parameter)), p.value = unlist(lapply(mgwidth, function(x) x$p.value)),
                           estimate = unlist(lapply(mgwidth, function(x) x$estimate)), CellType = "Glia", Comparison = "Median")), quote = F,
          file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/shared_DMVs_correlation.csv")

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/shared_DMV_width_correlation.pdf")
for (i in 1:length(mean)) {
g = ggplot(mean[[i]], aes(x = Age, y = V1, colour = celltype)) + geom_point() +
      geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
      labs(fill="") + theme_classic() +
      ylab("Width") + xlab("Age (Years)") +
      ggtitle(paste0("Mean Width of Overlapping DMVs:\n",names(w)[i])) +
      theme(title = element_text(size = 20)) +
      theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
print(g)
g = ggplot(med[[i]], aes(x = Age, y = V1, colour = celltype)) + geom_point() +
  geom_smooth(method=lm, se=T, fullrange=TRUE) + scale_colour_brewer(8, palette="Dark2") +
  labs(fill="") + theme_classic() +
  ylab("Width") + xlab("Age (Years)") +
  ggtitle(paste0("Median Width of Overlapping DMVs:\n",names(w)[i])) +
  theme(title = element_text(size = 20)) +
  theme(text = element_text(size = 20), legend.title=element_blank()) + theme(legend.position="bottom")
print(g)
}
dev.off()

## Do same comparison but of shared or absent genes in DMVs

inDMV = lapply(inDMV, as.character)
inDMV.byCT = list(Prenatal = unique(unlist(inDMV[-which(names(inDMV) %in% pd$Data.ID)])),
                  Neuron = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Neuron","Data.ID"])])),
                  Glia = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Glia","Data.ID"])])))

elementNROWS(inDMV.byCT)
# Prenatal   Neuron     Glia
#    16320     3556     2735

CTcomps = list(PnotN = inDMV.byCT$Prenatal[-which(inDMV.byCT$Prenatal %in% inDMV.byCT$Neuron)],
               NnotP = inDMV.byCT$Neuron[-which(inDMV.byCT$Neuron %in% inDMV.byCT$Prenatal)],
               sharedPN = inDMV.byCT$Prenatal[which(inDMV.byCT$Prenatal %in% inDMV.byCT$Neuron)],
               PnotG = inDMV.byCT$Prenatal[-which(inDMV.byCT$Prenatal %in% inDMV.byCT$Glia)],
               GnotP = inDMV.byCT$Glia[-which(inDMV.byCT$Glia %in% inDMV.byCT$Prenatal)],
               sharedPG = inDMV.byCT$Prenatal[which(inDMV.byCT$Prenatal %in% inDMV.byCT$Glia)],
               GnotN = inDMV.byCT$Glia[-which(inDMV.byCT$Glia %in% inDMV.byCT$Neuron)],
               NnotG = inDMV.byCT$Neuron[-which(inDMV.byCT$Neuron %in% inDMV.byCT$Glia)],
               sharedGN = inDMV.byCT$Glia[which(inDMV.byCT$Glia %in% inDMV.byCT$Neuron)],
               allshared = Reduce(intersect, inDMV.byCT))
elementNROWS(CTcomps)
#PnotN     NnotP  sharedPN     PnotG     GnotP  sharedPG     GnotN     NnotG
#12940       176      3380     13635        50      2685       618      1439
#sharedGN allshared
#    2117      2105

inDMV.byAgeN = list(infant = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Neuron" & pd$Age<1,"Data.ID"])])),
                   child = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Neuron" & pd$Age>1 & pd$Age<=12,"Data.ID"])])),
                   teen = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Neuron" & pd$Age>12 & pd$Age<=17,"Data.ID"])])),
                   adult = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Neuron" & pd$Age>17,"Data.ID"])])))
elementNROWS(inDMV.byAgeN)
#infant  child   teen  adult
#  2576   2782   2825   2906

AgecompsN = list(InotC = inDMV.byAgeN$infant[-which(inDMV.byAgeN$infant %in% inDMV.byAgeN$child)],
                CnotI = inDMV.byAgeN$child[-which(inDMV.byAgeN$child %in% inDMV.byAgeN$infant)],
                sharedIC = inDMV.byAgeN$infant[which(inDMV.byAgeN$infant %in% inDMV.byAgeN$child)],
                InotT = inDMV.byAgeN$infant[-which(inDMV.byAgeN$infant %in% inDMV.byAgeN$teen)],
                TnotI = inDMV.byAgeN$teen[-which(inDMV.byAgeN$teen %in% inDMV.byAgeN$infant)],
                sharedIT = inDMV.byAgeN$infant[which(inDMV.byAgeN$infant %in% inDMV.byAgeN$teen)],
                InotA = inDMV.byAgeN$infant[-which(inDMV.byAgeN$infant %in% inDMV.byAgeN$adult)],
                AnotI = inDMV.byAgeN$adult[-which(inDMV.byAgeN$adult %in% inDMV.byAgeN$infant)],
                sharedIA = inDMV.byAgeN$infant[which(inDMV.byAgeN$infant %in% inDMV.byAgeN$adult)],
                CnotT = inDMV.byAgeN$child[-which(inDMV.byAgeN$child %in% inDMV.byAgeN$teen)],
                TnotC = inDMV.byAgeN$teen[-which(inDMV.byAgeN$teen %in% inDMV.byAgeN$child)],
                sharedCT = inDMV.byAgeN$child[which(inDMV.byAgeN$child %in% inDMV.byAgeN$teen)],
                CnotA = inDMV.byAgeN$child[-which(inDMV.byAgeN$child %in% inDMV.byAgeN$adult)],
                AnotC = inDMV.byAgeN$adult[-which(inDMV.byAgeN$adult %in% inDMV.byAgeN$child)],
                sharedCA = inDMV.byAgeN$child[which(inDMV.byAgeN$child %in% inDMV.byAgeN$adult)],
                TnotA = inDMV.byAgeN$teen[-which(inDMV.byAgeN$teen %in% inDMV.byAgeN$adult)],
                AnotT = inDMV.byAgeN$adult[-which(inDMV.byAgeN$adult %in% inDMV.byAgeN$teen)],
                sharedTA = inDMV.byAgeN$teen[which(inDMV.byAgeN$teen %in% inDMV.byAgeN$adult)])
elementNROWS(AgecompsN)
#   InotC    CnotI sharedIC    InotT    TnotI sharedIT    InotA    AnotI
#     373      579     2203      368      617     2208      356      686
#sharedIA    CnotT    TnotC sharedCT    CnotA    AnotC sharedCA    TnotA
#    2220      285      328     2497      284      408     2498      234
#AnotT sharedTA
#  315     2591

inDMV.byAgeG = list(infant = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Glia" & pd$Age<1,"Data.ID"])])),
                    child = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Glia" & pd$Age>1 & pd$Age<=12,"Data.ID"])])),
                    teen = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Glia" & pd$Age>12 & pd$Age<=17,"Data.ID"])])),
                    adult = unique(unlist(inDMV[which(names(inDMV) %in% pd[pd$Cell.Type=="Glia" & pd$Age>17,"Data.ID"])])))
elementNROWS(inDMV.byAgeG)
#infant  child   teen  adult
#  1936   2175   1617   2036

AgecompsG = list(InotC = inDMV.byAgeG$infant[-which(inDMV.byAgeG$infant %in% inDMV.byAgeG$child)],
                 CnotI = inDMV.byAgeG$child[-which(inDMV.byAgeG$child %in% inDMV.byAgeG$infant)],
                 sharedIC = inDMV.byAgeG$infant[which(inDMV.byAgeG$infant %in% inDMV.byAgeG$child)],
                 InotT = inDMV.byAgeG$infant[-which(inDMV.byAgeG$infant %in% inDMV.byAgeG$teen)],
                 TnotI = inDMV.byAgeG$teen[-which(inDMV.byAgeG$teen %in% inDMV.byAgeG$infant)],
                 sharedIT = inDMV.byAgeG$infant[which(inDMV.byAgeG$infant %in% inDMV.byAgeG$teen)],
                 InotA = inDMV.byAgeG$infant[-which(inDMV.byAgeG$infant %in% inDMV.byAgeG$adult)],
                 AnotI = inDMV.byAgeG$adult[-which(inDMV.byAgeG$adult %in% inDMV.byAgeG$infant)],
                 sharedIA = inDMV.byAgeG$infant[which(inDMV.byAgeG$infant %in% inDMV.byAgeG$adult)],
                 CnotT = inDMV.byAgeG$child[-which(inDMV.byAgeG$child %in% inDMV.byAgeG$teen)],
                 TnotC = inDMV.byAgeG$teen[-which(inDMV.byAgeG$teen %in% inDMV.byAgeG$child)],
                 sharedCT = inDMV.byAgeG$child[which(inDMV.byAgeG$child %in% inDMV.byAgeG$teen)],
                 CnotA = inDMV.byAgeG$child[-which(inDMV.byAgeG$child %in% inDMV.byAgeG$adult)],
                 AnotC = inDMV.byAgeG$adult[-which(inDMV.byAgeG$adult %in% inDMV.byAgeG$child)],
                 sharedCA = inDMV.byAgeG$child[which(inDMV.byAgeG$child %in% inDMV.byAgeG$adult)],
                 TnotA = inDMV.byAgeG$teen[-which(inDMV.byAgeG$teen %in% inDMV.byAgeG$adult)],
                 AnotT = inDMV.byAgeG$adult[-which(inDMV.byAgeG$adult %in% inDMV.byAgeG$teen)],
                 sharedTA = inDMV.byAgeG$teen[which(inDMV.byAgeG$teen %in% inDMV.byAgeG$adult)])
elementNROWS(AgecompsG)
# InotC    CnotI sharedIC    InotT    TnotI sharedIT    InotA    AnotI
#   300      539     1636      605      286     1331      438      538
# sharedIA    CnotT    TnotC sharedCT    CnotA    AnotC sharedCA    TnotA
#     1498      695      137     1480      408      269     1767      203
# AnotT sharedTA
#   622     1414

## Gene ontology of genes in different categories

entrez = list(CellType = lapply(CTcomps, function(x) na.omit(unique(geneMap[which(geneMap$gencodeID %in% x),"EntrezID"]))),
              Age = lapply(AgecompsN, function(x) na.omit(unique(geneMap[which(geneMap$gencodeID %in% x),"EntrezID"]))),
              Age.Glia = lapply(AgecompsG, function(x) na.omit(unique(geneMap[which(geneMap$gencodeID %in% x),"EntrezID"]))))

# Compare the enriched terms between 7 groups

compareKegg = lapply(entrez, function(x) compareCluster(x, fun="enrichKEGG", qvalueCutoff = 0.05, pvalueCutoff = 0.05))
compareBP = lapply(entrez, function(x) compareCluster(x, fun="enrichGO", ont = "BP", OrgDb = org.Hs.eg.db, qvalueCutoff = 0.05, pvalueCutoff = 0.05))
compareMF = lapply(entrez, function(x) compareCluster(x, fun="enrichGO",  ont = "MF", OrgDb = org.Hs.eg.db, qvalueCutoff = 0.05, pvalueCutoff = 0.05))
compareCC = lapply(entrez, function(x) compareCluster(x, fun="enrichGO",  ont = "CC", OrgDb = org.Hs.eg.db, qvalueCutoff = 0.05, pvalueCutoff = 0.05))
compareDO = lapply(entrez, function(x) compareCluster(x, fun="enrichDO",  ont = "DO", qvalueCutoff = 0.05, pvalueCutoff = 0.05))

# save object
save(compareKegg, compareBP, compareMF, compareCC, compareDO, file="/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_KEGG_GO_DO_objects.rda")

# plot compared results
pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_KEGG_GO_DO_plots.pdf", height = 160, width = 22)
for (i in 1:length(compareKegg)) {
  print(plot(compareKegg[[i]], colorBy="p.adjust", showCategory = 1500, title= paste0("KEGG Pathway Enrichment: ", names(compareKegg)[i])))
  print(plot(compareBP[[i]], colorBy="p.adjust", showCategory = 1500, title= paste0("Biological Process GO Enrichment: ", names(compareKegg)[i])))
  print(plot(compareMF[[i]], colorBy="p.adjust", showCategory = 1500, title= paste0("Molecular Function GO Enrichment: ", names(compareKegg)[i])))
  print(plot(compareCC[[i]], colorBy="p.adjust", showCategory = 1500, title= paste0("Cellular Compartment GO Enrichment: ", names(compareKegg)[i])))
  print(plot(compareDO[[i]], colorBy="p.adjust", showCategory = 1500, title= paste0("Disease Ontology Enrichment: ", names(compareKegg)[i])))
}
dev.off()

## Plot select terms

plotCellType = compareBP$CellType # clusterProfiler output
plotCellType@compareClusterResult = plotCellType@compareClusterResult[-grep("shared", as.character(plotCellType@compareClusterResult$Cluster)),]
plotAge = compareBP$Age # clusterProfiler output
plotAge@compareClusterResult = plotAge@compareClusterResult[-grep("shared", as.character(plotAge@compareClusterResult$Cluster)),]
plotAge.Glia = compareBP$Age.Glia # clusterProfiler output
plotAge.Glia@compareClusterResult = plotAge.Glia@compareClusterResult[-grep("shared", as.character(plotAge.Glia@compareClusterResult$Cluster)),]

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_BP_noshared.pdf", height = 70, width=16)
plot(plotCellType, colorBy="p.adjust", showCategory = 450, title= "Biological Process Enrichment: Cell type")
plot(plotAge, colorBy="p.adjust", showCategory = 450, title= "Biological Process Enrichment: Development")
dev.off()

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_BP_noshared_top10.pdf", height = 16, width=12)
plot(plotCellType, colorBy="p.adjust", showCategory = 10, title= "Biological Process Enrichment: Cell type")
plot(plotAge, colorBy="p.adjust", showCategory = 10, title= "Biological Process Enrichment: Neuronal Development")
plot(plotAge.Glia, colorBy="p.adjust", showCategory = 10, title= "Biological Process Enrichment: Glial Development")
dev.off()

pdf("/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/DMV_BP_noshared_top5.pdf", height = 8, width=12)
plot(plotCellType, colorBy="p.adjust", showCategory = 5, title= "Biological Process Enrichment: Cell type")
plot(plotAge, colorBy="p.adjust", showCategory = 5, title= "Biological Process Enrichment: Neuronal Development")
plot(plotAge.Glia, colorBy="p.adjust", showCategory = 5, title= "Biological Process Enrichment: Glial Development")
dev.off()

DMV.CTcomps = lapply(CTcomps, function(x) geneMap[match(x, geneMap$gencodeID),])
DMV.Agecomps = lapply(Agecomps, function(x) geneMap[match(x, geneMap$gencodeID),])
DMV.AgecompsG = lapply(AgecompsG, function(x) geneMap[match(x, geneMap$gencodeID),])

## Plot venn diagram of genes

library(VennDiagram)
venn.diagram(inDMV.byCT, "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMVgenes_byCellType.jpeg",
             main="Genes within DMVs by Cell Type",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

venn.diagram(inDMV.byAgeN, "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMVgenes_byAge_inNeurons.jpeg",
             main="Genes within DMVs by Age in Neurons",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2", "khaki1"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4", "lightgoldenrod4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

venn.diagram(inDMV.byAgeG, "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMVgenes_byAge_inGlia.jpeg",
             main="Genes within DMVs by Age in Glia",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2", "khaki1"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4", "lightgoldenrod4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

## Plot venn diagram of TF genes

venn.diagram(lapply(inDMV.byCT, function(x) x[which(x %in% tfs$gencodeID)]), "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMV_TFgenes_byCellType.jpeg",
             main="TF Genes within DMVs by Cell Type",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

venn.diagram(lapply(inDMV.byAgeN, function(x) x[which(x %in% tfs$gencodeID)]), "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMV_TFgenes_byAge_inNeurons.jpeg",
             main="TF Genes within DMVs by Age in Neurons",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2", "khaki1"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4", "lightgoldenrod4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

venn.diagram(lapply(inDMV.byAgeG, function(x) x[which(x %in% tfs$gencodeID)]), "/dcl01/lieber/ajaffe/lab/brain-epigenomics/CREs/figures/venn_diagram_DMV_TFgenes_byAge_inGlia.jpeg",
             main="TF Genes within DMVs by Age in Glia",
             col = "transparent",
             fill = c("lightpink2","cornflowerblue", "olivedrab2", "khaki1"),
             cat.col = c("palevioletred4", "darkblue", "olivedrab4", "lightgoldenrod4"),
             alpha = 0.50, fontfamily = "Arial", fontface = "bold", cat.fontfamily = "Arial", margin=0.2)

calculate.overlap(inttargets)

## Which ones are TFs, and what are they doing in terms of our TF analysis?

DMV.CTcomps = Map(cbind, DMV.CTcomps, TF = lapply(DMV.CTcomps, function(x) ifelse(x$gencodeID %in% tfs$gencodeID, "TF","no")))
DMV.Agecomps = Map(cbind, DMV.Agecomps, TF = lapply(DMV.Agecomps, function(x) ifelse(x$gencodeID %in% tfs$gencodeID, "TF","no")))
DMV.AgecompsG = Map(cbind, DMV.AgecompsG, TF = lapply(DMV.AgecompsG, function(x) ifelse(x$gencodeID %in% tfs$gencodeID, "TF","no")))

save(DMV.CTcomps, DMV.Agecomps, DMV.AgecompsG, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_gene_comps.rda")

elementNROWS(lapply(DMV.CTcomps, function(x) x[x$TF=="TF",]))
elementNROWS(lapply(DMV.Agecomps, function(x) x[x$TF=="TF",]))
elementNROWS(lapply(DMV.AgecompsG, function(x) x[x$TF=="TF",]))

## Write the lists for supplementary tables

x = lapply(DMV.CTcomps[-grep("shared", names(DMV.CTcomps))], function(y) y[,c(1:13,31,18,22,26,29)])
x = do.call(rbind, Map(cbind, x, DMV.Group = as.list(names(x))))
colnames(x) = c("Chr","Start","End","Strand","Length","gencodeID","ensemblID","gene_type","Symbol","EntrezID","Class","meanExprs","NumTx","TF","Coeff",
                "Tstat", "pval", "padj","DMV.Group")

y = lapply(DMV.Agecomps[-grep("shared", names(DMV.Agecomps))], function(y) y[,c(1:13,15,17,19,21:22)])
y2 = lapply(DMV.AgecompsG[-grep("shared", names(DMV.AgecompsG))], function(y) y[,c(1:13,15,17,19,21:22)])
y = do.call(rbind, Map(cbind, y, DMV.Group = as.list(names(y))))
y2 = do.call(rbind, Map(cbind, y2, DMV.Group = as.list(names(y2))))
colnames(y) = colnames(y2) = colnames(x)

z = rbind(cbind(x, model = "Cell Type"), cbind(y, model = "Age: Neurons"), cbind(y2, model = "Age: Glia"))
z = z[,colnames(z)!="meanExprs"]
rownames(z) = NULL

write.csv(z[z$TF=="TF",], quote=F, file = "/dcl01/lieber/ajaffe/lab/brain-epigenomics/rdas/CREs/DMV_genes_annotated.csv")

length(unique(z$gencodeID)) # 15514
length(unique(z[z$TF=="TF","gencodeID"])) # 458

## What's the relationship between heterochromatin spreading and TFs in hypo-DMRs?

## Roadmap state in different groups of DMV genes? Compare to parent DMV roadmap state in different groups


BoundModwpt<- function(x, wf="la8", n.levels=4, oldtargets)
{

  N <- length(x); storage.mode(N) <- "integer"
  J <- n.levels
  if(2^J > N) stop("Too many depth levels in the wavelet transform")

  dict <- wave.filter(wf)
  L <- dict$length
  storage.mode(L) <- "integer"
  ht <- dict$hpf/sqrt(2)
  storage.mode(ht) <- "double"
  gt <- dict$lpf/sqrt(2)
  storage.mode(gt) <- "double"

  targets=prepareTargets(oldtargets)#encoding targets using gray code

  y <- vector("list", sum(2^(1:J)))
  yn <- length(y)
  crystals1 <- rep(1:J, 2^(1:J))
  crystals2 <- unlist(apply(as.matrix(2^(1:J) - 1), 1, seq, from=0))
  names(y) <- paste("w", crystals1, ".", crystals2, sep="")

  W <- numeric(N);V<- numeric(N)
  storage.mode(W)  <- "double"; storage.mode(V)<- "double"
  for(j in 1:J) {
    index <- 0
    jj <- min((1:yn)[crystals1 == j])
    for(n in 0:(2^j / 2 - 1)) {
      index <- index + 1
      #should filter parent node j-1,n
      sc=shouldCompute(c(j-1,n),targets)
      if(sum(sc)>0){
        if(j > 1)
          x <- y[[(1:yn)[crystals1 == j-1][index]]]
        if(n %% 2 == 0) {
          z <- .C("pmodwpt", as.double(x), N, as.integer(j),as.integer(2), L, ht, gt,
                  W = W, V = V, PACKAGE="RHRV")[8:9]
          y[[jj + 2*n + 1]] <- z$W
          y[[jj + 2*n]] <- z$V
        }
        else {
          z <- .C("pmodwpt", as.double(x), N, as.integer(j),as.integer(2), L, ht, gt,
                  W = W, V = V, PACKAGE="RHRV")[8:9]
          y[[jj + 2*n]] <- z$W
          y[[jj + 2*n + 1 ]] <- z$V
        }
      }

    }
  }

  return(y)
}

shouldCompute2intCode<-function(sc){
  if (codeEquals(sc,c(1,0))){  intCode=0}
  if (codeEquals(sc,c(0,1))) {  intCode=1}
  if (codeEquals(sc,c(1,1))){intCode=2}
  return (intCode)
}

prepareTargets <- function(targets){
  newTargets=list()
  numberTargets=length(targets)/2;
  for (n in 1:numberTargets){
    newTargets[[n]]=getC(targets[[2*n-1]],targets[[2*n]])
  }
  return(newTargets)
}

shouldCompute <- function(node,targets){

  #compute should be c(0,0)(none), c(1,0)(left),c(0,1)(right), c(1,1) (both)
  compute=c(0,0)
  # node c(0,0) must be computed
  if (nodeEquals(node,c(0,0))){
    len=length(targets)
    for (j in 1:len){
      if (targets[[j]][1]==0){
        compute[[1]]=1;
      }else{
        compute[[2]]=1;
      }
      if (codeEquals(compute,c(1,1))) break
    }

  }else{
    len=length(targets)
    nodeCode=getC(node[[1]],node[[2]])
    codeLen=length(nodeCode)
    compute=c(0,0)
    for (j in 1:len){
      if (length(targets[[j]])>codeLen){
        equals=codeEquals(nodeCode,targets[[j]][1:codeLen])
        if (equals){
          ntc=nodeToCompute(targets[[j]][[codeLen+1]])
          compute=compute+ntc
          compute=compute/max(compute)## avoid (2,0) or (0,2)
          if (codeEquals(compute,c(1,1))) break

        }
      }
    }
    return (compute)
  }

  return(compute);
}

nodeToCompute<-function(number){
  if (number==1){##high pass filter
    return (c(0,1))
  }else{## low pass filter
    return (c(1,0))
  }
}

codeEquals<-function(c1,c2){
  if (length(c1)!=length(c2))
  {
    return (FALSE)
  }else{
    return (prod(c1==c2)==1)
  }
}

nodeEquals<-function(n1,n2){
  return ((n1[1]==n2[1])&&(n1[2]==n2[2]));

}