jtRaw <- function(file = input$raw_file){
  #' Function to read the data from the file / only shiny
  #'
  #' @param file list with stores datapath element containing the path to the file
  #'
  #'
  #' @return Returns table with original data structure

  ext <- tools::file_ext(file$datapath)
  req(file)
  validate(need(ext == "csv", "Please upload a csv file"))

  table <- read.csv(file$datapath,
                   stringsAsFactors = FALSE) %>%
    janitor::clean_names()

  return(table)
}

jtGetLotto <- function(path = "http://www.mbnet.com.pl/dl.txt"){
  #' Function to read the data from the file / only shiny
  #'
  #' @param file web page storing results
  #'
  #' @return Returns table with winning numbers with dates

  data <- read.table(path,
                         sep = " ", stringsAsFactors = FALSE) %>%
    setNames(c("id", "date", "numbers")) %>%
    mutate(date = as.Date(paste(
      substr(date, 7, str_length(date)),
      substr(date, 4, 5),
      substr(date, 1, 2),
      sep = "-"))) %>%
    separate(numbers, paste0("number", 1:6),
             sep = ",")

  message("Data collected from: ", path)

  data <- data %>%
    mutate_at(vars(-date), as.numeric) %>%
    mutate(sum = round(number1 + number2 + number3 + number4 +
                         number5 + number6),
           sum_of_squares =  round(number1^2 + number2^2 + number3^2 +
                                     number4^2 + number5^2 + number6^2)) %>%
    mutate(distance_index = round(sum_of_squares / sum, 2))

  helper <- list()
  helper$prob_tables <- data %>%
    select(number1,
           number2,
           number3,
           number4,
           number5,
           number6) %>%
    pivot_longer(everything()) %>%
    count(value) %>%
    rename(Number = value,
           `Times selected` = n) %>%
    mutate(Probabilty = round(`Times selected`/sum(`Times selected`)*100,4))

  return(list(
    data = data,
    helper = helper
  ))
}

jtClean <- function(table, date, sales, product, min_date, max_date){
  #' Function to recognize and set data structure
  #'
  #' @param table whole data
  #' @param date string with the original name of the column containing dates
  #' @param date string with the original name of the column containing sales
  #' @param date string with the original name of the column containing product names
  #'
  #' @return Returns table with proper data structure

  if(!product){
    table <- table %>%
      rename(Data = parse_character(date),
             Sprzedaz = parse_character(sales)) %>%
      mutate(Data = anydate(Data)) %>%
      filter(Data >= anydate(min_date),
             Data <= anydate(max_date))
  }else{
    table <- table %>%
      rename(Data = parse_character(date),
             Sprzedaz = parse_character(sales),
             `Nazwa produktu` = parse_character(product)) %>%
      mutate(Data = anydate(Data)) %>%
      filter(Data >= anydate(min_date),
             Data <= anydate(max_date))
  }

  return(table)
}

jtGetDesiredLotto <- function(table){
  data <- list()
  data$clean_filtered <- table %>%
    filter(distance_index >= quantile(distance_index, .2) & distance_index <= quantile(distance_index, .8)) %>%
    filter(sum_of_squares >= quantile(sum_of_squares, .2) & sum_of_squares <= quantile(sum_of_squares, .8)) %>%
    filter(sum >= quantile(sum, .2) & sum <= quantile(sum, .8))

  data$random_pos1 <- quantile(data$clean_filtered$number1, .8)
  data$random_pos2 <- quantile(data$clean_filtered$number2, .8)
  data$random_pos3 <- quantile(data$clean_filtered$number3, .8)
  data$random_pos4 <- quantile(data$clean_filtered$number4, .2)
  data$random_pos5 <- quantile(data$clean_filtered$number5, .2)
  data$random_pos6 <- quantile(data$clean_filtered$number6, .2)

  data$clean_filtered_ids <- data$clean_filtered %>%
    filter(number1 <= data$random_pos1,
           number2 <= data$random_pos2,
           number3 <= data$random_pos3,
           number4 >= data$random_pos4,
           number5 >= data$random_pos5,
           number6 >= data$random_pos6) %>%
    pull(id)

  return(data$clean_filtered_ids)
}

jtEnhancets <- function(table, sales = "Sprzedaz"){

  table <- table %>%
    rename(Sprzedaz = parse_character(sales)) %>%
    mutate(lower1 = lag(Sprzedaz, n = 3) - lag(Sprzedaz, n = 1),
           lower2 = lag(Sprzedaz, n = 7) - lag(Sprzedaz, n = 3),
           higher1 = lag(Sprzedaz, n = 1) - lag(Sprzedaz, n = 3),
           higher2 = lag(Sprzedaz, n = 3) - lag(Sprzedaz, n = 7),
           h1 = lag(Sprzedaz) + higher1,
           l1 = lag(Sprzedaz) + lower1,
           h2 = lag(Sprzedaz) + higher2,
           l2 = lag(Sprzedaz) + lower2,
           explanator_h1l2 = round(sqrt(h1*l2), 4),
           explanator_h1h2 = round(sqrt(h1*h2), 4),
           explanator_h1l1 = round(sqrt(h1*l1), 4),
           explanator_h1h1 = round(sqrt(h1*h1), 4),
           explanator_l1l2 = round(sqrt(l1*l2), 4),
           explanator_l1h2 = round(sqrt(l1*h2), 4),
           explanator_l1l1 = round(sqrt(l1*l1), 4)) %>%
    ungroup() %>%
    select(-lower1, -lower2, -higher1, -higher2,
           -h1, -l1, -h2, -l2)

  for (i in 2:6) {
    col_name <- paste0("avg_", str_sub(paste0(0, i),-2,-1))
    table <- table %>%
      mutate(!!col_name := dplyr::lag(RcppRoll::roll_meanr(Sprzedaz, i), 1))
  }

  return(table)
}

jtEnhanceLotto <- function(table){

  model <- list()
  model$raw <- table %>%
    select(-id)

  model$raw[nrow(model$raw)+1,] <- NA
  model$raw <- model$raw %>%
    mutate(date = ifelse(!is.na(date),
                         date,
                         as.Date("2021-02-11")),
           date = anytime::anydate(date))

  model$data_full <- model$raw %>%
    tk_augment_timeseries_signature(.date_var = date) %>%
    select(-half, -hour, -minute, -second, -hour12, -am.pm, -week2, -mday, -week.iso,
           -wday.xts, -qday, -yday, -index.num, -year.iso, -month.xts,
           -wday.lbl, -month.lbl, -date, -diff)

  model$data <- model$data_full %>%
    filter(complete.cases(.))

  return(model)
}

jtTotals <- function(table){
  table <- table %>%
    group_by(Data) %>%
    summarise(Sprzedaz = sum(Sprzedaz, na.rm = TRUE),
              .groups = "drop") %>%
    arrange(Data) %>%
    mutate(Sprzedaz = ifelse(Sprzedaz <= 0, .00001, Sprzedaz)) %>%
    mutate(`Zmiana w stosunku do okresu poprzedniego [%]` = round((Sprzedaz / lag(Sprzedaz) - 1) *100, 2)) %>%
    jtEnhancets() %>%
    timetk::tk_augment_timeseries_signature(.date_var = Data)
  return(table)
}

jtSeasons <- function(table){
  table <- table %>%
    group_by(year) %>%
    mutate(year_mean = mean(Sprzedaz, na.rm = TRUE)) %>%
    ungroup() %>%
    group_by(month.lbl) %>%
    summarise(`Srednia miesieczna sprzedaz` = round(mean(Sprzedaz, na.rm = TRUE), 2),
              `Liczba lat` = n(),
              `Calkowita wartosc` = sum(Sprzedaz, na.rm = TRUE),
              `Srednie odchylenie` = round(mean((Sprzedaz / year_mean - 1), na.rm = TRUE)*100, 2),
              .groups = "drop") %>%
    rename(Miesiac = month.lbl)
  return(table)
}

jtXGBoost.sales <- function(dane, horizon){
  h <- 24

  set.seed(777)
  xgbData <- list()
  xgbData$df_test_complete <- dane
  xgbData$seasons <-  dane %>%
    jtTotals() %>%
    jtSeasons() %>%
    select(Miesiac, `Srednie odchylenie`) %>%
    clean_names()

  df <- xgbData$df_test_complete

  # add forecast rows (empty for now)

  df <- df %>%
    dplyr::mutate(horizon = "actual")

  cross_date <- na.omit(unique(c(as.Date(dane$Data), as.Date(max(dane$Data, na.rm = TRUE)) + months(1:h))))

  df1 <- df %>%
    group_by(Data, horizon) %>%
    summarise(Sprzedaz = sum(Sprzedaz, na.rm = TRUE),
              .groups = "drop") %>%
    as.data.frame()
  df1[nrow(df1)+h,] <- NA

  df2 <- df1 %>%
    mutate(Data = cross_date,
           Data = anydate(Data)) %>%
    mutate(horizon = ifelse(is.na(horizon),
                            "future", horizon)) %>%
    jtEnhancets() %>%
    timetk::tk_augment_timeseries_signature(.date_var = Data) %>%
    left_join(xgbData$seasons,
              by = c("month.lbl" = "miesiac"))

  xgbData$full_data <- df2 %>%
    dplyr::select(Data, Sprzedaz, explanator_h1l1,
                  explanator_h1h2, explanator_h1l2,
                  explanator_h1h1, explanator_l1l2,
                  explanator_l1h2, explanator_l1l1,
                  avg_02, avg_03, avg_05, half,
                  quarter, month, week, horizon,
                  srednie_odchylenie)

  xgbData$train_base <- xgbData$full_data %>%
    dplyr::select(-horizon) %>%
    filter(complete.cases(.))
  message("xgbData prepared.")

  xgbData$trainMatrix <- Matrix::sparse.model.matrix(Sprzedaz ~
                                                       Data + explanator_h1l1 +
                                                       explanator_h1h2 +
                                                       explanator_h1l2 +
                                                       explanator_h1h1 +
                                                       explanator_l1l2 +
                                                       explanator_l1h2 +
                                                       explanator_l1l1 +
                                                       avg_03 + avg_05 +
                                                       half + quarter +
                                                       srednie_odchylenie +
                                                       week,
                                                     data = xgbData$train_base,
                                                     contrasts.arg = c("srednie_odchylenie"),
                                                     sparse = FALSE,
                                                     sci = FALSE
  )

  xgbData$label <- xgbData$train_base$Sprzedaz
  xgbData$trainDMatrix <- xgboost::xgb.DMatrix(data = xgbData$trainMatrix,
                                               label = xgbData$label)

  # prepare model
  xgbData$params <- list(booster = "gbtree",
                         objective = "reg:linear",
                         eta = 0.1,
                         gamma = 0)

  xgbData$xgb.tab <- xgboost::xgb.cv(data = xgbData$trainDMatrix,
                                     param = xgbData$params,
                                     maximize = FALSE,
                                     evaluation = "rmse",
                                     nrounds = 70,
                                     nthreads = 7,
                                     nfold = 3,
                                     early_stopping_round = 5)
  xgbData$num_iterations <- xgbData$xgb.tab$best_iteration

  # train model
  xgbData$model <- xgboost::xgb.train(data = xgbData$trainDMatrix,
                                      maximize = FALSE,
                                      evaluation = 'rmse',
                                      nrounds = xgbData$num_iterations)

  message("Model trained.")
  # forecast
  xgbData$forecasts <- xgbData$full_data

  for (i in which(xgbData$forecasts$horizon == 'future')){

    test_xgb <- xgbData$forecasts[i, ]

    test_xgb <- test_xgb %>%
      dplyr::select(-horizon) %>%
      mutate(Sprzedaz = 0)

    testMatrix <- Matrix::sparse.model.matrix(Sprzedaz ~
                                                Data + explanator_h1l1 +
                                                explanator_h1h2 +
                                                explanator_h1l2 +
                                                explanator_h1h1 +
                                                explanator_l1l2 +
                                                explanator_l1h2 +
                                                explanator_l1l1 + avg_03 +
                                                avg_05 + half +
                                                quarter + srednie_odchylenie +
                                                week,
                                              data = test_xgb,
                                              contrasts.arg = c("srednie_odchylenie"),
                                              sparse = FALSE,
                                              sci = FALSE
    )

    xgbData$forecasts$Sprzedaz[i] <- predict(xgbData$model, testMatrix)

    # re-calculate rolling statistics
    xgbData$forecasts <- xgbData$forecasts %>%
      jtEnhancets() %>%
      dplyr::select(Data, Sprzedaz, explanator_h1l1,
                    explanator_h1h2, explanator_h1l2,
                    explanator_h1h1, explanator_l1l2,
                    explanator_l1h2, explanator_l1l1,
                    avg_02, avg_03, avg_05, half,
                    quarter, srednie_odchylenie, week, horizon)

  }

  return(xgbData)
}

jtXGBoost.lotto <- function(model, dependent_var){

  lotto <- read.csv("lotto.csv")
  set.seed(sample(10000, 1))
  xgbData <- list()
  xgbData$full_data <- model
  xgbData$train_base <- model %>%
    filter(complete.cases(.)) %>%
    select(-id)

  xgbData$trainMatrix <-
    Matrix::sparse.model.matrix(
      as.formula(paste(parse_character(dependent_var), "~ .")),
      data = xgbData$train_base,
      sparse = FALSE,
      sci = FALSE
    )

  xgbData$label <- xgbData$train_base %>%
    select(parse_character(dependent_var)) %>%
    pull()

  xgbData$trainDMatrix <- xgboost::xgb.DMatrix(data = xgbData$trainMatrix,
                                               label = xgbData$label)

  # prepare model
  xgbData$params <- list(booster = "dart",
                         objective = "reg:linear",
                         eta = 0.05,
                         gamma = 0.2,
                         sampling_method = "gradient_based",
                         tree_method = "exact",
                         max_delta_step = 7,
                         max_depth = 7)

  xgbData$xgb.tab <- xgboost::xgb.cv(data = xgbData$trainDMatrix,
                                     param = xgbData$params,
                                     maximize = FALSE,
                                     evaluation = "rmse",
                                     nrounds = 13,
                                     nthreads = 3,
                                     nfold = 5,
                                     early_stopping_round = 2)
  xgbData$num_iterations <- xgbData$xgb.tab$best_iteration

  # train model
  xgbData$model <- xgboost::xgb.train(data = xgbData$trainDMatrix,
                                      maximize = FALSE,
                                      evaluation = 'rmse',
                                      nrounds = xgbData$num_iterations)

  # forecast
  xgbData$forecasts <- data.frame()

  for (i in which(is.na(xgbData$full_data$sum))){

    test_xgb <- xgbData$full_data[i, ]
    test_lotto <- lotto[sample(1:nrow(lotto), 1),]

    test_xgb <- test_xgb %>%
      select(-id) %>%
      mutate_at(vars(parse_character(dependent_var)), function(x) 0) %>%
      mutate(sum = test_lotto$sum,
             sum_of_squares = test_lotto$sum_of_squares,
             distance_index = test_lotto$distance_index)

    testMatrix <- Matrix::sparse.model.matrix(as.formula(paste(parse_character(dependent_var), "~ .")),
                                              data = test_xgb,
                                              sparse = FALSE,
                                              sci = FALSE
    )

    prediction <- predict(xgbData$model, testMatrix)

    test_xgb <- test_xgb %>%
      mutate_at(vars(parse_character(dependent_var)), function(x) round(prediction))

    xgbData$forecasts <- rbind(xgbData$forecasts, test_xgb)
  }

  return(xgbData)
}

predictLotto <- function(data){

  model1 <- data %>%
    select(-number2, -number3, -number4,
           -number5, -number6)
  result1 <- jtXGBoost.lotto(model1, "number1")$forecasts

  message(paste("Number 1 predicted", result1$number1))

  model2 <- data %>%
    select(-number3, -number4,
           -number5, -number6) %>%
    mutate(number1 = ifelse(is.na(number1),
                            result1$number1,
                            number1))
  result2 <- jtXGBoost.lotto(model2, "number2")$forecasts
  message(paste("Number 2 predicted", result2$number2))

  model3 <- data %>%
    select(-number4,
           -number5, -number6) %>%
    mutate(number1 = ifelse(is.na(number1),
                            result1$number1,
                            number1),
           number2 = ifelse(is.na(number2),
                            result2$number2,
                            number2))
  result3 <- jtXGBoost.lotto(model3, "number3")$forecasts
  message(paste("Number 3 predicted", result3$number3))

  model4 <- data %>%
    select(-number5, -number6) %>%
    mutate(number1 = ifelse(is.na(number1),
                            result1$number1,
                            number1),
           number2 = ifelse(is.na(number2),
                            result2$number2,
                            number2),
           number3 = ifelse(is.na(number3),
                            result3$number3,
                            number3))
  result4 <- jtXGBoost.lotto(model4, "number4")$forecasts
  message(paste("Number 4 predicted", result4$number4))

  model5 <- data %>%
    select(-number6) %>%
    mutate(number1 = ifelse(is.na(number1),
                            result1$number1,
                            number1),
           number2 = ifelse(is.na(number2),
                            result2$number2,
                            number2),
           number3 = ifelse(is.na(number3),
                            result3$number3,
                            number3),
           number4 = ifelse(is.na(number4),
                            result4$number4,
                            number4))
  result5 <- jtXGBoost.lotto(model5, "number5")$forecasts
  message(paste("Number 5 predicted", result5$number5))

  model6 <- data %>%
    mutate(number1 = ifelse(is.na(number1),
                            result1$number1,
                            number1),
           number2 = ifelse(is.na(number2),
                            result2$number2,
                            number2),
           number3 = ifelse(is.na(number3),
                            result3$number3,
                            number3),
           number4 = ifelse(is.na(number4),
                            result4$number4,
                            number4),
           number5 = ifelse(is.na(number5),
                            result5$number5,
                            number5))
  result6 <- jtXGBoost.lotto(model6, "number6")$forecasts %>%
    mutate(number6 = ifelse(number6 > 49, 49, number6))
  message(paste("Number 6 predicted", result6$number6))

  return(result6)

}

jtTheme <- function(base_size = 13,
                    base_family = "",
                    base_line_size = base_size/20,
                    base_rect_size = base_size/20) {
  theme_light(
    base_size = base_size,
    base_family = base_family,
    base_line_size = base_line_size,
    base_rect_size = base_rect_size) +
    theme(panel.background = element_rect(fill = "#14113D",
                                          colour = "#14113D"),
          plot.background = element_rect(fill = "#14113D",
                                         colour = "#14113D"),
          panel.border = element_rect(fill = NA, colour = NA),
          panel.grid = element_line(colour = "#D9D9D9"),
          panel.grid.minor = element_line(size = rel(0.5)),
          strip.background = element_rect(fill = "#D9D9D9",
                                          colour = "#D9D9D9"),
          title = element_text(color = "329E99",
                               family = "Titillium Web"),
          legend.key = element_rect(fill = "#14113D", colour = NA),
          axis.title.x = element_text(color = "#329E99",
                                      size = base_size,
                                      family = base_family,
                                      face = "bold"),
          axis.title.y = element_text(color = "#329E99",
                                      size = base_size,
                                      family = base_family,
                                      face = "bold"),
          axis.text.x = element_text(color = "#329E99",
                                     size = base_size,
                                     family = base_family,
                                     face = "bold"),
          axis.text.y = element_text(color = "#329E99",
                                     size = base_size,
                                     family = base_family,
                                     face = "bold"),
          legend.background = element_rect(fill = "#14113D",
                                           color = "#071561"),
          legend.text = element_text(color = "#329E99"),
          complete = TRUE
    )
}


BiocManager::install("sva")

library(sva)
cdata <- read.xlsx("差异蛋白筛选结果.xlsx", sheet = "可信蛋白")
csif <- read.xlsx("sif.xlsx")#读取批次信息

modcombat = model.matrix(~1, data = csif)
batch = csif$batch
combat_edata = ComBat(dat=cdata[,-1:-3], batch=batch, mod=modcombat,par.prior=TRUE, prior.plots=FALSE)
#去除批次效应
write.table(combat_edata, "ComBat_data(all)(counts).txt", sep = "\t", quote = F,row.names = F)


##### Differential analysis of ADRC microarray data

### Load packages anda data for differential analysis
set.seed(1234)
library(limma)
library(minfi)
library(IlluminaHumanMethylationEPICanno.ilm10b2.hg19)
library(IlluminaHumanMethylationEPICanno.ilm10b4.hg19)
library(RColorBrewer)
library(missMethyl)
library(Gviz)
library(stringr)
library(NHMMfdr)
library(sva)

load("adrcPreprocessedData.RData")
ls()

### set variables for making models
# only groupwise model shown below

gran <- as.numeric(targets$Gran)
mono <- as.numeric(targets$Mono)
nk <- as.numeric(targets$NK)
bcell <- as.numeric(targets$Bcell)
cd8 <- as.numeric(targets$CD8T)
cd4 <- as.numeric(targets$CD4T)
sex <- as.factor(targets$sex)
age <- as.numeric(as.character(targets$age))
group <- as.factor(targets$cohort)
slide <- as.factor(targets$chip_id)

### generate model and identify surrogate variables

mod <- model.matrix(~ group + sex + age + slide + gran + nk + mono + bcell + cd8 + cd4)
mod0 <- model.matrix(~ sex + age + slide + gran + nk + mono + bcell + cd8 + cd4)

n.sv <- num.sv(mValues,mod,method='leek')
svobj <- sva(mValues,mod,mod0,n.sv=n.sv)
modSv <- cbind(mod,svobj$sv)

### basic linear fit using limma

# get suset of informatino of interest from annotation

annEPICSub <- annEPIC[match(rownames(mValues),annEPIC$Name),c(1:4,22,24,19)]

fit <- lmFit(mValues,modSv)
fit2 <- eBayes(fit)
DMPs <- topTable(fit2,coef=2,num=Inf,sort.by='none',genelist=annEPICSub)

### get lambda to check fit for inflation

lambda <- median(qchisq(1-DMPs$P.Value,1))/qchisq(0.5,1)
lambda

### generate z-scores for NHMMfdr

z <- qnorm(1-DMPs$P.Value)

### fit NHMMfdr

fitNHMM <- fdr.nhmm(z,alttype="mixnormal",L=1)
LIS.adjust <- LIS.adjust(fitNHMM$LIS,fdr=0.05,adjust=TRUE)
DMPs <- cbind(DMPs,LIS.adjust$States,LIS.adjust$aLIS)
length(which(LIS.adjust$States==1))

fwrite(DMPs,file="adrcLOAD.DMPs.csv",quote=F,row.names=T)
save(DMPs,file="adrcLOAD.DMPs.RData")

### repeat analysis with other variables of interest


# 1. Simulate random mutations ###########################################################################################################
tot_size= sum(225584828,204787373,185818997,172585720,
            190429646,180051392,169600520,144306982,
            129882849,92844088,133663169,125506784,
            108979918,127894412,111343173,77216781,
            95684472,70235451,53671032,74971481)

size1 <- 225584828
size2 <- size1 + 204787373
size3 <- size2 + 185818997
size4 <- size3 + 172585720
size5 <- size4 + 190429646
size6 <- size5 + 180051392
size7 <- size6 + 169600520
size8 <- size7 + 144306982
size9 <- size8 + 129882849
size10 <- size9 + 92844088
size11 <- size10 + 133663169
size12 <- size11 + 125506784
size13 <- size12 + 108979918
size14 <- size13 + 127894412
size15 <- size14 + 111343173
size16 <- size15 + 77216781
size17 <- size16 + 95684472
size18 <- size17 + 70235451
size19 <- size18 + 53671032
size20 <- size19 + 74971481

# 10 simulations
for(s in 1:10){
  fake_mut=runif(663, 1, tot_size)
  fake_mut_1=fake_mut[which(fake_mut>=1 & fake_mut<=size1)]
  fake_mut_2=fake_mut[which(fake_mut>size1 & fake_mut<=size2)]
  fake_mut_3=fake_mut[which(fake_mut>size2 & fake_mut<=size3)]
  fake_mut_4=fake_mut[which(fake_mut>size3 & fake_mut<=size4)]
  fake_mut_5=fake_mut[which(fake_mut>size4 & fake_mut<=size5)]
  fake_mut_6=fake_mut[which(fake_mut>size5 & fake_mut<=size6)]
  fake_mut_7=fake_mut[which(fake_mut>size6 & fake_mut<=size7)]
  fake_mut_8=fake_mut[which(fake_mut>size7 & fake_mut<=size8)]
  fake_mut_9=fake_mut[which(fake_mut>size8 & fake_mut<=size9)]
  fake_mut_10=fake_mut[which(fake_mut>size9 & fake_mut<=size10)]
  fake_mut_11=fake_mut[which(fake_mut>size10 & fake_mut<=size11)]
  fake_mut_12=fake_mut[which(fake_mut>size11 & fake_mut<=size12)]
  fake_mut_13=fake_mut[which(fake_mut>size12 & fake_mut<=size13)]
  fake_mut_14=fake_mut[which(fake_mut>size13 & fake_mut<=size14)]
  fake_mut_15=fake_mut[which(fake_mut>size14 & fake_mut<=size15)]
  fake_mut_16=fake_mut[which(fake_mut>size15 & fake_mut<=size16)]
  fake_mut_17=fake_mut[which(fake_mut>size16 & fake_mut<=size17)]
  fake_mut_18=fake_mut[which(fake_mut>size17 & fake_mut<=size18)]
  fake_mut_19=fake_mut[which(fake_mut>size18 & fake_mut<=size19)]
  fake_mut_20=fake_mut[which(fake_mut>size19 & fake_mut<=size20)]

  fake_pos_list <- c("fake_mut_1","fake_mut_2","fake_mut_3","fake_mut_4","fake_mut_5","fake_mut_6","fake_mut_7","fake_mut_8","fake_mut_9","fake_mut_10","fake_mut_11","fake_mut_12","fake_mut_13","fake_mut_14","fake_mut_15","fake_mut_16","fake_mut_17","fake_mut_18","fake_mut_19","fake_mut_20")

  for(j in fake_pos_list){
      distance=vector()
      sorted=sort(get(j))
      for(i in 2:length(get(j))){
          dist=sorted[i]-sorted[i-1]
          distance=cbind(distance,as.numeric(dist))
      }
      assign(paste0("distance_",j),distance)
  }

  distance_all_fake=c(distance_fake_mut_1,distance_fake_mut_2,distance_fake_mut_3,distance_fake_mut_4,distance_fake_mut_5,distance_fake_mut_6,distance_fake_mut_7,distance_fake_mut_8,distance_fake_mut_9,distance_fake_mut_10,distance_fake_mut_11,distance_fake_mut_12,distance_fake_mut_13,distance_fake_mut_14,distance_fake_mut_15,distance_fake_mut_16,distance_fake_mut_17,distance_fake_mut_18,distance_fake_mut_19,distance_fake_mut_20)

  # Create a table of fake mut:
  line1=cbind("chr1", as.integer(fake_mut_1))
  line2=cbind("chr2", as.integer(fake_mut_2-size1))
  line3=cbind("chr3", as.integer(fake_mut_3-size2))
  line4=cbind("chr4", as.integer(fake_mut_4-size3))
  line5=cbind("chr5", as.integer(fake_mut_5-size4))
  line6=cbind("chr6", as.integer(fake_mut_6-size5))
  line7=cbind("chr7", as.integer(fake_mut_7-size6))
  line8=cbind("chr8", as.integer(fake_mut_8-size7))
  line9=cbind("chr9", as.integer(fake_mut_9-size8))
  line10=cbind("chr10", as.integer(fake_mut_10-size9))
  line11=cbind("chr11", as.integer(fake_mut_11-size10))
  line12=cbind("chr12", as.integer(fake_mut_12-size11))
  line13=cbind("chr13", as.integer(fake_mut_13-size12))
  line14=cbind("chr14", as.integer(fake_mut_14-size13))
  line15=cbind("chr15", as.integer(fake_mut_15-size14))
  line16=cbind("chr16", as.integer(fake_mut_16-size15))
  line17=cbind("chr17", as.integer(fake_mut_17-size16))
  line18=cbind("chr18", as.integer(fake_mut_18-size17))
  line19=cbind("chr19", as.integer(fake_mut_19-size18))
  line20=cbind("chr20", as.integer(fake_mut_20-size19))

  table=rbind(line1,line2,line3,line4,line5,line6,line7,line8,line9,line10,line11,line12,line13,line14,line15,line16,line17,line18,line19,line20)
  colnames(table) <- c("chrom", "pos")
  write.table(table, paste0("fake_mut_",s,".txt"),row.names=FALSE)
}

# 2. Distance between mutation and qq plot #################################################################################################################
library(plyr) # to have count()
# Import data from detection of de novo candidates
data_R16089 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16089.tab", sep ="\t")
data_R16053 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16053.tab", sep ="\t")
data_R16062 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16062.tab", sep ="\t")
data_R15089 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R15089.tab", sep ="\t")
data_R15107 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R15107.tab", sep ="\t")
data_R15123 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R15123.tab", sep ="\t")
data_R16034 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16034.tab", sep ="\t")
data_R15140 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R15140.tab", sep ="\t")
data_R14024 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R14024.tab", sep ="\t")
data_R12006 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R12006.tab", sep ="\t")
data_R13024 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R13024.tab", sep ="\t")
data_R13074 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R13074.tab", sep ="\t")
data_R13023 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R13023.tab", sep ="\t")
data_R16100 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16100.tab", sep ="\t")
data_R17015 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R17015.tab", sep ="\t")
data_R17061 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R17061.tab", sep ="\t")
data_R17032 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R17032.tab", sep ="\t")
data_R17037 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R17037.tab", sep ="\t")
data_R16104 = read.csv("/home/lucie/MammalianMutation/faststorage/Macaca_mulatta_33/de_novo_mutation/data_denovo_R16104.tab", sep ="\t")
# Sample:
name="all"
# direct = 'directory with the candidate mutations'

chr_size= c(225584828, 204787373,185818997,172585720,
            190429646,180051392,169600520,144306982,
            129882849,92844088,133663169,125506784,
            108979918,127894412,111343173,77216781,
            95684472,70235451,53671032,74971481)
tot=chr_size[1]+chr_size[2]+chr_size[3]+chr_size[4]+chr_size[5]+chr_size[6]+chr_size[7]+chr_size[8]+chr_size[9]+chr_size[10]+chr_size[11]+chr_size[12]+chr_size[13]+chr_size[14]+chr_size[15]+chr_size[16]+chr_size[17]+chr_size[18]+chr_size[19]+chr_size[20]
lab_name=seq(1,20,1)

distance_between_indiv = vector()
distance_all=list()
list_indiv = c("data_R16089", "data_R16053", "data_R16062", "data_R15089", "data_R15107", "data_R15123", "data_R16034", "data_R15140", "data_R14024", "data_R12006", "data_R13024", "data_R13074", "data_R13023", "data_R16100", "data_R17015", "data_R17061", "data_R17032", "data_R17037", "data_R16104")

for(indiv in list_indiv){
    data_denovo=get(indiv)[,1:8]

# Find position per chromosomes
pos_chr1=vector()
pos_chr2=vector()
pos_chr3=vector()
pos_chr4=vector()
pos_chr5=vector()
pos_chr6=vector()
pos_chr7=vector()
pos_chr8=vector()
pos_chr9=vector()
pos_chr10=vector()
pos_chr11=vector()
pos_chr12=vector()
pos_chr13=vector()
pos_chr14=vector()
pos_chr15=vector()
pos_chr16=vector()
pos_chr17=vector()
pos_chr18=vector()
pos_chr19=vector()
pos_chr20=vector()
for(i in seq(1,nrow(data_denovo))){
  if(data_denovo[i,1]=='chr1'){pos_chr1=c(pos_chr1,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr2'){pos_chr2=c(pos_chr2,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr3'){pos_chr3=c(pos_chr3,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr4'){pos_chr4=c(pos_chr4,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr5'){pos_chr5=c(pos_chr5,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr6'){pos_chr6=c(pos_chr6,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr7'){pos_chr7=c(pos_chr7,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr8'){pos_chr8=c(pos_chr8,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr9'){pos_chr9=c(pos_chr9,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr10'){pos_chr10=c(pos_chr10,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr11'){pos_chr11=c(pos_chr11,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr12'){pos_chr12=c(pos_chr12,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr13'){pos_chr13=c(pos_chr13,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr14'){pos_chr14=c(pos_chr14,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr15'){pos_chr15=c(pos_chr15,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr16'){pos_chr16=c(pos_chr16,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr17'){pos_chr17=c(pos_chr17,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr18'){pos_chr18=c(pos_chr18,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr19'){pos_chr19=c(pos_chr19,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr20'){pos_chr20=c(pos_chr20,data_denovo[i,2])
  }
}

# Find distance for each chromosomes
pos_list <- c("pos_chr1","pos_chr2","pos_chr3","pos_chr4","pos_chr5","pos_chr6","pos_chr7","pos_chr8","pos_chr9","pos_chr10","pos_chr11","pos_chr12","pos_chr13","pos_chr14","pos_chr15","pos_chr16","pos_chr17","pos_chr18","pos_chr19","pos_chr20")
rm(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr12,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)
for(j in pos_list){
    distance=vector()
    sorted=sort(get(j))
    for(i in 2:length(get(j))){
        dist=sorted[i]-sorted[i-1]
        distance=c(distance,dist)
    }
    assign(paste0("distance_",j),distance)
}

distance_all=c(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr13,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)
distance_all <- distance_all[!is.na(distance_all)]

distance_between_indiv=c(distance_between_indiv,distance_all)
}
distance_within_same_ind = distance_between_indiv

# Between related individuals 2 by 2 #################################################################
data_r1=cbind(rbind(data_R14024[,1:2], data_R12006[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R12006', nrow(data_R12006))))
data_r2=cbind(rbind(data_R14024[,1:2], data_R13024[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R13024', nrow(data_R13024))))
data_r3=cbind(rbind(data_R14024[,1:2], data_R16100[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16100', nrow(data_R16100))))
data_r4=cbind(rbind(data_R14024[,1:2], data_R13074[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R13074', nrow(data_R13074))))
data_r5=cbind(rbind(data_R14024[,1:2], data_R13023[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R13023', nrow(data_R13023))))
data_r6=cbind(rbind(data_R12006[,1:2], data_R13024[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R13024', nrow(data_R13024))))
data_r7=cbind(rbind(data_R12006[,1:2], data_R16100[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16100', nrow(data_R16100))))
data_r8=cbind(rbind(data_R12006[,1:2], data_R13074[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R13074', nrow(data_R13074))))
data_r9=cbind(rbind(data_R12006[,1:2], data_R13023[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R13023', nrow(data_R13023))))
data_r10=cbind(rbind(data_R13024[,1:2], data_R16100[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16100', nrow(data_R16100))))
data_r11=cbind(rbind(data_R13024[,1:2], data_R13074[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R13074', nrow(data_R13074))))
data_r12=cbind(rbind(data_R13024[,1:2], data_R13023[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R13023', nrow(data_R13023))))
data_r13=cbind(rbind(data_R16100[,1:2], data_R13074[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R13074', nrow(data_R13074))))
data_r14=cbind(rbind(data_R16100[,1:2], data_R13023[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R13023', nrow(data_R13023))))
data_r15=cbind(rbind(data_R13074[,1:2], data_R13023[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R13023', nrow(data_R13023))))
data_r16=cbind(rbind(data_R13074[,1:2], data_R17032[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R17032', nrow(data_R17032))))
data_r17=cbind(rbind(data_R13023[,1:2], data_R16104[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R16104', nrow(data_R16104))))
data_r18=cbind(rbind(data_R16100[,1:2], data_R17015[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R17015', nrow(data_R17015))))
data_r19=cbind(rbind(data_R16100[,1:2], data_R17061[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R17061', nrow(data_R17061))))
data_r20=cbind(rbind(data_R16100[,1:2], data_R17037[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R17037', nrow(data_R17037))))
data_r21=cbind(rbind(data_R16100[,1:2], data_R17032[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R17032', nrow(data_R17032))))
data_r22=cbind(rbind(data_R16100[,1:2], data_R16104[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R16104', nrow(data_R16104))))
data_r23=cbind(rbind(data_R17015[,1:2], data_R17061[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R17061', nrow(data_R17061))))
data_r24=cbind(rbind(data_R17015[,1:2], data_R17037[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R17037', nrow(data_R17037))))
data_r25=cbind(rbind(data_R17015[,1:2], data_R17032[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R17032', nrow(data_R17032))))
data_r26=cbind(rbind(data_R17015[,1:2], data_R16104[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R16104', nrow(data_R16104))))
data_r27=cbind(rbind(data_R17061[,1:2], data_R17037[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R17037', nrow(data_R17037))))
data_r28=cbind(rbind(data_R17061[,1:2], data_R17032[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R17032', nrow(data_R17032))))
data_r29=cbind(rbind(data_R17061[,1:2], data_R16104[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R16104', nrow(data_R16104))))
data_r30=cbind(rbind(data_R17037[,1:2], data_R17032[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R17032', nrow(data_R17032))))
data_r31=cbind(rbind(data_R17037[,1:2], data_R16104[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R16104', nrow(data_R16104))))
data_r32=cbind(rbind(data_R17032[,1:2], data_R16104[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R16104', nrow(data_R16104))))
data_r33=cbind(rbind(data_R15107[,1:2], data_R16089[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R16089', nrow(data_R16089))))
data_r34=cbind(rbind(data_R15107[,1:2], data_R16062[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R16062', nrow(data_R16062))))
data_r35=cbind(rbind(data_R15107[,1:2], data_R15123[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R15123', nrow(data_R15123))))
data_r36=cbind(rbind(data_R15107[,1:2], data_R16034[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R16034', nrow(data_R16034))))
data_r37=cbind(rbind(data_R15107[,1:2], data_R15140[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R15140', nrow(data_R15140))))
data_r38=cbind(rbind(data_R15107[,1:2], data_R16053[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R16053', nrow(data_R16053))))
data_r39=cbind(rbind(data_R15107[,1:2], data_R15089[,1:2]), c(rep('R15107', nrow(data_R15107)), rep('R15089', nrow(data_R15089))))
data_r40=cbind(rbind(data_R16089[,1:2], data_R16062[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R16062', nrow(data_R16062))))
data_r41=cbind(rbind(data_R16089[,1:2], data_R15123[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R15123', nrow(data_R15123))))
data_r42=cbind(rbind(data_R16089[,1:2], data_R16034[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R16034', nrow(data_R16034))))
data_r43=cbind(rbind(data_R16089[,1:2], data_R15140[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R15140', nrow(data_R15140))))
data_r44=cbind(rbind(data_R16089[,1:2], data_R16053[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R16053', nrow(data_R16053))))
data_r45=cbind(rbind(data_R16089[,1:2], data_R15089[,1:2]), c(rep('R16089', nrow(data_R16089)), rep('R15089', nrow(data_R15089))))
data_r46=cbind(rbind(data_R16062[,1:2], data_R15123[,1:2]), c(rep('R16062', nrow(data_R16062)), rep('R15123', nrow(data_R15123))))
data_r47=cbind(rbind(data_R16062[,1:2], data_R16034[,1:2]), c(rep('R16062', nrow(data_R16062)), rep('R16034', nrow(data_R16034))))
data_r48=cbind(rbind(data_R16062[,1:2], data_R15140[,1:2]), c(rep('R16062', nrow(data_R16062)), rep('R15140', nrow(data_R15140))))
data_r49=cbind(rbind(data_R16062[,1:2], data_R16053[,1:2]), c(rep('R16062', nrow(data_R16062)), rep('R16053', nrow(data_R16053))))
data_r50=cbind(rbind(data_R16062[,1:2], data_R15089[,1:2]), c(rep('R16062', nrow(data_R16062)), rep('R15089', nrow(data_R15089))))
data_r51=cbind(rbind(data_R15123[,1:2], data_R16034[,1:2]), c(rep('R15123', nrow(data_R15123)), rep('R16034', nrow(data_R16034))))
data_r52=cbind(rbind(data_R15123[,1:2], data_R15140[,1:2]), c(rep('R15123', nrow(data_R15123)), rep('R15140', nrow(data_R15140))))
data_r53=cbind(rbind(data_R15123[,1:2], data_R16053[,1:2]), c(rep('R15123', nrow(data_R15123)), rep('R16053', nrow(data_R16053))))
data_r54=cbind(rbind(data_R15123[,1:2], data_R15089[,1:2]), c(rep('R15123', nrow(data_R15123)), rep('R15089', nrow(data_R15089))))
data_r55=cbind(rbind(data_R16034[,1:2], data_R15140[,1:2]), c(rep('R16034', nrow(data_R16034)), rep('R15140', nrow(data_R15140))))
data_r56=cbind(rbind(data_R16034[,1:2], data_R16053[,1:2]), c(rep('R16034', nrow(data_R16034)), rep('R16053', nrow(data_R16053))))
data_r57=cbind(rbind(data_R16034[,1:2], data_R15089[,1:2]), c(rep('R16034', nrow(data_R16034)), rep('R15089', nrow(data_R15089))))
data_r58=cbind(rbind(data_R15140[,1:2], data_R16053[,1:2]), c(rep('R15140', nrow(data_R15140)), rep('R16053', nrow(data_R16053))))
data_r59=cbind(rbind(data_R15140[,1:2], data_R15089[,1:2]), c(rep('R15140', nrow(data_R15140)), rep('R15089', nrow(data_R15089))))
data_r60=cbind(rbind(data_R16053[,1:2], data_R15089[,1:2]), c(rep('R16053', nrow(data_R16053)), rep('R15089', nrow(data_R15089))))
#
distance_related_indiv = vector()
distance_all=list()
list_data = c("data_r1", "data_r2", "data_r3", "data_r4", "data_r5", "data_r6", "data_r7", "data_r8", "data_r9", "data_r10", "data_r11", "data_r12", "data_r13", "data_r14", "data_r15", "data_r16", "data_r17", "data_r18", "data_r19", "data_r20", "data_r21", "data_r22", "data_r23", "data_r24", "data_r25", "data_r26", "data_r27", "data_r28", "data_r29", "data_r30", "data_r31", "data_r32", "data_r33", "data_r34", "data_r35", "data_r36", "data_r37", "data_r38", "data_r39", "data_r40","data_r41","data_r42", "data_r43", "data_r44", "data_r45", "data_r46", "data_r47", "data_r48", "data_r49", "data_r50", "data_r51", "data_r52", "data_r53", "data_r54", "data_r55", "data_r56", "data_r57", "data_r58", "data_r59", "data_r60")

for(indiv in list_data){
    data_denovo=get(indiv)[,1:3]
# Distance
pos_chr1=vector()
pos_chr2=vector()
pos_chr3=vector()
pos_chr4=vector()
pos_chr5=vector()
pos_chr6=vector()
pos_chr7=vector()
pos_chr8=vector()
pos_chr9=vector()
pos_chr10=vector()
pos_chr11=vector()
pos_chr12=vector()
pos_chr13=vector()
pos_chr14=vector()
pos_chr15=vector()
pos_chr16=vector()
pos_chr17=vector()
pos_chr18=vector()
pos_chr19=vector()
pos_chr20=vector()
for(i in seq(1,nrow(data_denovo))){
  if(data_denovo[i,1]=='chr1'){pos_chr1=rbind(pos_chr1,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr2'){pos_chr2=rbind(pos_chr2,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr3'){pos_chr3=rbind(pos_chr3,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr4'){pos_chr4=rbind(pos_chr4,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr5'){pos_chr5=rbind(pos_chr5,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr6'){pos_chr6=rbind(pos_chr6,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr7'){pos_chr7=rbind(pos_chr7,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr8'){pos_chr8=rbind(pos_chr8,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr9'){pos_chr9=rbind(pos_chr9,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr10'){pos_chr10=rbind(pos_chr10,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr11'){pos_chr11=rbind(pos_chr11,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr12'){pos_chr12=rbind(pos_chr12,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr13'){pos_chr13=rbind(pos_chr13,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr14'){pos_chr14=rbind(pos_chr14,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr15'){pos_chr15=rbind(pos_chr15,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr16'){pos_chr16=rbind(pos_chr16,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr17'){pos_chr17=rbind(pos_chr17,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr18'){pos_chr18=rbind(pos_chr18,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr19'){pos_chr19=rbind(pos_chr19,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr20'){pos_chr20=rbind(pos_chr20,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }
}

pos_list <- c("pos_chr1","pos_chr2","pos_chr3","pos_chr4","pos_chr5","pos_chr6","pos_chr7","pos_chr8","pos_chr9","pos_chr10","pos_chr11","pos_chr12","pos_chr13","pos_chr14","pos_chr15","pos_chr16","pos_chr17","pos_chr18","pos_chr19","pos_chr20")
rm(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr12,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)
for(j in pos_list){
    distance=vector()
    set=get(j)
    if(is.null(dim(set))){
        assign(paste0("distance_",j), NA)
    }else{
        sorted=set[order(set[,1]),]
        for(i in 2:nrow(set)){
            if(nrow(sorted)<2){
              distance=c(distance,NA)
            }else{
                if(sorted[i,2]==sorted[i-1,2]){
                    distance=c(distance,NA)
                }else{
                 dist=sorted[i,1]-sorted[i-1,1]
                distance=c(distance,dist)
               }
            }
        }
    assign(paste0("distance_",j),distance)
    }
}

distance_all=c(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr13,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)
distance_all <- distance_all[!is.na(distance_all)]
distance_related_indiv=c(distance_related_indiv,distance_all)

}

# 2 by 2 between non related ##
data_nr1=cbind(rbind(data_R14024[,1:2], data_R17015[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R17015', nrow(data_R17015))))
data_nr2=cbind(rbind(data_R14024[,1:2], data_R17061[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R17061', nrow(data_R17061))))
data_nr3=cbind(rbind(data_R14024[,1:2], data_R17037[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R17037', nrow(data_R17037))))
data_nr4=cbind(rbind(data_R14024[,1:2], data_R17032[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R17032', nrow(data_R17032))))
data_nr5=cbind(rbind(data_R14024[,1:2], data_R16104[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16104', nrow(data_R16104))))
data_nr6=cbind(rbind(data_R14024[,1:2], data_R15107[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R15107', nrow(data_R15107))))
data_nr7=cbind(rbind(data_R14024[,1:2], data_R16089[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16089', nrow(data_R16089))))
data_nr8=cbind(rbind(data_R14024[,1:2], data_R16062[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16062', nrow(data_R16062))))
data_nr9=cbind(rbind(data_R14024[,1:2], data_R15123[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R15123', nrow(data_R15123))))
data_nr10=cbind(rbind(data_R14024[,1:2], data_R16034[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16034', nrow(data_R16034))))
data_nr11=cbind(rbind(data_R14024[,1:2], data_R15140[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R15140', nrow(data_R15140))))
data_nr12=cbind(rbind(data_R14024[,1:2], data_R16053[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R16053', nrow(data_R16053))))
data_nr13=cbind(rbind(data_R14024[,1:2], data_R15089[,1:2]), c(rep('R14024', nrow(data_R14024)), rep('R15089', nrow(data_R15089))))
data_nr14=cbind(rbind(data_R12006[,1:2], data_R17061[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R17061', nrow(data_R17061))))
data_nr15=cbind(rbind(data_R12006[,1:2], data_R17037[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R17037', nrow(data_R17037))))
data_nr16=cbind(rbind(data_R12006[,1:2], data_R17032[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R17032', nrow(data_R17032))))
data_nr17=cbind(rbind(data_R12006[,1:2], data_R16104[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16104', nrow(data_R16104))))
data_nr18=cbind(rbind(data_R12006[,1:2], data_R15107[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R15107', nrow(data_R15107))))
data_nr19=cbind(rbind(data_R12006[,1:2], data_R16089[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16089', nrow(data_R16089))))
data_nr20=cbind(rbind(data_R12006[,1:2], data_R16062[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16062', nrow(data_R16062))))
data_nr21=cbind(rbind(data_R12006[,1:2], data_R15123[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R15123', nrow(data_R15123))))
data_nr22=cbind(rbind(data_R12006[,1:2], data_R16034[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16034', nrow(data_R16034))))
data_nr23=cbind(rbind(data_R12006[,1:2], data_R15140[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R15140', nrow(data_R15140))))
data_nr24=cbind(rbind(data_R12006[,1:2], data_R16053[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R16053', nrow(data_R16053))))
data_nr25=cbind(rbind(data_R12006[,1:2], data_R15089[,1:2]), c(rep('R12006', nrow(data_R12006)), rep('R15089', nrow(data_R15089))))
data_nr26=cbind(rbind(data_R13024[,1:2], data_R17015[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R17015', nrow(data_R17015))))
data_nr27=cbind(rbind(data_R13024[,1:2], data_R17061[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R17061', nrow(data_R17061))))
data_nr28=cbind(rbind(data_R13024[,1:2], data_R17037[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R17037', nrow(data_R17037))))
data_nr29=cbind(rbind(data_R13024[,1:2], data_R17032[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R17032', nrow(data_R17032))))
data_nr30=cbind(rbind(data_R13024[,1:2], data_R16104[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16104', nrow(data_R16104))))
data_nr31=cbind(rbind(data_R13024[,1:2], data_R15107[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R15107', nrow(data_R15107))))
data_nr32=cbind(rbind(data_R13024[,1:2], data_R16089[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16089', nrow(data_R16089))))
data_nr33=cbind(rbind(data_R13024[,1:2], data_R16062[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16062', nrow(data_R16062))))
data_nr34=cbind(rbind(data_R13024[,1:2], data_R15123[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R15123', nrow(data_R15123))))
data_nr35=cbind(rbind(data_R13024[,1:2], data_R16034[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16034', nrow(data_R16034))))
data_nr36=cbind(rbind(data_R13024[,1:2], data_R15140[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R15140', nrow(data_R15140))))
data_nr37=cbind(rbind(data_R13024[,1:2], data_R16053[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R16053', nrow(data_R16053))))
data_nr38=cbind(rbind(data_R13024[,1:2], data_R15089[,1:2]), c(rep('R13024', nrow(data_R13024)), rep('R15089', nrow(data_R15089))))
data_nr39=cbind(rbind(data_R16100[,1:2], data_R13074[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R13074', nrow(data_R13074))))
data_nr40=cbind(rbind(data_R16100[,1:2], data_R13023[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R13023', nrow(data_R13023))))
data_nr41=cbind(rbind(data_R16100[,1:2], data_R15107[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R15107', nrow(data_R15107))))
data_nr42=cbind(rbind(data_R16100[,1:2], data_R16089[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R16089', nrow(data_R16089))))
data_nr43=cbind(rbind(data_R16100[,1:2], data_R16062[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R16062', nrow(data_R16062))))
data_nr44=cbind(rbind(data_R16100[,1:2], data_R15123[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R15123', nrow(data_R15123))))
data_nr45=cbind(rbind(data_R16100[,1:2], data_R16034[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R16034', nrow(data_R16034))))
data_nr46=cbind(rbind(data_R16100[,1:2], data_R15140[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R15140', nrow(data_R15140))))
data_nr47=cbind(rbind(data_R16100[,1:2], data_R16053[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R16053', nrow(data_R16053))))
data_nr48=cbind(rbind(data_R16100[,1:2], data_R15089[,1:2]), c(rep('R16100', nrow(data_R16100)), rep('R15089', nrow(data_R15089))))
data_nr49=cbind(rbind(data_R17061[,1:2], data_R13023[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R13023', nrow(data_R13023))))
data_nr50=cbind(rbind(data_R17061[,1:2], data_R15107[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R15107', nrow(data_R15107))))
data_nr51=cbind(rbind(data_R17061[,1:2], data_R16089[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R16089', nrow(data_R16089))))
data_nr52=cbind(rbind(data_R17061[,1:2], data_R16062[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R16062', nrow(data_R16062))))
data_nr53=cbind(rbind(data_R17061[,1:2], data_R15123[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R15123', nrow(data_R15123))))
data_nr54=cbind(rbind(data_R17061[,1:2], data_R16034[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R16034', nrow(data_R16034))))
data_nr55=cbind(rbind(data_R17061[,1:2], data_R15140[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R15140', nrow(data_R15140))))
data_nr56=cbind(rbind(data_R17061[,1:2], data_R16053[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R16053', nrow(data_R16053))))
data_nr57=cbind(rbind(data_R17061[,1:2], data_R15089[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R15089', nrow(data_R15089))))
data_nr58=cbind(rbind(data_R13074[,1:2], data_R17015[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R17015', nrow(data_R17015))))
data_nr59=cbind(rbind(data_R13074[,1:2], data_R16104[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R16104', nrow(data_R16104))))
data_nr60=cbind(rbind(data_R13074[,1:2], data_R15107[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R15107', nrow(data_R15107))))
data_nr61=cbind(rbind(data_R13074[,1:2], data_R16089[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R16089', nrow(data_R16089))))
data_nr62=cbind(rbind(data_R13074[,1:2], data_R16062[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R16062', nrow(data_R16062))))
data_nr63=cbind(rbind(data_R13074[,1:2], data_R15123[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R15123', nrow(data_R15123))))
data_nr64=cbind(rbind(data_R13074[,1:2], data_R16034[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R16034', nrow(data_R16034))))
data_nr65=cbind(rbind(data_R13074[,1:2], data_R15140[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R15140', nrow(data_R15140))))
data_nr66=cbind(rbind(data_R13074[,1:2], data_R16053[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R16053', nrow(data_R16053))))
data_nr67=cbind(rbind(data_R13074[,1:2], data_R15089[,1:2]), c(rep('R13074', nrow(data_R13074)), rep('R15089', nrow(data_R15089))))
data_nr68=cbind(rbind(data_R17032[,1:2], data_R13023[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R13023', nrow(data_R13023))))
data_nr69=cbind(rbind(data_R17032[,1:2], data_R15107[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R15107', nrow(data_R15107))))
data_nr70=cbind(rbind(data_R17032[,1:2], data_R16089[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R16089', nrow(data_R16089))))
data_nr71=cbind(rbind(data_R17032[,1:2], data_R16062[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R16062', nrow(data_R16062))))
data_nr72=cbind(rbind(data_R17032[,1:2], data_R15123[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R15123', nrow(data_R15123))))
data_nr73=cbind(rbind(data_R17032[,1:2], data_R16034[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R16034', nrow(data_R16034))))
data_nr74=cbind(rbind(data_R17032[,1:2], data_R15140[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R15140', nrow(data_R15140))))
data_nr75=cbind(rbind(data_R17032[,1:2], data_R16053[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R16053', nrow(data_R16053))))
data_nr76=cbind(rbind(data_R17032[,1:2], data_R15089[,1:2]), c(rep('R17032', nrow(data_R17032)), rep('R15089', nrow(data_R15089))))
data_nr77=cbind(rbind(data_R13023[,1:2], data_R17015[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R17015', nrow(data_R17015))))
data_nr78=cbind(rbind(data_R13023[,1:2], data_R17037[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R17037', nrow(data_R17037))))
data_nr79=cbind(rbind(data_R13023[,1:2], data_R15107[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R15107', nrow(data_R15107))))
data_nr80=cbind(rbind(data_R13023[,1:2], data_R16089[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R16089', nrow(data_R16089))))
data_nr81=cbind(rbind(data_R13023[,1:2], data_R16062[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R16062', nrow(data_R16062))))
data_nr82=cbind(rbind(data_R13023[,1:2], data_R15123[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R15123', nrow(data_R15123))))
data_nr83=cbind(rbind(data_R13023[,1:2], data_R16034[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R16034', nrow(data_R16034))))
data_nr84=cbind(rbind(data_R13023[,1:2], data_R15140[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R15140', nrow(data_R15140))))
data_nr85=cbind(rbind(data_R13023[,1:2], data_R16053[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R16053', nrow(data_R16053))))
data_nr86=cbind(rbind(data_R13023[,1:2], data_R15089[,1:2]), c(rep('R13023', nrow(data_R13023)), rep('R15089', nrow(data_R15089))))
data_nr87=cbind(rbind(data_R16104[,1:2], data_R15107[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R15107', nrow(data_R15107))))
data_nr88=cbind(rbind(data_R16104[,1:2], data_R16089[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R16089', nrow(data_R16089))))
data_nr89=cbind(rbind(data_R16104[,1:2], data_R16062[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R16062', nrow(data_R16062))))
data_nr90=cbind(rbind(data_R16104[,1:2], data_R15123[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R15123', nrow(data_R15123))))
data_nr91=cbind(rbind(data_R16104[,1:2], data_R16034[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R16034', nrow(data_R16034))))
data_nr92=cbind(rbind(data_R16104[,1:2], data_R15140[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R15140', nrow(data_R15140))))
data_nr93=cbind(rbind(data_R16104[,1:2], data_R16053[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R16053', nrow(data_R16053))))
data_nr94=cbind(rbind(data_R16104[,1:2], data_R15089[,1:2]), c(rep('R16104', nrow(data_R16104)), rep('R15089', nrow(data_R15089))))
data_nr95=cbind(rbind(data_R17015[,1:2], data_R15107[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R15107', nrow(data_R15107))))
data_nr96=cbind(rbind(data_R17015[,1:2], data_R16089[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R16089', nrow(data_R16089))))
data_nr97=cbind(rbind(data_R17015[,1:2], data_R16062[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R16062', nrow(data_R16062))))
data_nr98=cbind(rbind(data_R17015[,1:2], data_R15123[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R15123', nrow(data_R15123))))
data_nr99=cbind(rbind(data_R17015[,1:2], data_R16034[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R16034', nrow(data_R16034))))
data_nr100=cbind(rbind(data_R17015[,1:2], data_R15140[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R15140', nrow(data_R15140))))
data_nr101=cbind(rbind(data_R17015[,1:2], data_R16053[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R16053', nrow(data_R16053))))
data_nr102=cbind(rbind(data_R17015[,1:2], data_R15089[,1:2]), c(rep('R17015', nrow(data_R17015)), rep('R15089', nrow(data_R15089))))
data_nr103=cbind(rbind(data_R17037[,1:2], data_R15107[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R15107', nrow(data_R15107))))
data_nr104=cbind(rbind(data_R17037[,1:2], data_R16089[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R16089', nrow(data_R16089))))
data_nr105=cbind(rbind(data_R17037[,1:2], data_R16062[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R16062', nrow(data_R16062))))
data_nr106=cbind(rbind(data_R17037[,1:2], data_R15123[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R15123', nrow(data_R15123))))
data_nr107=cbind(rbind(data_R17037[,1:2], data_R16034[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R16034', nrow(data_R16034))))
data_nr108=cbind(rbind(data_R17037[,1:2], data_R15140[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R15140', nrow(data_R15140))))
data_nr109=cbind(rbind(data_R17037[,1:2], data_R16053[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R16053', nrow(data_R16053))))
data_nr110=cbind(rbind(data_R17037[,1:2], data_R15089[,1:2]), c(rep('R17037', nrow(data_R17037)), rep('R15089', nrow(data_R15089))))
data_nr111=cbind(rbind(data_R17061[,1:2], data_R13074[,1:2]), c(rep('R17061', nrow(data_R17061)), rep('R13074', nrow(data_R13074))))
#
distance_non_related_indiv = vector()
distance_all=list()
list_data = c("data_nr1", "data_nr2", "data_nr3", "data_nr4", "data_nr5", "data_nr6", "data_nr7", "data_nr8", "data_nr9", "data_nr10", "data_nr11", "data_nr12", "data_nr13", "data_nr14", "data_nr15", "data_nr16", "data_nr17", "data_nr18", "data_nr19", "data_nr20", "data_nr21", "data_nr22", "data_nr23", "data_nr24", "data_nr25", "data_nr26", "data_nr27", "data_nr28", "data_nr29", "data_nr30", "data_nr31", "data_nr32", "data_nr33", "data_nr34", "data_nr35", "data_nr36", "data_nr37", "data_nr38", "data_nr39", "data_nr40","data_nr41","data_nr42", "data_nr43", "data_nr44", "data_nr45", "data_nr46", "data_nr47", "data_nr48", "data_nr49", "data_nr50", "data_nr51", "data_nr52", "data_nr53", "data_nr54", "data_nr55", "data_nr56", "data_nr57", "data_nr58", "data_nr59", "data_nr60", "data_nr70", "data_nr71", "data_nr72", "data_nr73", "data_nr74", "data_nr75", "data_nr76", "data_nr77", "data_nr78", "data_nr79", "data_nr80", "data_nr81", "data_nr82", "data_nr83", "data_nr84", "data_nr85", "data_nr86", "data_nr87", "data_nr88", "data_nr89", "data_nr90", "data_nr91", "data_nr92", "data_nr93", "data_nr94", "data_nr95", "data_nr96", "data_nr97", "data_nr98", "data_nr99", "data_nr100", "data_nr101", "data_nr102", "data_nr103", "data_nr104", "data_nr105", "data_nr106", "data_nr107", "data_nr108", "data_nr109", "data_nr110",  "data_nr111")

for(indiv in list_data){
    data_denovo=get(indiv)[,1:3]

## Distance
pos_chr1=vector()
pos_chr2=vector()
pos_chr3=vector()
pos_chr4=vector()
pos_chr5=vector()
pos_chr6=vector()
pos_chr7=vector()
pos_chr8=vector()
pos_chr9=vector()
pos_chr10=vector()
pos_chr11=vector()
pos_chr12=vector()
pos_chr13=vector()
pos_chr14=vector()
pos_chr15=vector()
pos_chr16=vector()
pos_chr17=vector()
pos_chr18=vector()
pos_chr19=vector()
pos_chr20=vector()
for(i in seq(1,nrow(data_denovo))){
  if(data_denovo[i,1]=='chr1'){pos_chr1=rbind(pos_chr1,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr2'){pos_chr2=rbind(pos_chr2,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr3'){pos_chr3=rbind(pos_chr3,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr4'){pos_chr4=rbind(pos_chr4,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr5'){pos_chr5=rbind(pos_chr5,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr6'){pos_chr6=rbind(pos_chr6,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr7'){pos_chr7=rbind(pos_chr7,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr8'){pos_chr8=rbind(pos_chr8,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr9'){pos_chr9=rbind(pos_chr9,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr10'){pos_chr10=rbind(pos_chr10,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr11'){pos_chr11=rbind(pos_chr11,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr12'){pos_chr12=rbind(pos_chr12,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr13'){pos_chr13=rbind(pos_chr13,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr14'){pos_chr14=rbind(pos_chr14,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr15'){pos_chr15=rbind(pos_chr15,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr16'){pos_chr16=rbind(pos_chr16,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr17'){pos_chr17=rbind(pos_chr17,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr18'){pos_chr18=rbind(pos_chr18,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr19'){pos_chr19=rbind(pos_chr19,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }else if(data_denovo[i,1]=='chr20'){pos_chr20=rbind(pos_chr20,data.frame(data_denovo[i,2], data_denovo[i,3]))
  }
}

pos_list <- c("pos_chr1","pos_chr2","pos_chr3","pos_chr4","pos_chr5","pos_chr6","pos_chr7","pos_chr8","pos_chr9","pos_chr10","pos_chr11","pos_chr12","pos_chr13","pos_chr14","pos_chr15","pos_chr16","pos_chr17","pos_chr18","pos_chr19","pos_chr20")
rm(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr12,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)

for(j in pos_list){
    distance=vector()
    set=get(j)
    if(is.null(dim(set))){
        assign(paste0("distance_",j), NA)
    }else{
        sorted=set[order(set[,1]),]
        for(i in 2:nrow(set)){
            if(nrow(sorted)<2){
              distance=c(distance,NA)
            }else{
                if(sorted[i,2]==sorted[i-1,2]){
                    distance=c(distance,NA)
                }else{
                 dist=sorted[i,1]-sorted[i-1,1]
                distance=c(distance,dist)
               }
            }
        }
    assign(paste0("distance_",j),distance)
    }
}

distance_all=c(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr13,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)
distance_all <- distance_all[!is.na(distance_all)]
distance_non_related_indiv=c(distance_non_related_indiv,distance_all)

}

###
## Now import the simulated for QQ_plot:
fake_mut_1 = read.csv("fake_mut_1.txt", sep=' ')

# Distance for fake_mut_1
data_denovo = fake_mut_1
pos_chr1=vector()
pos_chr2=vector()
pos_chr3=vector()
pos_chr4=vector()
pos_chr5=vector()
pos_chr6=vector()
pos_chr7=vector()
pos_chr8=vector()
pos_chr9=vector()
pos_chr10=vector()
pos_chr11=vector()
pos_chr12=vector()
pos_chr13=vector()
pos_chr14=vector()
pos_chr15=vector()
pos_chr16=vector()
pos_chr17=vector()
pos_chr18=vector()
pos_chr19=vector()
pos_chr20=vector()
for(i in seq(1,nrow(data_denovo))){
  if(data_denovo[i,1]=='chr1'){pos_chr1=c(pos_chr1,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr2'){pos_chr2=c(pos_chr2,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr3'){pos_chr3=c(pos_chr3,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr4'){pos_chr4=c(pos_chr4,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr5'){pos_chr5=c(pos_chr5,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr6'){pos_chr6=c(pos_chr6,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr7'){pos_chr7=c(pos_chr7,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr8'){pos_chr8=c(pos_chr8,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr9'){pos_chr9=c(pos_chr9,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr10'){pos_chr10=c(pos_chr10,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr11'){pos_chr11=c(pos_chr11,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr12'){pos_chr12=c(pos_chr12,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr13'){pos_chr13=c(pos_chr13,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr14'){pos_chr14=c(pos_chr14,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr15'){pos_chr15=c(pos_chr15,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr16'){pos_chr16=c(pos_chr16,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr17'){pos_chr17=c(pos_chr17,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr18'){pos_chr18=c(pos_chr18,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr19'){pos_chr19=c(pos_chr19,data_denovo[i,2])
  }else if(data_denovo[i,1]=='chr20'){pos_chr20=c(pos_chr20,data_denovo[i,2])
  }
}

## Distances:
pos_list <- c("pos_chr1","pos_chr2","pos_chr3","pos_chr4","pos_chr5","pos_chr6","pos_chr7","pos_chr8","pos_chr9","pos_chr10","pos_chr11","pos_chr12","pos_chr13","pos_chr14","pos_chr15","pos_chr16","pos_chr17","pos_chr18","pos_chr19","pos_chr20")
rm(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr13,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)

for(j in pos_list){
    distance=vector()
    sorted=sort(get(j))
    for(i in 2:length(get(j))){
        dist=sorted[i]-sorted[i-1]
        distance=c(distance,dist)
    }
    assign(paste0("distance_",j),distance)
}
distance_all=c(distance_pos_chr1,distance_pos_chr2,distance_pos_chr3,distance_pos_chr4,distance_pos_chr5,distance_pos_chr6,distance_pos_chr7,distance_pos_chr8,distance_pos_chr9,distance_pos_chr10,distance_pos_chr11,distance_pos_chr12,distance_pos_chr13,distance_pos_chr14,distance_pos_chr15,distance_pos_chr16,distance_pos_chr17,distance_pos_chr18,distance_pos_chr19,distance_pos_chr20)

distance_fake_1 = distance_all

##############################################################
# QQ plot with all:
# distance_within_same_ind, distance_related_indiv, distance_non_related_indiv
# "#9553CB", "#49C9A5", "#FF7B29"

png("QQ_plot_diff_better.png", width = 550, height = 850)
par(mar=c(10,11,4,2), mgp=c(3, 2, 0))
qqplot(log10(distance_fake_1),log10(distance_within_same_ind), xlim=c(0,9), ylim=c(0,9), pch='-', lwd=10, xlab="",ylab="",cex=3,cex.lab=2.5, cex.axis=2.5, xaxt="n", yaxt="n", col="#9553CB")
par(new=TRUE)
qqplot(log10(distance_fake_1),log10(distance_related_indiv), xlim=c(0,9), ylim=c(0,9), pch='-', lwd=10, xlab="",ylab="",cex=3,cex.lab=2.5, cex.axis=2.5, xaxt="n", yaxt="n", col="#49C9A5")
par(new=TRUE)
qqplot(log10(distance_fake_1),log10(distance_non_related_indiv), xlim=c(0,9), ylim=c(0,9), pch='-', lwd=10, xlab="",ylab="",cex=3,cex.lab=2.5, cex.axis=2.5, xaxt="n", yaxt="n", col="#FF7B29")
axis(1, cex.axis=3)
axis(2, cex.axis=3, las=2)
mtext(expression(paste("Expected distance (log"[10],")")), 1, line=6, cex=3)
mtext(expression(paste("Observed distance (log"[10],")")), 2, line=5, cex=3)
legend("topleft", bty = "n", fill=c("#9553CB", "#49C9A5", "#FF7B29"),c("same", "related", "non-related"), x.intersp =0.5, cex=2.8)
dev.off()

# 3. Shared between siblings:
## data table from de novo detection
data_denovo_last <- c(rep("R16089", nrow(data_R16089)), rep("R16053", nrow(data_R16053)),
                      rep("R16062", nrow(data_R16062)), rep("R15089", nrow(data_R15089)),
                      rep("R15107", nrow(data_R15107)), rep("R15123", nrow(data_R15123)),
                      rep("R16034", nrow(data_R16034)), rep("R15140", nrow(data_R15140)),
                      rep("R14024", nrow(data_R14024)), rep("R12006", nrow(data_R12006)),
                      rep("R13024", nrow(data_R13024)), rep("R13074", nrow(data_R13074)),
                      rep("R13023", nrow(data_R13023)), rep("R16100", nrow(data_R16100)),
                      rep("R17015", nrow(data_R17015)), rep("R17061", nrow(data_R17061)),
                      rep("R17032", nrow(data_R17032)), rep("R17037", nrow(data_R17037)),
                      rep("R16104", nrow(data_R16104)))

data_denovo_full <- cbind(data_denovo, data_denovo_last)
shared <- data_denovo_full[which(duplicated(data_denovo) | duplicated(data_denovo[nrow(data_denovo):1, ])[nrow(data_denovo):1]=="TRUE"),]
shared[order(shared$CHROM,shared$POS),]
write.table(shared[order(shared$CHROM,shared$POS),], "shared_denovo.tab", sep="\t")

# Chance to shared on the random data in distance
for(i in seq(1,10)){
    fake <-read.csv(paste0("../distance/fake_mut_",i,".txt"), sep =" ", header = TRUE)
    shared_fake <- fake[which(duplicated(fake) | duplicated(fake[nrow(fake):1, ])[nrow(fake):1]=="TRUE"),]
    print(shared_fake)
}


# Similar to the column binding

# star_wars_matrix and star_wars_matrix2 are available in your workspace
star_wars_matrix
star_wars_matrix2

# Combine both Star Wars trilogies in one matrix
all_wars_matrix <- rbind(star_wars_matrix, star_wars_matrix2)

# rbind(): can add matrices and vectors, similar to cbind()


#!/usr/bin/env Rscript

data = read.csv("hw1_data.csv")
isMay = data[['Month']] == 5
max(data[isMay,][['Ozone']], na.rm=TRUE)


######################################################################################
#                   Libraries
######################################################################################

# function to install missing packages
usePackage <- function(p) {
  if (!is.element(p, installed.packages()[,1]))
    install.packages(p, dep = TRUE)
  require(p, character.only = TRUE)
}

# load packages
set.seed(1)
usePackage("e1071")
usePackage("randomForest")
usePackage("ROCR")
usePackage("party")
usePackage("mlbench")
usePackage("caret")
usePackage("class")
usePackage("pROC")
usePackage("ROCR")

######################################################################################
#                   Features and load
######################################################################################
# seleciting attributes
features <- c("Class", "ORF_Length", "ORF_proportion", 'GCGG', 'TTTT', 'TCG', 'AAAA', 'ACG', 'TTGT', 'TAT', 'TAC',
              'GTT', 'GTG', 'AGT', 'CCGA', 'TACC', 'CGTG', 'CGCT', 'TACG',
              'TTAG', 'CGTA', 'ACCG', 'CCGT', 'CGGT', 'CGAC', 'CGCA', 'GCGT',
              'GTAG', 'CGTT', 'CGAA', 'GCGA', 'CGAT', 'TAGT')

# Data loading and pre-processing
train <- read.csv('temp/train.csv', sep=";", header = TRUE)
test  <- read.csv('temp/test.csv',  sep=";", header = TRUE)

# seleciting attributes
train_filtered <- train[features]
test_filtered  <- test[features]

row.has.na <- apply(train_filtered, 1, function(x){
  any(is.na(x))
  })
train_filtered <- train_filtered[!row.has.na,]

row.has.na <- apply(test_filtered, 1, function(x){
  any(is.na(x))
})
test_filtered <- test_filtered[!row.has.na,]

accuracy <- 0

print("Training and Test set loaded with success")

######################################################################################
#                   SVM
######################################################################################

print("Building svm model")

# tune svm parameters
svm_tune <- tune(svm, Class ~ ., data = train_filtered, kernel = "radial", ranges = list(cost=10^(-1:2), gamma=c(.5,1,2)),
                 tunecontrol = tune.control(sampling = "cross"))

svm_cost <- svm_tune$best.parameters$cost
svm_gamma <- svm_tune$best.parameters$gamma

print("SVM tuned.")
print("Cost:")
print(svm_cost)
print("Gamma:")
print(svm_gamma)

# building SVM model for train.csv
svm_model <- svm(Class ~ ., data = train_filtered, type = "C-classification",
                 cost = svm_cost, gamma = svm_gamma)
# summary(svm_model)

print("SVM model built.")

# predicts test set
prediction_svm <- predict(svm_model, newdata = test_filtered)
confusion_svm  <- table(pred = prediction_svm, true = test_filtered$Class)
confusion_svm2  <- confusionMatrix(prediction_svm, test_filtered$Class )

# evaluation metrics
# confusion matrix
accuracy_svm = sum(prediction_svm == test_filtered$Class)/length(test_filtered$Class)
precision = confusion_svm[1,1]/sum(confusion_svm[,1])
recall = confusion_svm[1,1]/sum(confusion_svm[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy_svm, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

if(accuracy_svm > accuracy){
  accuracy <- accuracy_svm
}

######################################################################################
#                   RF
######################################################################################
print("Building RF model")

# tunning RF
# Create model with default paramters
seed <- 7
metric <- "Accuracy"
set.seed(seed)
mtry <- sqrt(ncol(train_filtered))
tunegrid <- expand.grid(.mtry=mtry)
control <- trainControl(method="repeatedcv", number=10, repeats=3, search="grid")
#control <- trainControl(method="repeatedcv", number=5, repeats=2, search="grid")

set.seed(seed)
#tunegrid <- expand.grid(.mtry=c(1:2))
tunegrid <- expand.grid(.mtry=c(1:15))

rf_gridsearch <- train(Class ~ ., data = train_filtered, method="rf",
                       metric = metric, tuneGrid=tunegrid, trControl=control)

mtry_tune <- rf_gridsearch$bestTune$mtry

print("RF model tuned.")
print("mtry:")
print(mtry_tune)
print("RF model built.")

png(filename="result/rf_grid.png")
plot(rf_gridsearch)
dev.off()

# rf model
rf_model <- randomForest(Class ~ ., data = train_filtered,
                         mtry = mtry_tune, importance=TRUE, ntree = 1500)
# print(rf_model)

print("Feature importance:\n")
# print(importance(rf_model, type = 2))
png(filename="result/rf_features.png")
varImpPlot(rf_model,type=2)
dev.off()

# predicts test set
prediction_rf <- predict(rf_gridsearch, newdata = test_filtered)
confusion_rf  <- table(pred = prediction_rf, true = test_filtered$Class)
confusion_rf2  <- confusionMatrix(prediction_rf, test_filtered$Class )

# evaluation metrics
# confusion matrix
print(confusion_rf2)
accuracy_rf = sum(prediction_rf == test_filtered$Class)/length(test_filtered$Class)
precision = confusion_rf[1,1]/sum(confusion_rf[,1])
recall = confusion_rf[1,1]/sum(confusion_rf[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy_rf, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

if(accuracy_rf > accuracy){
  accuracy <- accuracy_rf
}

######################################################################################
#                  C-tree
######################################################################################
print("Building Ctree model")

# tunning ctree
seed <- 7
metric <- "Accuracy"
set.seed(seed)
mtry <- sqrt(ncol(train_filtered))
tunegrid <- expand.grid(.mtry=mtry)
control <- trainControl(method="repeatedcv", number=10, repeats=3, search="grid")
set.seed(seed)
tunegrid <- expand.grid(.mincriterion = .95,
                        .maxdepth = as.integer(seq(5, 10, 2)))
ct_gridsearch <- train(Class ~ ., data = train_filtered, method="ctree2",
                       metric = metric, tuneGrid=tunegrid, trControl=control,
                       controls = ctree_control(minbucket = 10))
# print(ct_gridsearch)

png(filename="result/ctree_tuning.png")
plot(ct_gridsearch)
dev.off()

maxdepth <- ct_gridsearch$bestTune$maxdepth
mincriterion <- ct_gridsearch$bestTune$mincriterion

print("Ctree tuned")
print("Max Depth:")
print(maxdepth)
print("Min Criterion")
print(mincriterion)

print("Ctree built")

# ctree model
ct_model <- ctree(Class ~ ., data = train_filtered,
                  controls = ctree_control(maxdepth = maxdepth,
                                           mincriterion = mincriterion))
png(filename="result/ctree_features.png")
plot(ct_model, type="simple")
dev.off()

# predicts test set
prediction_ct <- predict(ct_gridsearch, newdata = test_filtered)
confusion_ct  <- table(pred = prediction_ct, true = test_filtered$Class)
confusion_ct2  <- confusionMatrix(prediction_ct, test_filtered$Class )

# evaluation metrics
# confusion matrix
print(confusion_ct2)
accuracy_ct = sum(prediction_ct == test_filtered$Class)/length(test_filtered$Class)
precision = confusion_ct[1,1]/sum(confusion_ct[,1])
recall = confusion_ct[1,1]/sum(confusion_ct[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy_ct, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

if(accuracy_ct > accuracy){
  accuracy <- accuracy_ct
}

######################################################################################
#                  KNN
######################################################################################
print("Building KNN model")

# tunning knn
seed <- 7
metric <- "Accuracy"
set.seed(seed)
mtry <- sqrt(ncol(train_filtered))
tunegrid <- expand.grid(.mtry=mtry)
control <- trainControl(method="repeatedcv", number=10, repeats=3, search="grid")
set.seed(seed)
knn_gridsearch <- train(Class ~ ., data = train_filtered, method="knn",
                       metric = metric, trControl=control,
                       preProcess = c("center","scale"), tuneLength = 20)
print(knn_gridsearch)
png(filename="result/knn_tuning.png")
plot(knn_gridsearch)
dev.off()

k <- knn_gridsearch$bestTune$k

print("Knn model tuned")
print("K:")
print(k)

# predicts test set
prediction_knn <- predict(knn_gridsearch, newdata = test_filtered)
confusion_knn  <- table(pred = prediction_knn, true = test_filtered$Class)
confusion_knn2  <- confusionMatrix(prediction_knn, test_filtered$Class )

# confusion matrix
print(confusion_knn2)

accuracy_knn = sum(prediction_knn == test_filtered$Class)/length(test_filtered$Class)
precision = confusion_knn[1,1]/sum(confusion_knn[,1])
recall = confusion_knn[1,1]/sum(confusion_knn[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy_knn, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

if(accuracy_knn > accuracy){
  accuracy <- accuracy_knn
}

######################################################################################
#                  INTERSECTION
######################################################################################
print("Building consensus ensemble model")

tam             <- length(prediction_svm)
total_counter   <- 0
prediction_all  <- 1:tam

# checks for prediction intersections
for (index in 1:tam){
  if(prediction_svm[index] == prediction_rf[index] &&
     prediction_svm[index] == prediction_ct[index] &&
     prediction_svm[index] == prediction_knn[index]){
    prediction_all[index] <- prediction_svm[index]
    total_counter <- total_counter + 1
  }else{
    prediction_all[index] <- NA
  }
}

# evaluation metrics
confusion_all  <- table(pred = prediction_all, true = test_filtered$Class)
print(confusion_all)
accuracy = (confusion_all[1,1] + confusion_all[2,2])/total_counter
precision = confusion_all[1,1]/sum(confusion_all[,1])
recall = confusion_all[1,1]/sum(confusion_all[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

######################################################################################
#                  MAJORITY
######################################################################################
print("Building Majority ensemble model")

tam             <- length(prediction_svm)
total_counter   <- 0
prediction_all2  <- 1:tam
linc <- NA
pct <- NA

# class labels
for (index in 1:tam){
  voting_linc <- 0
  voting_pct <- 0

  if(prediction_svm[index] == 'linc'){
    linc <- prediction_svm[index]
  }else if(prediction_knn[index] == 'pct'){
    pct <- prediction_svm[index]
  }

  if(!is.na(linc) && !is.na(pct)){
    print(pct)
    print(linc)

    break;
  }
}

# checks for prediction intersections
for (index in 1:tam){
  voting_linc <- 0
  voting_pct <- 0

  if(prediction_svm[index] == 'linc'){
    voting_linc <- voting_linc + 1

    if( accuracy ==  accuracy_svm){
      voting_linc <- voting_linc + 0.1
    }
  }
  if(prediction_rf[index] == 'linc'){
    voting_linc <- voting_linc + 1

    if( accuracy ==  accuracy_rf){
      voting_linc <- voting_linc + 0.1
    }
  }
  if(prediction_ct[index] == 'linc'){
    voting_linc <- voting_linc + 1

    if( accuracy ==  accuracy_ct){
      voting_linc <- voting_linc + 0.1
    }
  }
  if(prediction_knn[index] == 'linc'){
    voting_linc <- voting_linc + 1

    if( accuracy ==  accuracy_knn){
      voting_linc <- voting_linc + 0.1
    }
  }

  if(prediction_svm[index] == 'pct'){
    pct <- prediction_svm[index]
    voting_pct <- voting_pct + 1

    if( accuracy ==  accuracy_svm){
      voting_pct <- voting_pct + 0.1
    }
  }
  if(prediction_rf[index] == 'pct'){
    voting_pct <- voting_pct + 1

    if( accuracy ==  accuracy_rf){
      voting_pct <- voting_pct + 0.1
    }
  }
  if(prediction_ct[index] == 'pct'){
    voting_pct <- voting_pct + 1

    if( accuracy ==  accuracy_ct){
      voting_pct <- voting_pct + 0.1
    }
  }
  if(prediction_knn[index] == 'pct'){
    voting_pct <- voting_pct + 1

    if( accuracy ==  accuracy_knn){
      voting_pct <- voting_pct + 0.1
    }
  }

  if(voting_pct > 2){
    prediction_all2[index] <- pct
    total_counter <- total_counter + 1
  }else if(voting_linc > 2){
    prediction_all2[index] <- linc
    total_counter <- total_counter + 1
  }else{
    prediction_all2[index] <- NA
  }
}

# evaluation metrics
confusion_all2  <- table(pred = prediction_all2, true = test_filtered$Class)
print(confusion_all2)
accuracy = (confusion_all2[1,1] + confusion_all2[2,2])/total_counter
precision = confusion_all2[1,1]/sum(confusion_all2[,1])
recall = confusion_all2[1,1]/sum(confusion_all2[1,])
f = 2 * (precision * recall) / (precision + recall)
cat(paste("Accuracy:\t" , format(accuracy, digits=2) , "\n",sep=" "))
cat(paste("Precision:\t", format(precision, digits=2), "\n",sep=" "))
cat(paste("Recall:\t\t" , format(recall, digits=2)   , "\n",sep=" "))
cat(paste("F-measure:\t", format(f, digits=2)        , "\n",sep=" "))

######################################################################################
#               ROC AUC
######################################################################################
pred_svm        <- prediction(as.numeric(prediction_svm), as.numeric(test_filtered$Class))
performance_svm <- performance(pred_svm, "tpr","fpr")
auc_svm         <- performance(pred_svm, "auc")@y.values[[1]]
pred_rf         <- prediction(as.numeric(prediction_rf), as.numeric(test_filtered$Class))
performance_rf  <- performance(pred_rf, "tpr","fpr")
auc_rf          <- performance(pred_rf, "auc")@y.values[[1]]
pred_ct         <- prediction(as.numeric(prediction_ct), as.numeric(test_filtered$Class))
performance_ct  <- performance(pred_ct, "tpr","fpr")
auc_ct          <- performance(pred_ct, "auc")@y.values[[1]]
pred_knn        <- prediction(as.numeric(prediction_knn), as.numeric(test_filtered$Class))
performance_knn <- performance(pred_knn, "tpr","fpr")
auc_knn         <- performance(pred_knn, "auc")@y.values[[1]]
pred_all        <- prediction(as.numeric(prediction_all), as.numeric(test_filtered$Class))
performance_all <- performance(pred_all, "tpr","fpr")
auc_all         <- performance(pred_all, "auc")@y.values[[1]]
pred_all2        <- prediction(as.numeric(prediction_all2), as.numeric(test_filtered$Class))
performance_all2 <- performance(pred_all2, "tpr","fpr")
auc_all2         <- performance(pred_all2, "auc")@y.values[[1]]

png(filename="result/roc.png")
plot(performance_svm, col='blue', main="ROC curve comparing classification performance")
legend(0.6, 0.6, c('svm', 'rf', 'ctree','knn', 'majority', 'consensus'), 2:7, fill = c('blue','red','gray','green','lightblue','purple'))
plot(performance_rf, col='red', add=TRUE,borders = "black")
plot(performance_ct, col='gray', add=TRUE,borders = "black")
plot(performance_knn, col='green', add=TRUE,borders = "black")
plot(performance_all2, col='lightblue', add=TRUE,borders = "black")
plot(performance_all, col='purple', add=TRUE,borders = "black")
dev.off()

print("AUC model:")
print(auc_all)

######################################################################################
#               Print predicted lincs
######################################################################################
write.row <- function(x, file = filepath, append = TRUE, quote = TRUE, sep=",",
                      eol = "\n", na = "NA", dec = ".", row.names = FALSE, col.names = FALSE,
                      qmethod = c("escape", "double"), fileEncoding = ""){
  write.table(as.matrix(t(x)), file, append , quote, sep,
              eol, na, dec, row.names, col.names,
              qmethod, fileEncoding)
}

# print predicted lincs
tam             <- length(test$Class)
total_counter   <- 0

write.row("id", file="result/consensus_lincs.csv", append=FALSE, sep=";")

# checks for prediction intersections
for (index in 1:tam){
  if(!is.na(prediction_all[index])){
    # type = if(prediction_all[index] == 1) 'linc' else 'pct'
    if(prediction_all[index] == 1){
      if(!is.null(test$id[index])){
        write.row(toString(test$id[index]), file="result/consensus_lincs.csv", sep=";")
      }
    }
  }
}

# print predicted lincs
tam             <- length(test$Class)
total_counter   <- 0

write.row("id", file="result/majority_lincs.csv", append=FALSE, sep=";")

# checks for prediction intersections
for (index in 1:tam){
  if(!is.na(prediction_all2[index])){
    # type = if(prediction_all[index] == 1) 'linc' else 'pct'
    if(prediction_all2[index] == 1){
      if(!is.null(test$id[index])){
        write.row(toString(test$id[index]), file="result/majority_lincs.csv", sep=";")
      }
    }
  }
}


#r = read.table("/recurrency-modelling2/dat/dataA.out", header=F)$V1
#x = read.table("/recurrency-modelling2/dat/processed/CFBsampled.out", header=F)$V1
#c = read.table("/recurrency-modelling2/dat/processed/chpsraw.out", header=F)$V1
#cs = read.table(/recurrency-modelling2/dat/processed/chpssampled.out", header=F)$V1
##y = read.table("D:/_tmp/backup/r_sampled_dataA.out", header=F)$V1#plot(x, type="b")
#abline(v=csfa)
#lines(y, col = "red")
# PARAMS TO CALC PROBS FUNCTION IN CLOJURE
realchps = read.table("/recurrency-modelling2/dat/processed/chpssampled.out", header=F)$V1
mean(diff(realchps[1:12]))
sd(diff(realchps[1:12]))
max (diff(realchps[1:12]))
outliers = c(9, 13); realchps[outliers]
W = 60
MU = 36.73
# PROBS FUNCTION
pltcfb <- function(sigfile, chpsfile, probsfile, periodicprobs, outplotpdf){
  sig = read.table(sigfile, header = F)$V1
  realchps = read.table(chpsfile, header = F)$V1
  #
  probs = read.table(probsfile , header = F)$V1
  periodcprobs = read.table(periodicprobs, header = F)$V1
  #
  XLIMT1 = 0
  XLIMT2 = 400
  #
  pdf(outplotpdf)
  par(mfrow=c(2,1), mar = c(2,2,0,1) , oma = c(2,2,1,1))
  #plot(sig, xlim=c(XLIMT1,XLIMT2), type = 'b', xlab = "Time", ylab = "Signal", cex = 0.7, axes = F)
  plot(sig, xlim=c(XLIMT1,XLIMT2), type = 'b', ann = F, cex = 0.7, axes = FALSE)
  abline(v = realchps[-outliers], lw = 1.5)
  abline(v = realchps[outliers], lw = 2.5, lty=6)
  points(realchps[outliers], sig[realchps[outliers]], col="red", pch = 22, lwd =2)
  box(); axis(2);
  mtext(side = 2, "Signal", line = 3, cex = 1.1)
  # SECOND PLOT
  plot(c(rep(NA,15), probs), xlim=c(XLIMT1,XLIMT2), type = 'l',
       #xlab = "Time", ylab = "Confidence",
       ann = F, axes =  FALSE,
       lwd = 2.0, col = "gray20", ylim = c(0, 0.1))
  ##[43 0.030756629013991815] [86 0.02378568143631794] [129 0.02291349354654735]
  #abline(h=0.03); abline(h=0)
  abline(v = realchps[-outliers], lw = 1.5)
  abline(v = realchps[outliers], lw = 2.5, lty=6)

  ## changes from chpssampledFA.out
  lwd0 = 2.5
  lines(c(rep(NA, 85), probs ), lty=1, lwd=lwd0/2, col="gray25")
  lines(c(rep(NA, 164), probs), lty=1, lwd=lwd0/2, col="gray25")
  lines(c(rep(NA, 230), probs), lty=1, lwd=lwd0/3, col="gray27")
  axis(2); axis(1)
  mtext(side = 1, "Time", line = 2, cex = 1.0 )
  mtext(side = 2, "Confidence", line = 3, cex = 1.1)
  text(56, 0.095, "1", cex = 1.7)
  text(121, 0.095, "2", cex = 1.7)
  text(203, 0.095, "3", cex = 1.7)
  text(260, 0.095, "4", cex = 1.7)
  box()
  dev.off()
}
pltcfb("/recurrency-modelling2/dat/processed/CFBsampled.out",
       "/recurrency-modelling2/dat/processed/chpssampledFA.out",
       "/recurrency-modelling2/dat/probs/cfbprobsHalf.txt",
       "D:/_tmp/sim/ProbsFunction/PeriodicProbs.txt",
       "D:/_tmp/sim/cfbproof2.pdf")

#"recurrency-modelling2/dat/probs/cfbprobsHalf.txt"
#probs1 = read.table("/recurrency-modelling2/dat/probs/cfbprobsHalf.txt" , header = F)$V1
#probs2 = read.table("D:_tmp/sim/ProbsFunction/RecurrentProbs.txt" , header = F)$V1
#plot(probs1); lines(probs2)


context('test combining force_cos with null st constr (degenerate case)')
test_that("combine force_cos ", {
	fmax_info <- maximize_wrench(H_matrix, bounds_tuple_of_numeric, direction=c(1,0,0,0))
	st_constr_str <- force_cos_ramp_constraint(H_matrix, bounds_tuple_of_numeric, fmax_info$output_vector_per_task[[1]], 1.0,1.0, n_task_values=4)
	points <- st_constr_str$nonredundant_constr %>% pb_har_sample(1e5, mc.cores=8, eliminate=FALSE)
	har_tall_df <- trajectory_har_df_melt(points,ncol(H_matrix))
	a <- plot_har_trajectory(har_tall_df)
	ggsave(a, filename="no_st.jpg", height=40, width=40)

	context('test force cosine ramp with 0.5 constr (highly generous)')
	fmax_info <- maximize_wrench(H_matrix, bounds_tuple_of_numeric, direction=c(1,0,0,0))
	st_constr_str <- force_cos_ramp_constraint(H_matrix, bounds_tuple_of_numeric, fmax_info$output_vector_per_task[[1]], 0.5,0.5, n_task_values=4)
	points <- st_constr_str$nonredundant_constr %>% pb_har_sample(1e5, mc.cores=8, eliminate=FALSE)
	har_tall_df <- trajectory_har_df_melt(points,ncol(H_matrix))
	a <- plot_har_trajectory(har_tall_df)
	ggsave(a, filename="st.jpg", height=40, width=40)
})