# PlotUtil.r

# Helper functions to make using ggplot slightly less obscure --
#  at the cost of passing way too many arguments to these functions.

source("./DataUtil.r")
library(ggplot2)

# Constants for ggplot point shapes.
point.DOT       <- 16
point.SQUARE    <- 15
point.TRIANGLE  <- 17
point.DIAMOND   <- 18
point.HDOT      <- 0
point.HSQUARE   <- 1
point.HTRIANGLE <- 2
point.HDIAMOND  <- 5

# f n P l o t B e g i n
#fnPlotBegin <- function(dat, xcol, ycol) {
#    gp <- ggplot(data=trows,aes(x=xcol, y=ycol))
fnPlotBegin <- function(dat=NULL, xcol=NULL, ycol=NULL, context=NULL) {
    gp <- ggplot(data=dat, mapping=aes(x=xcol, y=ycol))
    return(gp)
}

# f n P l o t A d d L i n e
fnPlotAddLine <- function(gp=NULL, dat=NULL,
                        dotcolor="black", dotsize=2, dotshape=point.DOT,
                        linecolor="black", linesize=2, lineshape) {
    if (is.null(gp)) {print("ERROR: missing first argument = plot in progress")}
    else {if (debugprint) cat("enter addline\n")}

    gp <- gp +
            geom_point(data=dat,
                    color=dotcolor, size=dotsize, shape=dotshape
                    , show.legend=TRUE) +
            geom_line(data=dat,
                    color=linecolor, size=linesize, linetype=lineshape
                    , show.legend=TRUE)
    if (debugprint) cat("exit addline\n")
    return(gp)
}

# f n P l o t L o g S c a l e s
fnPlotLogScales <- function(gp, x=NULL, y=NULL) {
    if (is.null(gp)) {print("ERROR: missing first argument = plot in progress")}
    if (!is.null(x)) {
        gp <- gp + scale_x_log10() + annotation_logticks()
    }
    if (!is.null(y)) {
        gp <- gp + scale_y_log10() + annotation_logticks()
    }
    return(gp)
}

# f n P l o t T i t l e s
fnPlotTitles <- function(gp, titleline, titlesize=22,
                    xlabel="XXX", ylabel="YYY", labelsize=18) {
    if (is.null(gp)) {print("ERROR: missing first argument = plot in progress")}
    gp <- gp + labs(title=titleline, x=xlabel, y=ylabel)
    gp <- gp + theme(
                axis.text=element_text(size=12),
                axis.title=element_text(size=labelsize),
                plot.title=element_text(size=titlesize,face="bold"),
                panel.border = element_rect(color = "black", fill=NA, size=1)
                )
    return(gp)
}

# f n P l o t P e r c e n t L i n e
# Seems a bizarre way to do this, to create a fake data point
#  and then plot it using text centered there instead of just a dot.
fnPlotPercentLine <- function(gp,
                    xloc=log10(1.7), yloc=log10(1.2),
                    labeltext="1%", labelsize=4,
                    percent=1.0){
    gp <- gp + geom_hline(yintercept=percent, linetype="dashed")
    gp <- gp + geom_text(x=xloc, y=yloc, label=labeltext, size=labelsize,
                    fontface="plain", family="mono")
    return(gp)
}

# f n P l o t M a k e F i l e
fnPlotMakeFile <- function(plotname, sFilename, sSize="4x3") {
# Capture graph in 16:9 aspect and reasonable size.
    if (is.null(plotname)) {print("ERROR: missing first argument = plot in progress")}
    if (sSize == "large")
        { png(sFilename,width=1600,height=900) }
    else if (sSize == "mediumlarge")
        { png(sFilename,width=1200,height=675) }
    else if (sSize == "mediumsmall")
        { png(sFilename,width=960,height=540) }
    else if (sSize == "4x3")
        { png(sFilename,width=800,height=600) }
    else
        { png(sFilename,width=800,height=450) }
      print(plotname)
      dev.off()
} #endfunction


#' @title Rsquared
#' @description unknown
#' @param \code{o} is an lm, glm or gam object
#' @family abysmally documented
#' @author  unknown, \email{<unknown>@@dfo-mpo.gc.ca}
#' @export
  Rsquared <- function(o){
    # # one of the three link/variance combinations:
    # (1) log and mu (which is canonical for Poisson),
    # (2) logit and mu(1-mu) (which is canonical for binomial), or
    # (3) identity and constant (which is canonical for gaussian).
    # However these link/variance pairs may have been passed to quasi()
    # to allow for overdispersion.
      UseMethod("Rsquared")
    }


require(rCharts)

shinyServer(function(input, output) {
  output$myChart <- renderChart({
    names(iris) = gsub("\\.", "", names(iris))
    p1 <- rPlot(input$x, input$y, data = iris, color = "Species",
      facet = "Species", type = 'point')
    p1$addParams(dom = 'myChart')
    return(p1)
  })
})


relhum <- function(mu, pws, pa) {
    .Call('biometeoR_relhum', PACKAGE = 'biometeoR', mu, pws, pa)
}


#' Convert a watershed to an sf LINESTRING collection
#'
#' Currently the only attribute that is preserved is the reachID
#'
#' @params x A Watershed
#' @return An sf LINESTRING
#' @export
as.sf.Watershed = function(x) {
	reaches = unique(x$data$reachID)
	res = parallel::mclapply(reaches, function(r) {
		i = which(x$data$reachID == r)
		.reach_to_sf(x$data[i,], x$adjacency[i,i, drop=FALSE])
	})
	do.call(sf:::rbind.sf, res)
}

#' Produce an sf LINESTRING from a single reach
#' @param x data portion of a watershed, containing only a single reach
#' @param adj Adjacency matrix, just for the reach
.reach_to_sf = function(x, adj) {
	if(nrow(adj) == 1)
		return(NULL)
	ord = vector("integer", length=nrow(adj))
	top = which(Matrix::rowSums(adj) == 0)
	bottom = which(Matrix::colSums(adj) == 0)
	visited = 0
	current = top
	while(current != bottom) {
		visited = visited + 1
		ord[current] = visited
		current = which(adj[,current] != 0)
		if(visited == nrow(adj)) {
			stop("Toplogical error")
		}
	}
	ord[bottom] = nrow(adj)
	x = sf::st_as_sf(x)
	x = x[order(ord),]
	res = sf::st_combine(x)
	res = sf::st_cast(res, "LINESTRING")
	res = sf::st_sf(res)
	res$reachID = x$reachID[1]
	res
}

#' @name [.Watershed
#' @aliases names.Watershed
#' @aliases head.Watershed
#' @aliases summary.Watershed
#' @aliases plot.Watershed
#'
#' @title S3 methods for watersheds
#' @param x A watershed object
#' @param i A vector of pixel IDs
#' @param j Variable names or column numbers to extract
#' @param variable Optional, name of variable to plot
#' @param transform A transformation function (e.g., `log`) to apply to the variable
#' @param size Size of the plotted points
#' @param xlim Optional x-axis limits
#' @param ylim Optional y-axis limits
#' @param ... Additional parameters to pass to other methods
#' 		(usually from [data.frame] or [SpatialPointsDataFrame])
#' @return A data.frame of the watershed data
#' @rdname methods
#' @export
`[.Watershed` <- function(x, i, j, ...) {
	if(missing(i)) {
		as.data.frame(x$data[,, ...])[,j]
	} else {
		as.data.frame(x$data)[match(i, x$data$id),j]
	}
}

#' @rdname methods
#' @return A vector of variable names
#' @export
names.Watershed <- function(x) {
	names(x$data)
}

#' @rdname methods
#' @return As [head.data.frame()]
#' @export
head.Watershed <- function(x, ...) head(x$data, ...)

#' @rdname methods
#' @return As [summary.data.frame()]
#' @export
summary.Watershed <- function(x, ...) summary(x$data, ...)

#' @rdname methods
#' @export
plot.Watershed <- function(x, variable, transform, size = 0.5, xlim, ylim, viridis_col = "inferno")
{
	if(!missing(transform))
		x$data[[variable]] <- transform(x$data[[variable]])
	pl <- ggplot2::ggplot(as.data.frame(x$data), ggplot2::aes(x=x, y=y))
	pl <- pl + ggplot2::geom_point(size=size)
	if(!missing(variable)){
		pl <- pl + ggplot2::aes_string(color=variable)
		if(requireNamespace("viridisLite")) {
			if(is.factor(x$data[[variable]])) {
				pl <- pl + ggplot2::scale_colour_viridis_d(option = viridis_col)
			} else {
				pl <- pl + ggplot2::scale_colour_viridis_c(option = viridis_col)
			}
		}
	}
	if(!missing(xlim))
		pl <- pl + ggplot2::xlim(xlim)
	if(!missing(ylim))
		pl <- pl + ggplot2::ylim(ylim)
	pl <- pl + ggplot2::coord_fixed()
	pl
}

## Dont forget assignment methods
## e.g.,
## `[<-.Watershed` <- function(x, i, j, value) {
	# x$data[i,j] <- value
	# x
## }
## also `$<-.Watershed`
## also add a subset function


# Compares  mean stem and crown ages for dating analyses implementing different fossil calibration sets.
# - SA = Constrained Calibration (CC)
# - SB = Relaxed Calibration (RC)
# - SD = Unconstrained Calibration (UC)
# - phylo = conservative set
# - complete = complete set.
# Uses additional files.
# NOTE TO SELF: this can easily be automated! Currently needs to be edited with the names for each of the three dating analyses with different calibration strategies.
# NOTE TO SELF: probably also need to homogenize strategies' codes.

rich<-read.table("SPP.RICHNESS.csv",heade=T,sep=",") # file in data folder
resSTRICT<-read.table("RES_SBphylo_v2/2.Ages_complete.csv",sep=",",header=T) # generated by the preceding scripts
resRAW<-read.table("RES_SBcomplete_v2/2.Ages_complete.csv",sep=",",header=T) # generated by the preceding scripts
pdf("1.ComparisonFossils_RC.pdf")
rich_ord<-rich[which(rich$Family==""),]; dim(rich_ord)
resSTRICT$Order_richness<-rich_ord[match(resSTRICT$Order,rich_ord$Order),"Spp_richness"]
rich_fams<-rich[which(rich$Subfamily==""),]; dim(rich_fams)
rich_fams<-rich_fams[which(rich_fams$Family!=""),]; dim(rich_fams)
resSTRICT$Family_richness<-rich_fams[match(resSTRICT$Family,rich_fams$Family),"Spp_richness"]
rich_subfams<-rich[which(rich$Subfamily!=""),]; dim(rich_subfams)
resSTRICT$Subfamily_richness<-rich_subfams[match(resSTRICT$Subfamily,rich_subfams$Subfamily),"Spp_richness"]
non<-which(rownames(resSTRICT) %in% c("Aphloiaceae", "Daphniphyllaceae",
                                      "Brunelliaceae", "Cynomoriaceae", "Mayacaceae",
                                      "Mitrastemonaceae", "Oncothecaceae"))
resSTRICT[non,"CG_age_Acal"]<-NA
resSTRICT$Total_Richness <- rep(NA,dim(resSTRICT)[1])
for (i in 1:dim(resSTRICT)[1]){
  if(!is.na(resSTRICT$Subfamily_richness[i])){resSTRICT$Total_Richness[i] <- resSTRICT$Subfamily_richness[i];next}
  if(!is.na(resSTRICT$Family_richness[i])){resSTRICT$Total_Richness[i] <- resSTRICT$Family_richness[i];next}
  resSTRICT$Total_Richness[i] <- resSTRICT$Order_richness[i]}
out.group<-which(resSTRICT$Order%in%c("Pinales","Gnetales","Ginkgoales","Cycadales"))
resSTRICT<-resSTRICT[-out.group,]
resRAW$Order_richness<-rich_ord[match(resRAW$Order,rich_ord$Order),"Spp_richness"]
rich_fams<-rich[which(rich$Subfamily==""),]; dim(rich_fams)
rich_fams<-rich_fams[which(rich_fams$Family!=""),]; dim(rich_fams)
resRAW$Family_richness<-rich_fams[match(resRAW$Family,rich_fams$Family),"Spp_richness"]
rich_subfams<-rich[which(rich$Subfamily!=""),]; dim(rich_subfams)
resRAW$Subfamily_richness<-rich_subfams[match(resRAW$Subfamily,rich_subfams$Subfamily),"Spp_richness"]
non<-which(rownames(resRAW) %in% c("Aphloiaceae", "Daphniphyllaceae",
                                   "Brunelliaceae", "Cynomoriaceae", "Mayacaceae",
                                   "Mitrastemonaceae", "Oncothecaceae"))
resRAW[non,"CG_age_Acal"]<-NA
resRAW$Total_Richness <- rep(NA,dim(resRAW)[1])
for (i in 1:dim(resRAW)[1]){
  if(!is.na(resRAW$Subfamily_richness[i])){resRAW$Total_Richness[i] <- resRAW$Subfamily_richness[i];next}
  if(!is.na(resRAW$Family_richness[i])){resRAW$Total_Richness[i] <- resRAW$Family_richness[i];next}
  resRAW$Total_Richness[i] <- resRAW$Order_richness[i]}
out.group<-which(resRAW$Order%in%c("Pinales","Gnetales","Ginkgoales","Cycadales"))
resRAW<-resRAW[-out.group,]
orders<-1:64
fams<-65:500
subfams<-501:dim(resRAW)[1]
print(unique(rownames(resRAW)==rownames(resSTRICT)))
resRAW<-resRAW[fams,]
resSTRICT<-resSTRICT[fams,]

plot(resRAW$SG_age_Acal,resSTRICT$SG_age_Acal,type="n",ylim=c(0,200),xlim=c(0,200),
     xlab="Stem ages (complete set)",ylab="Stem ages (phylo-only set)");abline(a=0,b=1)
title("Stem age estimates (families)")
for (i in 1:length(resSTRICT$SG_age_Acal)) segments(y0=resRAW$SG_minHPD[i],y1=resRAW$SG_maxHPD[i],
                                                          x0=resSTRICT$SG_age_Acal[i],x1=resSTRICT$SG_age_Acal[i],lwd = 0.5,col="grey40")
for (i in 1:length(resSTRICT$SG_age_Acal)) segments(x0=resSTRICT$SG_minHPD[i],x1=resSTRICT$SG_maxHPD[i],
                                                          y0=resRAW$SG_age_Acal[i],y1=resRAW$SG_age_Acal[i],lwd = 0.5,col="grey40")
points(resSTRICT$SG_age_Acal,resRAW$SG_age_Acal,pch=21,cex=1,bg=rgb(1,0,0,0.8))
plot(resSTRICT$CG_age_Acal,resRAW$CG_age_Acal,type="n",ylim=c(0,200),xlim=c(0,200),
     xlab="Crown ages (complete set)",ylab="Crown ages (phylo-only set)");abline(a=0,b=1)
title("Crown age estimates (families)")
for (i in 1:length(resSTRICT$SG_age_Acal)) segments(y0=resRAW$CG_minHPD[i],y1=resRAW$CG_maxHPD[i],
                                                          x0=resSTRICT$CG_age_Acal[i],x1=resSTRICT$CG_age_Acal[i],lwd = 0.5,col="grey40")
for (i in 1:length(resSTRICT$SG_age_Acal)) segments(x0=resSTRICT$CG_minHPD[i],x1=resSTRICT$CG_maxHPD[i],
                                                          y0=resRAW$CG_age_Acal[i],y1=resRAW$CG_age_Acal[i],lwd = 0.5,col="grey40")
points(resSTRICT$CG_age_Acal,resRAW$CG_age_Acal,pch=21,cex=1,bg=rgb(1,0,0,0.8))
HPD_complete<-resRAW$CG_maxHPD-resRAW$CG_minHPD
HPD_strict<-resSTRICT$CG_maxHPD-resSTRICT$CG_minHPD
HPDsC<-cbind(HPD_complete,HPD_strict)
boxplot(HPDsC,pch=19,cex=0.6,col=c("grey80","grey30"),pch=19,cex=0.5,
        ylab="95% HPD crown ages")
title("HPD crown age estimates")

HPD_complete<-resRAW$SG_maxHPD-resRAW$SG_minHPD
HPD_strict<-resSTRICT$SG_maxHPD-resSTRICT$SG_minHPD
HPDsC<-cbind(HPD_complete,HPD_strict)
boxplot(HPDsC,pch=19,cex=0.6,col=c("grey80","grey30"),pch=19,cex=0.5,
        ylab="95% HPD stem ages")
title("HPD stem age estimates")
dev.off()


#!/usr/bin/env Rscript

args <- commandArgs(trailingonly = TRUE)
if (length(args)==0) {
  stop ("Path not supplied\n")
}
datapath <- args[1]

halolist <- c(1,2,4,7,8,9,11,12)

getdata <- function(id) {
  fileroot0 <- paste(datapath,"/v1-",id,"-0-0.3-0/", sep="")
  fileroot1 <- paste(datapath,"/v1-",id,"-0-0.3-0.5/", sep="")

  data <- read.table(paste(fileroot0,"obscurve/r2-histogram.txt", sep=""), head=FALSE)
  r2obs1 <- data[[1]][which.max(data[[2]])]

  data <- read.table(paste(fileroot0,"obscurve/gamma-histogram.txt", sep=""), head=FALSE)
  gammaobs1 <- abs(data[[1]][which.max(data[[2]])])

  data <- read.table(paste(fileroot0,"obscurve/vmax-histogram.txt", sep=""), head=FALSE)
  vmax1 <- data[[1]][which.max(data[[2]])]

  data <- read.table(paste(fileroot1,"obscurve/r2-histogram.txt", sep=""), head=FALSE)
  r2obs2 <- data[[1]][which.max(data[[2]])]

  data <- read.table(paste(fileroot1,"obscurve/gamma-histogram.txt", sep=""), head=FALSE)
  gammaobs2 <- abs(data[[1]][which.max(data[[2]])])

  data <- read.table(paste(fileroot1,"obscurve/vmax-histogram.txt", sep=""), head=FALSE)
  vmax2 <- data[[1]][which.max(data[[2]])]

  data <- read.table(paste(fileroot0,"realcurve/r2-histogram.txt", sep=""), head=FALSE)
  r2real <- data[[1]][which.max(data[[2]])]

  data <- read.table(paste(fileroot0,"realcurve/gamma-histogram.txt", sep=""), head=FALSE)
  gammareal <- abs(data[[1]][which.max(data[[2]])])

  data <- read.table(paste(fileroot0,"trcurve.txt", sep=""), head=TRUE)
  Rd <- 1000*data$r[which.max(data$stars)]
  vmax <- max(data$v)

  return (c(Rd,vmax,r2real,gammareal,r2obs1,gammaobs1,vmax1,r2obs2,gammaobs2,vmax2))
}

sink("dwarfdata.txt")
cat("Rd vmax r2 gamma r2_a gamma_a vmax_a r2_b gamma_b vmax_b\n")
for (id in halolist) {
  cat(getdata(id))
  cat("\n")
}
sink()

data <- read.table("dwarfdata.txt", head=TRUE)

pdf("dwplot1.pdf")
plot(data$r2/data$Rd, data$gamma, pch=19, cex=0.5, ylim=c(0,2), xlab=expression(r[2]/R[D]), ylab=expression(gamma))
points(data$r2_a/data$Rd, data$gamma_a, pch=19, cex=0.5,col="red")
segments(data$r2/data$Rd,data$gamma, data$r2_a/data$Rd, data$gamma_a, col="red")
dev.off()

pdf("realgammahist.pdf")
hist(data$gamma, xlim=c(0,2), breaks=10, xlab=expression(gamma), ylab="n", main=NULL)
dev.off()

pdf("obsgammahist.pdf")
hist(c(data$gamma_a, data$gamma_b), xlim=c(0,2), breaks=10, xlab=expression(gamma), ylab="n", main=NULL)
dev.off()

pdf("result1.pdf")
n <- c(1,3,4,5,6,7,8)
plot(data$gamma[n], data$vmax[n], xlim=c(0,2), ylim=c(0,250), xlab=expression(gamma), ylab=expression(v[max]), pch=seq(length(n)))
points(data$gamma_a[n], data$vmax[n], pch=seq(length(n)), col="red")
points(data$gamma_b[n], data$vmax[n], pch=seq(length(n)), col="blue")
dev.off()


# SNOPSIS

# runs k-means cluster analysis

# AUTHOR Isaak Y Tecle (iyt2@cornell.edu)

options(echo = FALSE)

library(randomForest)
library(data.table)
library(genoDataFilter)
library(tibble)
library(dplyr)
library(fpc)
library(cluster)
library(ggfortify)
library(tibble)
library(stringi)
library(phenoAnalysis)
library(factoextra)

allArgs <- commandArgs()

outputFiles <- grep("output_files", allArgs, value = TRUE)
outputFiles <- scan(outputFiles, what = "character")

inputFiles <- grep("input_files", allArgs, value = TRUE)
inputFiles <- scan(inputFiles, what = "character")

optionsFile <- grep("options", inputFiles, value = TRUE)
clusterOptions <- read.table(optionsFile, header = TRUE, sep = "\t", stringsAsFactors = FALSE,
    na.strings = "")
print(clusterOptions)

kmeansPlotFile <- grep("k-means_plot", outputFiles, value = TRUE)
kResultFile <- grep("result", outputFiles, value = TRUE)
elbowPlotFile <- grep("elbow_plot", outputFiles, value = TRUE)
clusterMeansFile <- grep("k-means_means", outputFiles, value = TRUE)
variancesFile <- grep("k-means_variances", outputFiles, value = TRUE)
reportFile <- grep("report", outputFiles, value = TRUE)
errorFile <- grep("error", outputFiles, value = TRUE)

combinedDataFile <- grep("combined_cluster_data_file", outputFiles, value = TRUE)

clusterOptions <- column_to_rownames(clusterOptions, var = "Params")
userKNumbers <- as.numeric(clusterOptions["k_numbers", 1])
dataType <- clusterOptions["data_type", 1]
selectionProp <- as.numeric(clusterOptions["selection_proportion", 1])
predictedTraits <- clusterOptions["predicted_traits", 1]
predictedTraits <- unlist(strsplit(predictedTraits, ","))

if (is.null(kResultFile)) {
    stop("Clustering output file is missing.")
    q("no", 1, FALSE)
}

clusterData <- c()
genoData <- c()
genoFiles <- c()
reportNotes <- c()
genoDataMissing <- c()

extractGenotype <- function(inputFiles) {

    genoFiles <- grep("genotype_data", inputFiles, value = TRUE)

    genoMetaData <- c()
    filteredGenoFile <- c()

    if (length(genoFiles) > 1) {
        genoData <- combineGenoData(genoFiles)

        genoMetaData <- genoData$trial
        genoData$trial <- NULL

    } else {
        genoFile <- genoFiles
        genoData <- fread(genoFile, header = TRUE, na.strings = c("NA", " ", "--",
            "-", "."))

        if (is.null(genoData)) {
            filteredGenoFile <- grep("filtered_genotype_data_", genoFile, value = TRUE)
            genoData <- fread(filteredGenoFile, header = TRUE)
        }

        genoData <- unique(genoData, by = "V1")
        genoData <- data.frame(genoData)
        genoData <- column_to_rownames(genoData, "V1")
    }

    if (is.null(genoData)) {
        stop("There is no genotype dataset.")
        q("no", 1, FALSE)
    } else {

        ## genoDataFilter::filterGenoData
        genoData <- convertToNumeric(genoData)
        genoData <- filterGenoData(genoData, maf = 0.01)
        genoData <- roundAlleleDosage(genoData)

        message("No. of geno missing values, ", sum(is.na(genoData)))
        if (sum(is.na(genoData)) > 0) {
            genoDataMissing <- c("yes")
            genoData <- na.roughfix(genoData)
        }

        genoData <- data.frame(genoData)
    }
}

set.seed(235)

clusterDataNotScaled <- c()

if (grepl("genotype", dataType, ignore.case = TRUE)) {
    clusterData <- extractGenotype(inputFiles)

    pca <- prcomp(clusterData, retx = TRUE)
    pca <- summary(pca)

    variances <- data.frame(pca$importance)

    varProp <- variances[3, ]
    varProp <- data.frame(t(varProp))
    names(varProp) <- c("cumVar")

    selectPcs <- varProp %>%
        filter(cumVar <= 0.9)
    pcsCnt <- nrow(selectPcs)

    reportNotes <- paste0("Before clustering this dataset, principal component analysis (PCA) was perforemd on it to reduce the number of variables (dimensions). ")
    reportNotes <- paste0(reportNotes, "Based on the PCA, ", pcsCnt, " PCs were used to do the clustering. ")
    reportNotes <- paste0(reportNotes, "\n\nThe ", pcsCnt, " PCs explain 90% of the variance in the original dataset.",
        "\n")

    scores <- data.frame(pca$x)
    scores <- scores[, 1:pcsCnt]
    scores <- round(scores, 3)

    variances <- variances[2, 1:pcsCnt]
    variances <- round(variances, 4) * 100
    variances <- data.frame(t(variances))

    clusterData <- scores
} else {

    if (grepl("gebv", dataType, ignore.case = TRUE)) {
        gebvsFile <- grep("combined_gebvs", inputFiles, value = TRUE)
        gebvsData <- data.frame(fread(gebvsFile, header = TRUE))

        clusterNa <- gebvsData %>%
            filter_all(any_vars(is.na(.)))
        clusterData <- column_to_rownames(gebvsData, "V1")
    } else if (grepl("phenotype", dataType, ignore.case = TRUE)) {

        metaFile <- grep("meta", inputFiles, value = TRUE)

        clusterData <- cleanAveragePhenotypes(inputFiles, metaDataFile = metaFile)

        if (!is.na(predictedTraits) & length(predictedTraits) > 1) {
            clusterData <- rownames_to_column(clusterData, var = "germplasmName")
            clusterData <- clusterData %>%
                select(c(germplasmName, predictedTraits))
            clusterData <- column_to_rownames(clusterData, var = "germplasmName")
        }
    }

    clusterData <- clusterData[, apply(clusterData, 2, function(x) var(x, na.rm=TRUE)) != 0]
    clusterDataNotScaled <- na.omit(clusterData)
    clusterData <- scale(clusterDataNotScaled, center = TRUE, scale = TRUE)
    clusterData <- round(clusterData, 3)
    reportNotes <- paste0(reportNotes, "Note: Data was standardized before clustering.",
        "\n")
}
sIndexFile <- grep("selection_index", inputFiles, value = TRUE)
selectedIndexGenotypes <- c()

if (length(sIndexFile) != 0) {
    sIndexData <- data.frame(fread(sIndexFile, header = TRUE))
    selectionProp <- selectionProp * 0.01
    selectedIndexGenotypes <- sIndexData %>%
        top_frac(selectionProp)

    selectedIndexGenotypes <- column_to_rownames(selectedIndexGenotypes, var = "V1")

    if (!is.null(selectedIndexGenotypes)) {
        clusterData <- rownames_to_column(clusterData, var = "germplasmName")
        clusterData <- clusterData %>%
            filter(germplasmName %in% rownames(selectedIndexGenotypes))

        clusterData <- column_to_rownames(clusterData, var = "germplasmName")
    }
}

kMeansOut <- kmeansruns(clusterData, runs = 10)
kCenters <- kMeansOut$bestk

if (!is.na(userKNumbers)) {
    if (userKNumbers != 0) {
        kCenters <- as.integer(userKNumbers)
        reportNotes <- paste0(reportNotes, "\n\nThe data was partitioned into ", userKNumbers,
            " clusters.\n")
    }
}

reportNotes <- paste0(reportNotes, "\n\nAccording the kmeansruns algorithm from the fpc R package, the recommended number of clusters (k) for this data set is: ",
    kCenters, "\n\nYou can also check the Elbow plot to evaluate how many clusters may be better suited for your purpose.")

kMeansOut <- kmeans(clusterData, centers = kCenters, nstart = 10)
kClusters <- data.frame(kMeansOut$cluster)
kClusters <- rownames_to_column(kClusters)
names(kClusters) <- c("germplasmName", "Cluster")

if (!is.null(clusterDataNotScaled)) {
    clusterDataNotScaled <- rownames_to_column(clusterDataNotScaled, var = "germplasmName")

    clusteredData <- inner_join(kClusters, clusterDataNotScaled, by = "germplasmName")
} else if (!is.null(selectedIndexGenotypes)) {
    selectedIndexGenotypes <- rownames_to_column(selectedIndexGenotypes, var = "germplasmName")
    clusteredData <- inner_join(kClusters, selectedIndexGenotypes, by = "germplasmName")
} else {
    clusteredData <- kClusters
}

clusteredData <- clusteredData %>%
    mutate_if(is.double, round, 2) %>%
    arrange(Cluster)

# print(paste('size: ', '\n', '\n', round(kMeansOut$size, 2)))
# print(paste('centers: ','\n', round(kMeansOut$centers, 2)))

if (length(elbowPlotFile) & !file.info(elbowPlotFile)$size) {
    message("running elbow method...")
    png(elbowPlotFile)
    print(fviz_nbclust(clusterData, k.max = 20, FUNcluster = kmeans, method = "wss"))
    dev.off()
}

png(kmeansPlotFile)
ggplot2::autoplot(kMeansOut, data = clusterData, frame = TRUE, x = 1, y = 2)
# fviz_cluster(kMeansOut, geom = "point", main = "", data = clusterData)
dev.off()

clusterMeans <- c()
if (!grepl('genotype', kResultFile)) {
    message('adding cluster means to clusters...')
    clusterMeans <- aggregate(clusterDataNotScaled, by = list(cluster = kMeansOut$cluster),
    mean)

    clusterMeans <- clusterMeans %>%
        select(-germplasmName) %>%
        mutate_if(is.double, round, 2)

}

cat(reportNotes, file = reportFile, sep = "\n", append = TRUE)

variances <- paste0("Variances output: ")
variances <- append(variances, (paste0("\nBetween clusters sum of squares (betweenss): ",
    "\t", round(kMeansOut$betweenss, 2))))
withinss <- round(kMeansOut$withinss, 2)
withinss <- c(paste0("\nWithin clusters sum of squares (withinss): "), "\t", paste("\n\t",
    withinss))
variances <- append(variances, withinss)
variances <- append(variances, (paste0("\nTotal within cluster sum of squares (tot.withinss): ",
    "\t", round(kMeansOut$tot.withinss, 2))))

variances <- append(variances, (paste0("\nTotal sum of squares (totss): ", "\t",
    round(kMeansOut$totss, 2))))

cat(variances, file = variancesFile, sep = "", append = TRUE)

if (length(genoFiles) > 1) {
    fwrite(genoData, file = combinedDataFile, sep = "\t", row.names = TRUE, quote = FALSE,
        )
}

if (length(kResultFile)) {
    fwrite(clusteredData, file = kResultFile, sep = "\t", row.names = FALSE, quote = FALSE,
        )
}

if (length(clusterMeansFile) && !is.null(clusterMeans)) {
    fwrite(clusterMeans, file = clusterMeansFile, sep = "\t", row.names = FALSE,
        quote = FALSE, )
}

message("Done clustering.")

####
q(save = "no", runLast = FALSE)
####


###VISUALISE MODEL OUTPUTS###
# install.packages("tidyverse")
library(tidyverse)
library(ggpubr)

####################SUPPLY-DEMAND METRICS
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

data <- read.csv(file.path(folder, '..', 'results', 'decile_mno_results_technology_options.csv'))

names(data)[names(data) == 'GID_0'] <- 'country'

#select desired columns
data <- select(data, country, scenario, strategy, confidence, decile, #population, area_km2,
               population_km2, total_estimated_sites, existing_mno_sites,
               total_estimated_sites_km2, existing_mno_sites_km2,
               phone_density_on_network_km2, sp_density_on_network_km2,
               total_mno_revenue, total_mno_cost, cost_per_network_user,
               # cost_per_sp_user
               )

data <- data[(data$confidence == 50),]

data$combined <- paste(data$country, data$scenario, sep="_")

data$country = factor(data$country, levels=c('MWI',
                                             "UGA",
                                             "SEN",
                                             "KEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Malawi","Uganda",
                               "Senegal","Kenya",
                               "Pakistan",
                               "Albania",
                               "Peru",
                               "Mexico"))

demand = data[(
  data$scenario == 'S1_25_10_2' &
  data$strategy == '4G_epc_fiber_baseline_baseline_baseline_baseline'
    ),]

demand <- select(demand, country, decile, population_km2,
                 phone_density_on_network_km2,
                 sp_density_on_network_km2)

demand <- gather(demand, metric, value, population_km2:sp_density_on_network_km2)

demand$metric = factor(demand$metric,
                     levels=c("population_km2",
                             "phone_density_on_network_km2",
                             "sp_density_on_network_km2"),
                       labels=c("Population Density",
                                "Modeled Network Phone Density",
                                "Modeled Network Smartphone Density"))

demand_densities <- ggplot(demand, aes(x=decile, y=value, colour=metric, group=metric)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme( legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(colour=NULL,
       title = "Demand-Side Density Metrics by Population Decile",
       # subtitle = "Population and user densities",
       x = "Population Decile", y = "Density (per km^2)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) + #, limits = c(0,20)) +
  theme(panel.spacing = unit(0.6, "lines")) + expand_limits(y=0) +
  guides(colour=guide_legend(ncol=3)) +
  facet_wrap(~country, scales = "free", ncol=4)

supply = data[(
  data$scenario == 'S1_25_10_2' &
    data$strategy == '4G_epc_fiber_baseline_baseline_baseline_baseline'
),]

supply <- select(supply, country, decile, total_estimated_sites_km2, existing_mno_sites_km2)

supply <- gather(supply, metric, value, total_estimated_sites_km2:existing_mno_sites_km2)

supply$metric = factor(supply$metric,
                       levels=c("total_estimated_sites_km2",
                                "existing_mno_sites_km2"),
                       labels=c("Total Site Density",
                                "Modeled Network Site Density"))

supply_densities <- ggplot(supply, aes(x=decile, y=value, colour=metric, group=metric)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme( legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(colour=NULL,
       title = "Supply-Side Density Metrics by Population Decile",
       # subtitle = "Cumulative cost reported by percentage of population covered",
       x = "Population decile", y = "Density (per km^2)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) + #, limits = c(0,20)) +
  theme(panel.spacing = unit(0.6, "lines")) + expand_limits(y=0) +
  guides(colour=guide_legend(ncol=3)) +
  facet_wrap(~country, scales = "free", ncol=4)

demand_supply <- ggarrange(demand_densities, supply_densities, ncol = 1, nrow = 2, align = c("hv"))

#export to folder
path = file.path(folder, 'figures', 'a_demand_supply_panel.png')
ggsave(path,  units="in", width=8, height=9, dpi=300)
print(demand_supply)
dev.off()

####################TECHNOLOGIES BY DECILE
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

data <- read.csv(file.path(folder, '..', 'results', 'decile_market_results_technology_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]

names(data)[names(data) == 'GID_0'] <- 'country'

#select desired columns
data <- select(data, country, scenario, strategy, confidence, decile,
               area_km2, population,
               total_market_cost, total_market_revenue)

data <- data[(data$confidence == 50),]

data$combined <- paste(data$country, data$scenario, sep="_")

data$scenario = factor(data$scenario, levels=c("S1_25_10_2",
                                               "S2_200_50_5",
                                               "S3_400_100_10"),
                       labels=c("S1 (25 Mbps)",
                                "S2 (200 Mbps)",
                                "S3 (400 Mbps)"))

data$country = factor(data$country, levels=c('MWI',
                                             "UGA",
                                             "SEN",
                                             "KEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Malawi","Uganda",
                               "Senegal","Kenya",
                               "Pakistan",
                               "Albania",
                               "Peru",
                               "Mexico"))

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
         "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
         "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
          "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
          "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
          "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
          "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
          "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data <- data[order(data$country, data$scenario, data$strategy, data$decile),]

data1 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_revenue)
data1 <- data1[(data1$strategy == "4G_epc_microwave_baseline_baseline_baseline_baseline"),]
data1$strategy <- "Revenue"
names(data1)[names(data1) == 'total_market_revenue'] <- 'value'
data2 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_cost)
names(data2)[names(data2) == 'total_market_cost'] <- 'value'
data <- rbind(data1, data2)
remove(data1, data2)

data$strategy = factor(data$strategy, levels=c("Revenue",
                                               "4G_epc_microwave_baseline_baseline_baseline_baseline",
                                               "4G_epc_fiber_baseline_baseline_baseline_baseline",
                                               "5G_nsa_microwave_baseline_baseline_baseline_baseline",
                                               "5G_sa_fiber_baseline_baseline_baseline_baseline"),
                       labels=c("Revenue",
                                "4G (Microwave)",
                                "4G (Fiber)",
                                "5G NSA (Microwave)",
                                "5G SA (Fiber)"))

data <- data[order(data$combined, data$country, data$scenario, data$strategy, data$decile),]

data <- data %>%
  group_by(combined, country, scenario, strategy) %>%
  mutate(cumulative_value_bn = cumsum(round(value / 1e9, 3)))

panel <- ggplot(data, aes(x=decile, y=cumulative_value_bn, colour=strategy, group=strategy)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme( legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(colour=NULL,
       title = "Viability of Technology Decisions (4G vs 5G & Fiber vs Microwave)",
       subtitle = "Cumulative cost reported by percentage of population covered",
       x = "Population Covered (%)", y = "Cumulative Cost (Billions $USD)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) + #, limits = c(0,20)) +
  theme(panel.spacing = unit(0.6, "lines")) + expand_limits(y=0) +
  guides(colour=guide_legend(ncol=5)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'b_results_technology_options_wrap.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

##################BUSINESS MODELS BY DECILE
#load data
data <- read.csv(file.path(folder, '..', 'results', 'decile_market_results_business_model_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]

data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

data$combined <- paste(data$country, data$scenario, sep="_")

data$scenario = factor(data$scenario, levels=c("S1_25_10_2",
                                               "S2_200_50_5",
                                               "S3_400_100_10"),
                       labels=c("S1 (25 Mbps)",
                                "S2 (200 Mbps)",
                                "S3 (400 Mbps)"))

data$country = factor(data$country, levels=c("UGA",
                                             'MWI',
                                             "KEN",
                                             "SEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Uganda",
                               "Malawi",
                               "Kenya",
                               "Senegal",
                               "Pakistan",
                               "Albania",
                               "Peru",
                               "Mexico"))

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data1 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_revenue)
data1 <- data1[(data1$strategy == "5G_nsa_microwave_baseline_baseline_baseline_baseline"),]
data1$strategy <- "Revenue"
names(data1)[names(data1) == 'total_market_revenue'] <- 'value'
data2 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_cost)
names(data2)[names(data2) == 'total_market_cost'] <- 'value'
data <- rbind(data1, data2)
remove(data1, data2)

data <- data[(
  data$strategy == 'Revenue' |
    data$strategy == '5G_nsa_microwave_baseline_baseline_baseline_baseline' |
    data$strategy == '5G_nsa_microwave_passive_baseline_baseline_baseline' |
    data$strategy == '5G_nsa_microwave_active_baseline_baseline_baseline' |
    data$strategy == '5G_nsa_microwave_srn_baseline_baseline_baseline'),]

data <- data[!(data$value == "NA"),]

data$strategy = factor(data$strategy, levels=c("Revenue",
                        "5G_nsa_microwave_baseline_baseline_baseline_baseline",
                         "5G_nsa_microwave_passive_baseline_baseline_baseline",
                         "5G_nsa_microwave_active_baseline_baseline_baseline",
                         "5G_nsa_microwave_srn_baseline_baseline_baseline"),
                labels=c("Revenue",
                        "Baseline (No sharing)",
                        "Passive (Site Sharing)",
                        "Active (RAN and Site Sharing)",
                        "Single Wholesale Network"))

data <- data[order(data$combined, data$country, data$scenario, data$strategy, data$decile),]

data <- data %>%
  group_by(combined, country, strategy, scenario) %>%
  mutate(cumulative_value_bn = cumsum(round(value / 1e9, 3)))

panel <- ggplot(data, aes(x=decile, y=cumulative_value_bn, colour=strategy, group=strategy)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme(legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(title = "Impact of Infrastructure Sharing by Scenario, Decile and Country", colour=NULL,
       subtitle = "Cumulative cost reported by percentage of population covered for 5G NSA (Microwave)",
       x = "Population Covered (%)", y = "Cumulative Cost (Billions $USD)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(colour=guide_legend(ncol=5)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'c_results_business_model_options_wrap.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

##################SPECTRUM BY DECILE

folder <- dirname(rstudioapi::getSourceEditorContext()$path)

#load data
data <- read.csv(file.path(folder, '..', 'results', 'decile_market_results_policy_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]
data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

data$combined <- paste(data$country, data$scenario, sep="_")

data1 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_revenue)
data1 <- data1[(data1$strategy == "5G_nsa_microwave_baseline_baseline_baseline_baseline"),]
data1$strategy <- "Revenue"
names(data1)[names(data1) == 'total_market_revenue'] <- 'value'
data2 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_cost)
names(data2)[names(data2) == 'total_market_cost'] <- 'value'
data <- rbind(data1, data2)

data <- data[(
                data$strategy == 'Revenue' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_baseline_baseline' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_low_baseline' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_high_baseline'),]

#select desired columns
data <- select(data, combined, country, scenario, strategy, decile, value)

data <- data[!(data$value == "NA"),]

data$scenario = factor(data$scenario, levels=c("S1_25_10_2",
                                               "S2_200_50_5",
                                               "S3_400_100_10"),
                       labels=c("S1 (25 Mbps)",
                                "S2 (200 Mbps)",
                                "S3 (400 Mbps)"))

data$country = factor(data$country, levels=c("UGA",
                                             'MWI',
                                             "KEN",
                                             "SEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Uganda (C1)",
                               "Malawi (C1)",
                               "Kenya (C2)",
                               "Senegal (C2)",
                               "Pakistan (C3)",
                               "Albania (C4)",
                               "Peru (C5)",
                               "Mexico (C6)"))

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data$strategy = factor(data$strategy, levels=c(
                                                "Revenue",
                                                "5G_nsa_microwave_baseline_baseline_low_baseline",
                                                "5G_nsa_microwave_baseline_baseline_baseline_baseline",
                                               "5G_nsa_microwave_baseline_baseline_high_baseline"),
                       labels=c(
                                "Revenue",
                                "Low",
                                "Baseline",
                                "High"))

data <- data[order(data$combined, data$country, data$scenario, data$strategy, data$decile),]

data <- data %>%
  group_by(combined, country, strategy, scenario) %>%
  mutate(cumulative_value_bn = cumsum(round(value / 1e9, 3)))

panel <- ggplot(data, aes(x=decile, y=cumulative_value_bn, colour=strategy, group=strategy)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme(legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(title = "Impact of Spectrum Prices by Scenario, Decile and Country", colour=NULL,
    subtitle = "Cumulative cost reported by percentage of population covered for 5G NSA (Microwave)",
    x = "Population Covered (%)", y = "Cumulative Cost (Billions $USD)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(fill=guide_legend(ncol =6)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'd_results_spectrum_costs_wrap.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

##################TAX BY DECILE

folder <- dirname(rstudioapi::getSourceEditorContext()$path)

#load data
data <- read.csv(file.path(folder, '..', 'results', 'decile_market_results_policy_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]
data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

data$combined <- paste(data$country, data$scenario, sep="_")

data1 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_revenue)
data1 <- data1[(data1$strategy == "5G_nsa_microwave_baseline_baseline_baseline_baseline"),]
data1$strategy <- "Revenue"
names(data1)[names(data1) == 'total_market_revenue'] <- 'value'
data2 <- select(data, combined, country, scenario, strategy, confidence, decile, total_market_cost)
names(data2)[names(data2) == 'total_market_cost'] <- 'value'
data <- rbind(data1, data2)

remove(data1, data2)

data <- data[(data$strategy == 'Revenue' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_baseline_baseline' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_baseline_low' |
                data$strategy == '5G_nsa_microwave_baseline_baseline_baseline_high'),]

data <- data[!(data$value == "NA"),]

data$scenario = factor(data$scenario, levels=c("S1_25_10_2",
                                               "S2_200_50_5",
                                               "S3_400_100_10"),
                       labels=c("S1 (25 Mbps)",
                                "S2 (200 Mbps)",
                                "S3 (400 Mbps)"))

data$country = factor(data$country, levels=c("UGA",
                                             'MWI',
                                             "KEN",
                                             "SEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Uganda (C1)",
                               "Malawi (C1)",
                               "Kenya (C2)",
                               "Senegal (C2)",
                               "Pakistan (C3)",
                               "Albania (C4)",
                               "Peru (C5)",
                               "Mexico (C6)"))

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data$strategy = factor(data$strategy, levels=c('Revenue',
                                               "5G_nsa_microwave_baseline_baseline_baseline_low",
                                              "5G_nsa_microwave_baseline_baseline_baseline_baseline",
                                               "5G_nsa_microwave_baseline_baseline_baseline_high"),
                                 labels=c('Revenue',
                                          "Low (10% Tax Rate)",
                                          "Baseline (30% Tax Rate)",
                                          "High (40% Tax Rate)"))

data <- data[order(data$combined, data$country, data$scenario, data$strategy, data$decile),]

data <- data %>%
  group_by(combined, country, strategy, scenario) %>%
  mutate(cumulative_value_bn = cumsum(round(value / 1e9, 3)))

panel <- ggplot(data, aes(x=decile, y=cumulative_value_bn, colour=strategy, group=strategy)) +
  geom_line() +
  scale_fill_brewer(palette="Spectral", name = expression('Cost Type'), direction=1) +
  theme(legend.position = "bottom") + #axis.text.x = element_text(angle = 45, hjust = 1),
  labs(title = "Impact of Tax Rate by Scenario, Decile and Country", colour=NULL,
       subtitle = "Cumulative cost reported by percentage of population covered for 5G NSA (Microwave)",
       x = "Population Covered (%)", y = "Cumulative Cost (Billions $USD)") +
  scale_x_continuous(expand = c(0, 0), breaks = seq(0,100,20)) +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(fill=guide_legend(ncol =6)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'e_results_tax_rate_wrap.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

###################NATIONAL COST PROFILE FOR BASELINE
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

#load data
data <- read.csv(file.path(folder, '..', 'results', 'national_market_cost_results_technology_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]
data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

data$combined <- paste(data$country, data$scenario, sep="_")

data <- select(data, strategy, combined, total_ran, total_backhaul_fronthaul,
               total_civils, total_core_network, total_administration,
               total_spectrum_cost, total_tax, total_profit_margin,
               total_used_cross_subsidy, total_required_state_subsidy)

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data$strategy = factor(data$strategy, levels=c(
  "4G_epc_microwave_baseline_baseline_baseline_baseline",
  "4G_epc_fiber_baseline_baseline_baseline_baseline",
  "5G_nsa_microwave_baseline_baseline_baseline_baseline",
  "5G_sa_fiber_baseline_baseline_baseline_baseline"),
  labels=c("4G (MW)",
           "4G (FB)",
           "5G NSA (MW)",
           "5G SA (FB)"))

data <- gather(data, metric, value, total_ran:total_required_state_subsidy)

data$metric = factor(data$metric, levels=c("total_required_state_subsidy",
                                           "total_used_cross_subsidy",
                                           "total_profit_margin",
                                           "total_tax",
                                           "total_spectrum_cost",
                                           "total_administration",
                                           "total_ran",
                                           'total_backhaul_fronthaul',
                                           'total_civils',
                                           'total_core_network'
),
labels=c("Required Subsidy",
         "Cross-Subsidy",
         "Profit",
         "Tax",
         "Spectrum",
         "Admin",
         "RAN",
         "FH/BH",
         "Sites",
         'Core/Regional Fiber'
))

panel <- ggplot(data, aes(x=strategy, y=(value/1e9), group=metric, fill=metric)) +
  geom_bar(stat = "identity") +
  coord_flip() +
  scale_fill_brewer(palette="Spectral", name = NULL, direction=1) +
  theme(legend.position = "bottom") +
  labs(title = "Aggregate Cost Composition by Scenario, Strategy and Country", colour=NULL,
       subtitle = "Illustrates required state subsidy under baseline conditions",
       x = NULL, y = "Total Cost (Billions $USD)") +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(fill=guide_legend(ncol =10, reverse = TRUE)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'f_cost_composition_baseline_national.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

##############NATIONAL COST PROFILE FOR MIXED OPTIONS
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

data <- read.csv(file.path(folder, '..', 'results', 'national_market_cost_results_mixed_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]
data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

# data$confidence = factor(data$confidence, levels=c('5','50', '95'),
#                             labels=c("lower", 'mean', "upper"))

data$combined <- paste(data$country, data$scenario, sep="_")

data <- select(data, strategy, combined, total_ran, total_backhaul_fronthaul,
               total_civils, total_core_network, total_administration,
               total_spectrum_cost, total_tax, total_profit_margin,
               total_used_cross_subsidy, total_required_state_subsidy)

required_subsidy <- select(data, combined, strategy, total_required_state_subsidy)
required_subsidy <- spread(required_subsidy, strategy, total_required_state_subsidy)
path = file.path(folder, '..','results', 'required_subsidy.csv')
write.csv(required_subsidy, path)

data <- gather(data, metric, value, total_ran:total_required_state_subsidy)
#
# lower_data <- data[(data$confidence == 'lower'),]
# mean_data <- data[(data$confidence == 'mean'),]
# upper_data <- data[(data$confidence == 'upper'),]
#
# names(lower_data)[names(lower_data) == 'value'] <- 'lower'
# names(mean_data)[names(upper_data) == 'value'] <- 'mean'
# names(upper_data)[names(upper_data) == 'value'] <- 'upper'
#
# lower_data <- select(lower_data, strategy, combined, metric, lower)
# mean_data <- select(mean_data, strategy, combined, metric, mean)
# upper_data <- select(upper_data, strategy, combined, metric, upper)
#
# data <- merge(mean_data, lower_data, by=c('strategy', 'combined', 'metric'))
# data <- merge(data, upper_data, by=c('strategy', 'combined', 'metric'))
#
# rm(lower_data, mean_data, upper_data)

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data$metric = factor(data$metric, levels=c("total_required_state_subsidy",
                                           "total_used_cross_subsidy",
                                           "total_profit_margin",
                                           "total_tax",
                                           "total_spectrum_cost",
                                           "total_administration",
                                           "total_ran",
                                           'total_backhaul_fronthaul',
                                           'total_civils',
                                           'total_core_network'
),
labels=c("Required Subsidy",
         "Cross-Subsidy",
         "Profit",
         "Tax",
         "Spectrum",
         "Admin",
         "RAN",
         "FH/BH",
         "Sites",
         'Core/Regional Fiber'
))

data$strategy = factor(data$strategy, levels=c(
  "4G_epc_microwave_srn_baseline_low_low",
  "4G_epc_fiber_srn_baseline_low_low",
  "5G_nsa_microwave_srn_baseline_low_low",
  "5G_sa_fiber_srn_baseline_low_low"),
  labels=c("4G (MW)",
           "4G (FB)",
           "5G NSA (MW)",
           "5G SA (FB)"))

panel <- ggplot(data, aes(x=strategy, y=(value/1e9), group=metric, fill=metric)) +
  geom_bar(stat = "identity") +
  coord_flip() +
  scale_fill_brewer(palette="Spectral", name = NULL, direction=1) +
  theme(legend.position = "bottom") +
  labs(title = "Aggregate Cost Composition by Scenario, Strategy and Country", colour=NULL,
       subtitle = "Illustrates required state subsidy after exhausting all other policy options",
       x = NULL, y = "Total Cost (Billions $USD)") +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(fill=guide_legend(ncol =10, reverse = TRUE)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'g_cost_composition_mixed_national.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

##############DECILE COST PROFILE FOR BASELINE OPTIONS

folder <- dirname(rstudioapi::getSourceEditorContext()$path)

#load data
data <- read.csv(file.path(folder, '..', 'results', 'decile_market_cost_results_mixed_options.csv'))

data <- data[!(data$total_market_cost == "NA"),]
data <- data[(data$confidence == 50),]

names(data)[names(data) == 'GID_0'] <- 'country'

data <- data[(data$strategy == '5G_nsa_microwave_srn_baseline_low_low'),]

data$combined <- paste(data$country, data$scenario, sep="_")

data <- select(data, combined, decile, total_ran, total_backhaul_fronthaul,
               total_civils, total_core_network, total_administration,
               total_spectrum_cost, total_tax, total_profit_margin,
               total_used_cross_subsidy, total_required_state_subsidy)

data <- gather(data, metric, value, total_ran:total_required_state_subsidy)

data$combined = factor(data$combined, levels=c('MWI_S1_25_10_2',
                                               'MWI_S2_200_50_5',
                                               'MWI_S3_400_100_10',
                                               "UGA_S1_25_10_2",
                                               "UGA_S2_200_50_5",
                                               "UGA_S3_400_100_10",
                                               "SEN_S1_25_10_2",
                                               "SEN_S2_200_50_5",
                                               "SEN_S3_400_100_10",
                                               "KEN_S1_25_10_2",
                                               "KEN_S2_200_50_5",
                                               "KEN_S3_400_100_10",
                                               "PAK_S1_25_10_2",
                                               "PAK_S2_200_50_5",
                                               "PAK_S3_400_100_10",
                                               "ALB_S1_25_10_2",
                                               "ALB_S2_200_50_5",
                                               "ALB_S3_400_100_10",
                                               "PER_S1_25_10_2",
                                               "PER_S2_200_50_5",
                                               "PER_S3_400_100_10",
                                               "MEX_S1_25_10_2",
                                               "MEX_S2_200_50_5",
                                               "MEX_S3_400_100_10"),
                       labels=c("Malawi (C1) (S1: 25 Mbps)", "Malawi (C1) (S2: 200 Mbps)", "Malawi (C1) (S3: 400 Mbps)",
                                "Uganda (C1) (S1: 25 Mbps)", "Uganda (C1) (S2: 200 Mbps)", "Uganda (C1) (S3: 400 Mbps)",
                                "Senegal (C2) (S1: 25 Mbps)", "Senegal (C2) (S2: 200 Mbps)", "Senegal (C2) (S3: 400 Mbps)",
                                "Kenya (C2) (S1: 25 Mbps)", "Kenya (C2) (S2: 200 Mbps)", "Kenya (C2) (S3: 400 Mbps)",
                                "Pakistan (C3) (S1: 25 Mbps)", "Pakistan (C3) (S2: 200 Mbps)", "Pakistan (C3) (S3: 400 Mbps)",
                                "Albania (C4) (S1: 25 Mbps)", "Albania (C4) (S2: 200 Mbps)", "Albania (C4) (S3: 400 Mbps)",
                                "Peru (C5) (S1: 25 Mbps)", "Peru (C5) (S2: 200 Mbps)", "Peru (C5) (S3: 400 Mbps)",
                                "Mexico (C6) (S1: 25 Mbps)", "Mexico (C6) (S2: 200 Mbps)", "Mexico (C6) (S3: 400 Mbps)" ))

data$decile = factor(data$decile, levels=c(0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100))

data$metric = factor(data$metric, levels=c("total_required_state_subsidy",
                                           "total_used_cross_subsidy",
                                           "total_profit_margin",
                                           "total_tax",
                                           "total_spectrum_cost",
                                           "total_administration",
                                           "total_ran",
                                           'total_backhaul_fronthaul',
                                           'total_civils',
                                           'total_core_network'
),
labels=c("Required Subsidy",
         "Cross-Subsidy",
         "Profit",
         "Tax",
         "Spectrum",
         "Admin",
         "RAN",
         "FH/BH",
         "Sites",
         'Core/Regional Fiber'
))

panel <- ggplot(data, aes(x=decile, y=(value/1e9), group=metric, fill=metric)) +
  geom_bar(stat = "identity") +
  scale_fill_brewer(palette="Spectral", name = NULL, direction=1) +
  theme(legend.position = "bottom") +
  labs(title = "Cost Composition by Scenario, Decile and Country", colour=NULL,
       subtitle = "Cumulative cost reported by percentage of population covered for 5G NSA (Microwave)",
       x = "Population Covered (%)", y = "Cumulative Cost (Billions $USD)") +
  scale_y_continuous(expand = c(0, 0)) +  theme(panel.spacing = unit(0.6, "lines")) +
  guides(fill=guide_legend(ncol =10, reverse = TRUE)) +
  facet_wrap(~combined, scales = "free", ncol=3)

path = file.path(folder, 'figures', 'h_cost_composition_mixed_decile.png')
ggsave(path, units="in", width=10, height=14.5, dpi=300)
print(panel)
dev.off()

####################################TOTAL COST AS % OF GDP
#get folder directory
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

results <- read.csv(file.path(folder, '..', 'results', 'national_market_results_technology_options.csv'))
names(results)[names(results) == 'GID_0'] <- 'iso3'

gdp <- read.csv(file.path(folder, 'gdp.csv'))
names(gdp)[names(gdp) == 'iso3'] <- 'iso3'

results <- merge(results, gdp, by='iso3', all=FALSE)

results$gdp_percentage <- (results$total_market_cost / 5) / results$gdp * 100

results$combined <- paste(results$iso3, results$scenario, sep="_")

results$scenario = factor(results$scenario, levels=c("S1_25_10_2",
                                                     "S2_200_50_5",
                                                     "S3_400_100_10"),
                          labels=c("S1 (25 Mbps)",
                                   "S2 (200 Mbps)",
                                   "S3 (400 Mbps)"))

results$country = factor(results$iso3, levels=c('MWI',
                                             "UGA",
                                             "SEN",
                                             "KEN",
                                             "PAK",
                                             "ALB",
                                             "PER",
                                             "MEX"),
                      labels=c("Malawi","Uganda",
                               "Senegal","Kenya",
                               "Pakistan",
                               "Albania",
                               "Peru",
                               "Mexico"))

results$strategy = factor(results$strategy, levels=c("Revenue",
                                                     "4G_epc_microwave_baseline_baseline_baseline_baseline",
                                                     "4G_epc_fiber_baseline_baseline_baseline_baseline",
                                                     "5G_nsa_microwave_baseline_baseline_baseline_baseline",
                                                     "5G_sa_fiber_baseline_baseline_baseline_baseline"),
                          labels=c("Revenue",
                                   "4G (Microwave)",
                                   "4G (Fiber)",
                                   "5G NSA (Microwave)",
                                   "5G SA (Fiber)"))

results <- select(results, country, scenario, strategy, confidence, gdp_percentage)
wide <- results %>%
             group_by(country, scenario, strategy) %>%
             summarize(lower = min(gdp_percentage),
                       mean = mean(gdp_percentage),
                       upper = max(gdp_percentage))

# results$confidence = factor(results$confidence, levels=c('5','50', '95'),
#                             labels=c("lower", 'mean', "upper"))
#
# wide <- select(results, country, scenario, strategy, confidence, gdp_percentage)
# wide <- spread(wide, confidence, gdp_percentage)

gdp_percentage_ci <- ggplot(wide, aes(x=strategy, y=mean, fill=strategy)) +
  coord_flip() +
  geom_bar(stat="identity") +
  geom_errorbar(aes(ymin = lower, ymax = upper),width = 0.25) +
  theme( legend.position = NULL) +
  labs(colour=NULL,
       title = "Total Investment Cost as a Percentage of GDP Over 5 Years",
       subtitle = "Results reported by scenario, strategy and country for 95% confidence",
       x = NULL, y = "Percent of Annual GDP Over 5 Years") +
  theme(panel.spacing = unit(0.6, "lines")) + expand_limits(y=0) +
  guides(fill=FALSE) +
  facet_grid(country~scenario)

path = file.path(folder, 'figures', 'i_country_costs_by_gpd.png')
ggsave(path, units="in", width=8, height=10)
print(gdp_percentage_ci)
dev.off()

##################CLUSTER COSTS

#get folder directory
folder <- dirname(rstudioapi::getSourceEditorContext()$path)

results <- read.csv(file.path(folder, '..', 'results', 'national_market_results_technology_options.csv'))
names(results)[names(results) == 'GID_0'] <- 'iso3'

clusters <- read.csv(file.path(folder, 'clustering', 'results', 'data_clustering_results.csv'))
names(clusters)[names(clusters) == 'ISO_3digit'] <- 'iso3'
clusters <- select(clusters, iso3, cluster, country)

results <- merge(results, clusters, x.by='iso3', y.by='iso3', all=FALSE)

mean_user_cost_by_cluster <- select(results, scenario, strategy, confidence, cost_per_network_user, cluster)

mean_user_cost_by_cluster <- mean_user_cost_by_cluster %>%
  group_by(scenario, strategy, confidence, cluster) %>%
  summarise(mean_cost_per_user = mean(cost_per_network_user))

results <- merge(clusters, mean_user_cost_by_cluster, x.by='cluster', y.by='cluster', all=TRUE)

gdp <- read.csv(file.path(folder, 'gdp.csv'))
names(gdp)[names(gdp) == 'iso3'] <- 'iso3'
gdp <- select(gdp, iso3, gdp)

results <- merge(results, gdp, by='iso3', all=FALSE)

pop <- read.csv(file.path(folder, 'population_2018.csv'))
pop <- select(pop, iso3, population)
pop$iso3 <- as.character(pop$iso3)

results <- merge(results, pop, by='iso3', all=FALSE)

#Not sure if this is right
#Current costs are for a user on a network with 25% market share
#We then multiply the user cost across the whole population?
#check and revise this
results$total_market_cost <- results$mean_cost_per_user * results$population

results$gdp_percentage <- (results$total_market_cost / 5) / results$gdp * 100

results$confidence = factor(results$confidence, levels=c('5','50', '95'),
                            labels=c("lower", 'mean', "upper"))

results$scenario = factor(results$scenario, levels=c("S1_25_10_2",
                                                     "S2_200_50_5",
                                                     "S3_400_100_10"),
                          labels=c("S1 (25 Mbps)",
                                   "S2 (200 Mbps)",
                                   "S3 (400 Mbps)"))

results$strategy = factor(results$strategy, levels=c(
  "4G_epc_microwave_baseline_baseline_baseline_baseline",
  "4G_epc_fiber_baseline_baseline_baseline_baseline",
  "5G_nsa_microwave_baseline_baseline_baseline_baseline",
  "5G_sa_fiber_baseline_baseline_baseline_baseline"),
  labels=c(
    "4G (MW)",
    "4G (FB)",
    "5G NSA (MW)",
    "5G SA (FB)"))

results <- results[complete.cases(results), ]

raw_num <- select(results, country, cluster, scenario, strategy, confidence, total_market_cost)
raw_num <- spread(raw_num, confidence, total_market_cost)

cost_by_strategy <- ggplot(raw_num, aes(x=raw_num$cluster, y=raw_num$mean/1e9, colour=raw_num$cluster)) +
  geom_boxplot(aes(group=factor(raw_num$cluster))) +
  # geom_jitter(width = 0.4, height=0.5, size=1.7) +
  scale_colour_manual(values = c("#F0E442","#E69F00","#D55E00", "#0072B2", "#56B4E9","#009E73")) +
  theme(legend.position = NULL) +
  labs(colour=NULL,
       title = "Total Investment Cost",
       subtitle = "Boxplot of results by scenario, strategy and cluster",
       x = NULL, y = "Total Investment ($USD Billions)") +
  theme(panel.spacing = unit(0.6, "lines")) +
  expand_limits(x = 0, y = 0) +
  scale_y_continuous(limits=c(0,40), expand=c(0,0)) +
  guides(fill=FALSE, colour=FALSE) +
  facet_grid(strategy~scenario)

gdp_num <- select(results, country, cluster, scenario, strategy, confidence, gdp_percentage)
gdp_num <- spread(gdp_num, confidence, gdp_percentage)

gdp_mean_summary_stats <- gdp_num %>%
  group_by(cluster, scenario, strategy) %>%
  summarize(Mean = mean(mean))

gdp_perc_by_strategy <- ggplot(gdp_num, aes(x=gdp_num$cluster, y=gdp_num$mean, colour=gdp_num$cluster)) +
  geom_boxplot(aes(group=factor(gdp_num$cluster))) +
  # geom_jitter(width = 0.4, height=0.5, size=1.7) +
  scale_colour_manual(values = c("#F0E442","#E69F00","#D55E00", "#0072B2", "#56B4E9","#009E73")) +
  theme(legend.position = NULL) +
  labs(colour=NULL,
       title = "Annual Investment Cost as GDP Share Over 5 Years",
       subtitle = "Boxplot of results by scenario, strategy and cluster",
       x = NULL, y = "Annual GDP (% over 5 years)") +
  theme(panel.spacing = unit(0.4, "lines")) +
  expand_limits(x = 0, y = 0) +
  scale_y_continuous(limits=c(0,10), expand=c(0,0), breaks=seq(0, 9.7, 2)) +
  guides(fill=FALSE, colour=FALSE) +
  facet_grid(strategy~scenario)

panel <- ggarrange(cost_by_strategy, gdp_perc_by_strategy, ncol = 1, nrow = 2, align = c("hv"))

#export to folder
path = file.path(folder, 'figures', 'j_cluster_costs_by_strategy.png')
ggsave(path, units="cm", width=22, height=33)
print(panel)
dev.off()

######GLOBAL COST

raw_mean_summary_stats <- raw_num %>%
  group_by(cluster, scenario, strategy) %>%
  summarize(Mean = mean(mean)/1e9)

raw_total_summary_stats <- raw_num %>%
  group_by(scenario, strategy) %>%
  summarize(lower = sum(lower)/1e9,
            mean = sum(mean)/1e9,
            upper = sum(upper)/1e9,)

global_cost <- ggplot(raw_total_summary_stats, aes(x=strategy, y=mean/1e3, fill=strategy)) +
  coord_flip() +
  geom_bar(stat="identity") +
  geom_errorbar(aes(ymin = lower/1e3, ymax = upper/1e3),width = 0.25) +
  theme(legend.position = NULL) +
  labs(colour=NULL,
       title = "Global Investment Cost: Low- and Middle-Income Countries (N=173)",
       subtitle = "Results reported by scenario and strategy at 95% confidence",
       x = NULL, y = "Investment Cost ($USD Trillions)") +
  theme(panel.spacing = unit(0.6, "lines")) + expand_limits(y=0) +
  scale_y_continuous(expand = c(0, 0), limits=c(0,9.7), breaks=c(0,2,4,6,8,10)) +
  guides(fill=FALSE) +
  facet_grid(~scenario)

#export to folder
path = file.path(folder, 'figures', 'k_global_costs_by_strategy.png')
ggsave(path, units="cm", width=18, height=8)
print(global_cost)
dev.off()


#!/usr/bin/Rscript

##Author: ZHU O. Yuan
##Script name: Script.r
##Generate 2 QC figures from output files of 01_BAsE-Seq_alignment.sh
##Run as: ./Script.r (from within sample folder)

#A plot of genome-wide coverage plots for the first 9 barcodes
#Check if coverage is even, if there are glaring patterns of uneveness
bc1<-read.table(file="IndividualGenomes/pileup/bc1_sorted.pileup",sep="\t",header=TRUE)
bc2<-read.table(file="IndividualGenomes/pileup/bc2_sorted.pileup",sep="\t",header=TRUE)
bc3<-read.table(file="IndividualGenomes/pileup/bc3_sorted.pileup",sep="\t",header=TRUE)
bc4<-read.table(file="IndividualGenomes/pileup/bc4_sorted.pileup",sep="\t",header=TRUE)
bc5<-read.table(file="IndividualGenomes/pileup/bc5_sorted.pileup",sep="\t",header=TRUE)
bc6<-read.table(file="IndividualGenomes/pileup/bc6_sorted.pileup",sep="\t",header=TRUE)
bc7<-read.table(file="IndividualGenomes/pileup/bc7_sorted.pileup",sep="\t",header=TRUE)
bc8<-read.table(file="IndividualGenomes/pileup/bc8_sorted.pileup",sep="\t",header=TRUE)
bc9<-read.table(file="IndividualGenomes/pileup/bc9_sorted.pileup",sep="\t",header=TRUE)
pdf("IndividualGenomes/First9_pileup_plots.2.pdf")
par(mfrow=c(3,3))
plot(bc1[,2],bc1[,4],xlim=c(0,3300),pch=20,main="bc1",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc2[,2],bc2[,4],xlim=c(0,3300),pch=20,main="bc2",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc3[,2],bc3[,4],xlim=c(0,3300),pch=20,main="bc3",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc4[,2],bc4[,4],xlim=c(0,3300),pch=20,main="bc4",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc5[,2],bc5[,4],xlim=c(0,3300),pch=20,main="bc5",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc6[,2],bc6[,4],xlim=c(0,3300),pch=20,main="bc6",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc7[,2],bc7[,4],xlim=c(0,3300),pch=20,main="bc7",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc8[,2],bc8[,4],xlim=c(0,3300),pch=20,main="bc8",xlab="HBV Genome",ylab="Coverage");abline(h=4)
plot(bc9[,2],bc9[,4],xlim=c(0,3300),pch=20,main="bc9",xlab="HBV Genome",ylab="Coverage");abline(h=4)
dev.off()

#A QC plot with four panels
#Top left: Histogram of read pairs per barcode. X-read pairs per barcode. Y-frequency.
#Top right: Scatter plot of reads per barcode sorted in decreasing order. X-barcode. Y-reads per barcode.
#Bottom left: Histogram of number of bases covered to 4x per barcode. X-number of bases >4x per barcode. Y-frequency.
#Bottom right: Number of genomes with at least n bases covered. X-number of genomes. Y-number of bases covered.
#Use these figures to determine library quality and cutoff for the next step.
pairs<-read.table("IndividualGenomes/ReadPairsPerSam.txt",sep=" ",header=FALSE)
cov<-read.table(file="IndividualGenomes/Cov_per_barcode_4.txt",sep="\t",header=FALSE)
pdf("IndividualGenomes/Barcodes_QC.pdf")
par(mfrow=c(2,2))
hist(as.numeric(pairs[,1]),breaks=100,xlab="Read Pairs per Barcode",main="Hist of Read Pairs per Barcode")
plot(sort(pairs[,1],decreasing=TRUE),xlab="Barcode Sorted by Decreasing Read Pairs",ylab="Read Pairs per Barcode",pch=20,main="Reads per Barcode Sorted Decreasing")
#plot(c(0,cumsum(sort(pairs[,1],decreasing=TRUE))),pch=20,xlab="Barcode Sorted by Decreasing Read Pairs",ylab="Cumulative Read Pairs",main="CDF of Reads per Barcode Sorted Decreasing")
hist(cov[,1],breaks=100,xlab="Bases of Barcode Covered",main="Coverage Distribution")
plot(sort(cov[,1],decreasing=TRUE),xlab="Number of Genomes",ylab="Read Pairs per Barcode Cutoff",pch=20,main="No. Genomes by Cov Cutoff")
dev.off()


library(ggplot2)
library(odbc)
library(DBI)
library(rjson)

rm(list=ls())

con <- DBI::dbConnect(odbc::odbc(), "GLASSv3")

myDir1 <- "/projects/varnf/GLASS-III/GLASS-III/results/kallisto/kallisto/aliquot"
myinf1 <- paste(myDir1,dir(myDir1),sep="/")
myinf1 <- sapply(myinf1,function(x)paste(x,dir(x)[3],sep="/"),USE.NAMES=FALSE)

jsons <- lapply(myinf1,function(x)fromJSON(file=x))
names(jsons) <- dir(myDir1)

qc_table <- do.call(rbind,jsons)

qc_table <- as.data.frame(apply(qc_table,2,unlist))
qc_table[,1:7] <- apply(qc_table[,1:7],2,as.numeric)
range(qc_table[,"p_pseudoaligned"])
mean(qc_table[,"p_pseudoaligned"])
median(qc_table[,"p_pseudoaligned"])

qc_table[which(qc_table[,"p_pseudoaligned"] < 50),4:6]

qc_table <- qc_table[,1:7]
qc_table <- cbind(rownames(qc_table),qc_table)
rownames(qc_table) <- NULL
colnames(qc_table)[1] <- "aliquot_barcode"
dbWriteTable(con, Id(schema="analysis", table="kallisto_qc"), qc_table, overwrite=TRUE)


library("sleuth")
library("files")
library("optparse")

#gene info for sleuth
print("Fetching bioMart info:")
mart <- biomaRt::useMart(host = "metazoa.ensembl.org",
                         biomart = "metazoa_mart",
                         dataset = "celegans_eg_gene")
t2g <- biomaRt::getBM(attributes =  c("ensembl_transcript_id",
                                      "ensembl_gene_id", "external_gene_name",
                                      "description", "transcript_biotype"),
                      mart = mart)
print('#renaming genes:')
t2g <- dplyr::rename(t2g, target_id = ensembl_transcript_id,
                     ens_gene = ensembl_gene_id, ext_gene = external_gene_name)

#point to your directory
print('Loading projects')
base_dir <- '../kallisto'

#get ids
sample_id <- dir(file.path(base_dir))
kal_dirs <- sapply(sample_id, function(id) file.path(base_dir, id))
s2c <- read.table("rna_seq_info.txt", header = TRUE, stringsAsFactors= FALSE)

# rename the 'name' column bc sleuth asks for a 'sample' column
s2c <- dplyr::select(s2c, sample=name, genotype, project, sync_method, person_collected_worms, library_kind, strain,
                     allele1, allele2, date)
s2c <- dplyr::mutate(s2c, path=kal_dirs)
s2c = s2c[(s2c$project != 'female_state'),]
# s2c = s2c[(s2c$project != 'single_worm_benchmarking'),]

print('Fitting model')
so <- sleuth_prep(s2c, ~ strain, target_mapping=t2g)
so <- sleuth_fit(so,~ strain + person_collected_worms + library_kind, fit_name='full')

#Wald test implementations
print("Differential Expression Testing")
for (strain in unique(s2c$strain)){
  if (strain != 'a_n2'){
    beta = paste('strain', strain, sep='')
    so <- sleuth_wt(so, which_beta = beta, which_model='full')
    results_table <- sleuth_results(so, beta, 'full', test_type='wt')

    fname = paste(strain, '.csv', sep='')
    write.csv(results_table, paste('sleuth_strains', fname, sep='/'))
  }
}

# go live on the first model built
sleuth_live(so)