#Arctic Cultural Ecosystem Service Modelling
#code by Claire Runge runge@nceas.ucsb.edu
#Project in collaboration with Vera Hausner Associate Professor, Department of Arctic and Marine Biology, The Arctic University of Tromso vera.hausner@uit.no
#This script takes PPGIS data and generates predictive spatial maps of cultural ecosystem services across Norway

##########################################
#Setup

#options(java.parameters = "-Xmx16g" ) #to give java more memory (16G)
options(java.parameters = "-Xmx2g" ) #to give java more memory (2G)
options(stringsAsFactors=FALSE) # turn off automatic factor coersion
# options(scipen=9999)            # turn off scientific notation

library(dismo) #for maxent interface
library(ENMeval) #for AICc
library(raster)
library(foreign) #to read dbfs
library(rgdal) #to read shapefiles
library(corrplot) #to plot correlation matrix
#library(parallel)

wd <- "C:/Claire/Arctic_Cultural_ES/"
#wd <- "/home/runge/Data/Arctic_Cultural_ES/"
setwd(wd)

rastDir <- paste0(wd, "Spatial data/Processed/3_forMaxent_asc/")
outDir <- paste0(wd, "Maxent runs/")

currDate <- paste(strsplit(as.character(Sys.Date()), "-")[[1]], collapse="") #formatting the date for folder naming

########################
## Setting system preferences for better work with raster
# Define your temp folder
my_tmpdir=paste0(wd, "tmpRaster")

# Create it (handles the case where the folder already exists)
dir.create(my_tmpdir, showWarnings=F)

# Set the raster option to this folder
rasterOptions(tmpdir= my_tmpdir)

## Maximum Memory
# Often server has more RAM than desktop. Therefore the mamximum amount of memory that can be allocated
# to process a raster can be increased. Aurora has a lot of RAM (384GB), so we recommend to use the following
# settings:
rasterOptions(maxmemory =1e+09)
rasterOptions(chunksize=1e+08)

#number of core to use
#ncore=8

###########################
#PRELIMINARY PROCESSING
#load ces observations (PPGIS data)
markersN <- read.csv(paste0(wd, "PPGIS data/Processed/", "PPGIS_Markers_north_UTM33N_alpine.csv"), header=TRUE)
markersS <- read.csv(paste0(wd, "PPGIS data/Processed/", "PPGIS_Markers_south_UTM33N_municipal.csv"), header=TRUE)
markersNshp <- readOGR(paste0(wd, "PPGIS data/Processed/shps"), "PPGIS_Markers_north_UTM33N_alpine")
markersSshp <- readOGR(paste0(wd, "PPGIS data/Processed/shps"), "PPGIS_Markers_south_UTM33N_municipal")

#CES we are interested in
markersN$species[markersN$species=="hunt/fish"] <- "hunt_fish"
markersS$species[markersS$species=="hunt/fish"] <- "hunt_fish"

cultESlist <- c("biological", "cabin", "cleanwater", "cultureident", "gathering", "hunt_fish", "income", "pasture", "recreation","scenic", "social", "specialplace", "spiritual", "therapuetic", "undisturbnature")
#All c("-boats", "-development", "-energy", "-fishing", "-grazing", "-helicopter", "-hunting", "-logging", "-predator", "-roads_atv", "-snowmobiles", "-tourism", "+boats", "+development", "+energy", "+fishing", "+grazing", "+helicopter", "+hunting", "+logging", "+predator", "+roads_atv", "+snowmobiles", "+tourism", "biological", "cabin", "cleanwater", "cultureident", "gathering", "hunt/fish", "income", "otherchange", "pasture", "recnofac", "recreation","scenic", "social", "specialplace", "spiritual", "therapuetic", "undisturbnature")

markersNsub <- subset(markersN, species %in% cultESlist)
markersSsub <- subset(markersS, species %in% cultESlist)

#load environmental data
envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])

#load bias grid
biasGrid <- raster(paste0(basename(rastDir), "/Bias_grids/BiasGrid_distancetoroad_nowater.asc"))

# #############################
# #remove unwanted variables
#included
#exclude distance to road from model with bias grid
#replace "Governance_plus_protectedareas_norway" with "State_commons_norway_binary" & "Protected_areas_norway_forbiological" for biological & undisturbnature
varnames <- c("Norway_alpine", "North_municipalities_alpine", "South_municipalities, AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway","Distance_to_Road_norway", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway", "State_commons_norway_binary",  "Protected_areas_norway_forbiological")
envStack <- envStack[[varnames]]
length(varnames)

#not included
# c("Corrine2012_norway_wetland_1km","Distance_to_waterbodies_norway2", , "State_commons_norway_binary", "Distance_to_Coast_norway2" "Percent_Waterbodies_majorriversandlakesbiggerthan2ha_1_norway",
# "Protected_areas_norway_forbiological.tif")
# dput(list.files(rastDir, "*.asc$"))
# dput(list.files(rastDir, "*.tif$"))

#############################
###CHECK CORRELATION OF VARIABLES
#############################

#check correlation of environmental variables
envStackTIF <- raster::stack(list.files(paste0(wd, "Spatial data/Processed/2b_masked_no_water/"), "*.tif$", full.names=TRUE))
names(envStackTIF) <- sapply(list.files(paste0(wd, "Spatial data/Processed/2b_masked_no_water/"), "*.tif$"), function(x) strsplit(x, "\\.")[[1]][1])
#envStackTIF <- envStackTIF[[varnames]]
cors <- layerStats(envStackTIF, 'pearson', na.rm=TRUE)
saveRDS(cors, file=paste0(outDir, "/Correlation of variables/CorrelationofEnvironmentalVariables_nowater2.rds")) #readRDS to open

#plot correlation matrix
png(filename=paste0(outDir, "/Correlation of variables/CorrelationPlotofEnvironmentalVariables_nowater2.png"), width=1860, height=1440)
corrplot::corrplot(cors[[1]], method='number', type='lower')
dev.off()

#############################
###BACKGROUND
#############################

#set up background points for the combined NS models
	alpineMask <- raster(paste0(dirname(rastDir), "/Templates and boundaries/Norway_alpine_clip.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "Norway_alpine_clip")
	bg <- randomPoints(alpineMask, n=10000, ext=extent(alpineshp), tryf=100)
	write.csv(bg, paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion.csv"), row.names=FALSE)

	#plot the background points
		bgshp <- SpatialPoints(bg, proj4string=crs(markersNshp))
		png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_wholeregion.png"), width=480, height=620)
			plot(alpineshp, border="grey70")
			plot(markersSshp, col="darkslateblue", add=TRUE, pch=20, cex=0.5, alpha=0.5)
			plot(markersNshp, col="darkslateblue", add=TRUE, pch=20, cex=0.5, alpha=0.5)
			plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
		dev.off()

	#set up background points for the combined NS models with bias
	alpineMask <- raster(paste0(dirname(rastDir), "/Bias_grids/BiasGrid_distancetoroad_Norway_alpine.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "Norway_alpine_clip")
	bg <- randomPoints(alpineMask, n=10000, ext=extent(alpineshp), tryf=100, prob=TRUE)
	write.csv(bg, paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion_bias.csv"), row.names=FALSE)

	#plot the background points
		bgshp <- SpatialPoints(bg, proj4string=crs(markersNshp))
		png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_wholeregion_bias.png"), width=480, height=620)
			plot(alpineshp, border="grey70")
			plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
		dev.off()

#set up background points for North models
	Nmask <- raster(paste0(dirname(rastDir), "/Templates and boundaries/North_municipalities_alpine.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "North_municipalities_alpine")
	bgN <- randomPoints(Nmask, n=10000, ext=extent(alpineshp), tryf=100)
	write.csv(bgN, paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"), row.names=FALSE)

	#plot the background points
	bgshp <- SpatialPoints(bgN, proj4string=crs(markersNshp))
	png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_north.png"), width=480, height=620)
		plot(alpineshp, border="grey70")
		plot(markersNshp, col="darkslateblue", add=TRUE, pch=20, cex=0.5, alpha=0.5)
		plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
	dev.off()

	#set up background points for North models with bias
	Nmask <- raster(paste0(dirname(rastDir), "/Bias_grids/BiasGrid_distancetoroad_North_municipalities_alpine.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "North_municipalities_alpine")
	bgN <- randomPoints(Nmask, n=7500, ext=extent(alpineshp), tryf=100, prob=TRUE)
	write.csv(bgN, paste0(dirname(rastDir), "/Background_points/backgroundpoints_north_bias.csv"), row.names=FALSE)

	#plot the background points
	bgshp <- SpatialPoints(bgN, proj4string=crs(markersNshp))
	png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_north_bias.png"), width=480, height=620)
		plot(alpineshp, border="grey70")
		plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
	dev.off()

#set up background points for South models
	Smask <- raster(paste0(dirname(rastDir), "/Templates and boundaries/South_municipalities.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "South_municipalities")
	bgS <- randomPoints(Smask, n=10000, ext=extent(alpineshp), tryf=100)
	write.csv(bgS, paste0(dirname(rastDir), "/Background_points/backgroundpoints_south.csv"), row.names=FALSE)

	#plot the background points
	bgshp <- SpatialPoints(bgS, proj4string=crs(markersNshp))
	png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_south.png"), width=480, height=620)
		plot(alpineshp, border="grey70")
		plot(markersSshp, col="darkslateblue", add=TRUE, pch=20, cex=0.5, alpha=0.5)
		plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
	dev.off()

	#set up background points for South models with bias
	Smask <- raster(paste0(dirname(rastDir), "/Bias_grids/BiasGrid_distancetoroad_South_municipalities.tif"))
	alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "South_municipalities")
	bgS <- randomPoints(Smask, n=10000, ext=extent(alpineshp), tryf=100, prob=TRUE)
	write.csv(bgS, paste0(dirname(rastDir), "/Background_points/backgroundpoints_south_bias.csv"), row.names=FALSE)

		#plot the background points
	bgshp <- SpatialPoints(bgS, proj4string=crs(markersNshp))
	png(paste0(dirname(rastDir), "/Background_points/Map_of_backgroundpoints_south_bias.png"), width=480, height=620)
		plot(alpineshp, border="grey70")
		plot(bgshp, col="darkslategray2", add=TRUE, pch=20, cex=0.5, alpha=0.5)
	dev.off()

#############################
#CHOOSE REGULARIZATION MULTIPLIER
#############################
#Conduct sensitivity analysis to evaluate optimal set of regularization multipliers and evaluate model fit #hinge and linear features only
#occ=two column matrix or data.fram of lon and lat in that order
#Sensitivity analysis of features & regularization of recreation with North dataset

#set up variables
	x=9 #recreation #x=10 #scenic
	currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]), c("lon", "lat")]
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"))
	currEnvStack <- envStack[[varnames[1:(length(varnames)-2)]]]

#run evaluation
	currEval <- ENMeval::ENMevaluate(occ=currOcc, bg.coords=bg, env=currEnvStack,
			RMvalue=seq(0.5, 2.5, 0.5), #regularization parameters
			#fc=c("H", "L"), #hinge & linear
			fc=c("H"), #hinge only
			categoricals=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Governance_plus_protectedareas_norway"),
			n.bg=10000,
			method='randomkfold',
			kfolds=10,
			bin.output=TRUE, #appends evaluations metrics for each evaluation bin to results table
			rasterPreds=FALSE, #if TRUE predict each model across input variables (for AICc)
			#overlap=TRUE, #pairwise metric of niche overlap
			parallel=TRUE, numCores=6)

		#save evaluation
		saveRDS(currEval, file=paste0(outDir, "North model/ENM eval/ENMevalofNmodel_", as.character(cultESlist[x]), ".rds"))
		write.csv(currEval@results, paste0(outDir, "North model/ENM eval/ENMevalofNmodel_", as.character(cultESlist[x]), ".csv"), row.names=FALSE)

#look at it
	data(currEval)
	currEval@results
	which.min(currEval@results$AICc)
	which.max(currEval@results$Mean.AUC)

### Plot prediction AICc
	png(paste0(outDir, "North model/ENM eval/PlotofAICc_forregularizationselection_", as.character(cultESlist[x]), ".png"))
		plot(currEval@predictions[[which (currEval@results$delta.AICc == 0) ]])
		points(currEval@occ.pts, pch=21, bg=currEval@occ.grp)
	dev.off()

### Plot model AUC
#Mean.AUC.DIFF gives difference between test & training AUCs averaged across 10 reps - expected to be high for models overfit to the training data
#full.AUC is AUC of full model (all data, no bins) - evaluation of degree to which a model can successfully discriminate occurrence from background localities
	png(paste0(outDir, "North model/ENM eval/PlotofAUC_forregularizationselection_", as.character(cultESlist[x]), ".png"))
		par(mfrow=c(1,2))
		plot(currEval@results$full.AUC, xaxt='n', xlab="Settings", ylab="AUC", main="Test AUC full model")
		axis(1, at=1:10, labels=currEval@results$settings, las=2)

		plot(currEval@results$Mean.AUC.DIFF, xaxt='n', xlab="Settings", ylab="Difference", main="Test vs Train AUC")
		axis(1, at=1:10, labels=currEval@results$settings, las=2)
	dev.off()

########################
###MAXENT RUNS NORTH MODEL
########################
#TEST MODEL
#Response curves, jackknife & 10-fold cross-validation of N model
#set up variables

	varnames <- c("North_municipalities_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"))

#run model
Nmodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of N model
#Assuming these models all look ok, we now create a model with all the data which we then use for prediction

#run maxent model across all cultES
Nbasemodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN N MODELS biological & undisturbnature values
#set up variables for biological & undisturbnature values
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("North_municipalities_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"))

#run model for biological & undisturbnature values
Nmodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of N model for biological & undisturbnature
#Assuming these models all look ok, we now create a model with all the data which we then use for prediction

#run maxent model across all cultES
Nbasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###MAXENT RUNS NORTH MODEL WITH BIAS GRID
########################

###SET UP N DATA WITH BIAS GRID
#set up variables
	varnames <- c("North_municipalities_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway",  "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"))

#TEST N MODEL WITH BIAS GRID
#Response curves, jackknife & 10-fold cross-validation of N model with bias grid
Nmodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Response curves and jacknife bias grid ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of N model WITH BIAS GRID
#Assuming these models all look ok, we now create a model with all the data which we then use for prediction

#run maxent model across all cultES
Nbasemodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Base_run_bias_grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

#SET UP DATA N model WITH BIAS GRID for biological & undisturbnature values
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("North_municipalities_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_north.csv"))

##TEST N MODEL WITH BIAS GRID for biological & undisturbnature values
Nmodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Response curves and jacknife bias grid ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of N model WITH BIAS GRID for biological & undisturbnature
#Assuming these models all look ok, we now create a model with all the data which we then use for prediction
#run maxent model across biological & undisturbnature
Nbasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "North model/Base_run_bias_grid_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###MAXENT RUNS SOUTH MODEL
########################

#set up variables
	varnames <- c("South_municipalities", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_south.csv"))

###TEST RUN S MODEL
#Response curves, jackknife & 10-fold cross-validation of S model
Smodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of S model
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Sbasemodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return()
			})
			#run maxent model across all cultES

###SET UP DATA S MODEL for biological & undisturbnature values
#set up variables for biological & undisturbnature values
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("South_municipalities", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_south.csv"))

###TEST RUN S MODEL for biological & undisturbnature values
#Response curves, jackknife & 10-fold cross-validation of S model
Smodel <- lapply(1:length(cultESlist), function(x) {
			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "State_commons_norway_binary",  "Protected_areas_norway_forbiological"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of S model for biological & undisturbnature
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Sbasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###MAXENT RUNS SOUTH MODEL WITH BIAS GRID
########################
###SET UP DATA
#Response curves, jackknife & 10-fold cross-validation of S model with bias grid
#set up variables
	varnames <- c("South_municipalities", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway",  "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_south.csv"))

###TEST MODEL WITH BIAS GRID
#run model with k-fold cross validation & jackknife
Smodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Response curves and jacknife bias grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of S model WITH BIAS GRID
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Sbasemodel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Base_run_bias_grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return()
			})
			#run maxent model across all cultES

###SET UP DATA FOR S MODEL biological & undisturbnature values WITH BIAS GRID
#set up variables for biological & undisturbnature values
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("South_municipalities", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway",  "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_south.csv"))

###TEST MODEL FOR S MODEL biological & undisturbnature values WITH BIAS GRID
#run model for biological & undisturbnature values
Smodel <- lapply(1:length(cultESlist), function(x) {
			currOcc <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Response curves and jacknife bias grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "State_commons_norway_binary",  "Protected_areas_norway_forbiological"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL WITH BIAS GRID of S model for biological & undisturbnature
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Sbasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "South model/Base_run_bias_grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###MAXENT RUNS NORTHSOUTH COMBINED MODEL
########################
###SET UP DATA for NORTHSOUTH COMBINED MODEL
	varnames <- c("Norway_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	markersCombined <- rbind(markersNsub, markersSsub)
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion.csv"))

###TEST MODEL
#Response curves, jackknife & 10-fold cross-validation of model created with both north and south data
ComboModel <- lapply(2:length(cultESlist-1), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of model created with both north and south data
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
CombobaseModel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return()
			})

###SET UP DATA FOR biological & undisturbnature values NORTHSOUTH COMBINED MODEL
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("Norway_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway", "Distance_to_Road_norway", "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion.csv"))

###TEST MODEL FOR biological & undisturbnature values NORTHSOUTH COMBINED MODEL
#run model for biological & undisturbnature values
Combomodel <- lapply(1:length(cultESlist), function(x) {
			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Response curves and jacknife ", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "State_commons_norway_binary",  "Protected_areas_norway_forbiological"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of Combo model for biological & undisturbnature NORTHSOUTH COMBINED MODEL
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Combobasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Base_run_", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###MAXENT RUNS NORTHSOUTH COMBINED MODEL WITH BIAS GRID
########################
###SET UP DATA WITH BIAS GRID
	varnames <- c("Norway_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway",  "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	markersCombined <- rbind(markersNsub, markersSsub)
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion.csv"))

###TEST NORTHSOUTH COMBINED MODEL WITH BIAS GRID
#Response curves, jackknife & 10-fold cross-validation of model created with both north and south data and bias grid
ComboModel <- lapply(2:length(cultESlist-1), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Response curves and jacknife bias grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE NORTHSOUTH COMBINED MODEL WITH BIAS GRID
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
CombobaseModel <- lapply(2:(length(cultESlist)-1), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Base_run_bias_grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Governance_plus_protectedareas_norway"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return()
			})

###SET UP DATA FOR biological & undisturbnature values NORTHSOUTH COMBINED MODEL WITH BIAS GRID
	cultESlist <- c("biological", "undisturbnature")
	varnames <- c("Norway_alpine", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway",  "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Percent_industrial_1km_norway_nowater","State_commons_norway_binary",  "Protected_areas_norway_forbiological")
	envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
	names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
	envStack <- envStack[[varnames]]
	names(envStack)[[1]] <- "mask"
	bg <- read.csv(paste0(dirname(rastDir), "/Background_points/backgroundpoints_wholeregion.csv"))

###TEST MODEL FOR biological & undisturbnature values NORTHSOUTH COMBINED MODEL WITH BIAS GRID
#run model for biological & undisturbnature values
Combomodel <- lapply(1:length(cultESlist), function(x) {
			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Response curves and jacknife bias grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c("AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "State_commons_norway_binary",  "Protected_areas_norway_forbiological"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE',  'responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false', 'outputfiletype=asc', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

###RUN BASE MODEL of Combo model for biological & undisturbnature NORTHSOUTH COMBINED MODEL WITH BIAS GRID
#Assuming these models all look ok, we now create a model with all the data which we later use for prediction
Combobasemodel <- lapply(1:length(cultESlist), function(x) {

			currOcc <- markersCombined[markersCombined$species==as.character(cultESlist[x]),c("lon", "lat")]
			currOutPath <- paste0(outDir, "Combined model/Base_run_bias_grid", currDate, "/", as.character(cultESlist[x]))
			dir.create(currOutPath, recursive=TRUE)

			#run the model
			currMod <- dismo::maxent(envStack, currOcc, a=bg, factors=c( "AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water","Protected_areas_norway_forbiological", "State_commons_norway_binary"), args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE', 'responsecurves=TRUE', 'jackknife=TRUE', 'randomtestpoints=25', 'beta_hinge=1', 'biasfile=biasGrid'), path=currOutPath)

			#save model
			#names(currMod) <- cultESlist[x]
			saveRDS(currMod, file=paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
			return(currMod)
			})

########################
###PREDICT MODELS ONTO ENVIRONMENTAL VARIABLES
########################
#Predict models
i <- 1 #1:12, for different folders in folderlist

folderlist <- paste0(rep(c("Combined model", "North model", "South model"), each=4), paste0(rep(c("/Base run ", "/Base run bias grid "), each=2), c("20170116/", "20170117/")))

alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "Norway_alpine")

rastDir <- paste0(wd, "Spatial data/Processed/3_forMaxent_asc/")
#rastDir <- paste0(wd, "Spatial data/Processed/4_forMaxent_prediction/")
varnames <- c("Norway_alpine", "North_municipalities_alpine", "South_municipalities","AnyWaterbodies_majorriversandlakesbiggerthan2ha_1km_norway_no_water", "Corrine2012_norway_broadleafforest_1km", "Corrine2012_norway_coniferforest_1km", "Corrine2012_norway_cropland_1km", "Corrine2012_norway_heathshrub_1km", "Corrine2012_norway_sparselyvegetated_1km", "Distance_to_Town2_norway","Distance_to_Road_CR_no_water", "Percent_industrial_1km_norway_nowater", "Governance_plus_protectedareas_norway", "State_commons_norway_binary",  "Protected_areas_norway_forbiological") # dput(list.files(rastDir, "*.asc$"))

envStack <- raster::stack(list.files(rastDir, "*.asc$", full.names=TRUE))
names(envStack) <- sapply(list.files(rastDir, "*.asc$"), function(x) strsplit(x, "\\.")[[1]][1])
envStack <- envStack[[varnames]] #remove unwanted variables

#using a for loop because of memory allocation issues
#for (i in folderlist){
#rename the appropriate layer in envStack "mask" to match names of model
# if (i %in% c(1:4))  {
	# names(envStack[[1]]) <- "mask" #combined
	# } else if (i %in% c(5:8)) {
	# names(envStack[[2]]) <- "mask" #north
	# } else if (i %in% c(9:12)) {
	# names(envStack[[3]]) <- "mask" #south
# }

names(envStack)[[1]] <- "mask" #predict onto all study region

###PREDICT across all models
	for (x in 1:length(cultESlist)){

		currOutPath <- paste0(outDir, folderlist[i], as.character(cultESlist[x]))
		currMod <- readRDS(paste0(currOutPath, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))
		print(paste0("starting prediction ", folderlist[i], cultESlist[x]))
		print(Sys.time())
		#make predictive maps
		currMap <- dismo::predict(currMod, envStack, ext=extent(alpineshp), args=c('outputformat=logistic'), progress='text', filename=paste0(currOutPath,"/", cultESlist[x], "_prediction_alpine.tif"), format="GTiff", overwrite=TRUE)
		#currMap <- dismo::predict(currMod, envStack, ext=extent(alpineshp), args=c('outputformat=logistic', "doclamp=TRUE", "writeclampgrid=TRUE", "writemess=TRUE"), progress='text', filename=paste0(currOutPath,"/", cultESlist[x], "_basemodel.tif"), format="GTiff")
		print(paste0("finished prediction ", folderlist[i], cultESlist[x]))
		print(Sys.time())
		rm(currMap) #trying to clear memory issues

	}
#}

######################
###COMPARE MODELS
######################
#test N model against S data
bg <- read.csv(paste0(outDir, "backgroundpoints.csv"))[,2:3]
for (x in 1:length(cultESlist)){
	currOutPath <- paste0(outDir, "/North model/Test NS data/", as.character(cultESlist[x]))
	dir.create(currOutPath, recursive=TRUE)
	currInp <- 	paste0(outDir, "/North model/Base run/", as.character(cultESlist[x]))
	currTestData <- markersSsub[markersSsub$species==as.character(cultESlist[x]),c("lon", "lat")]
	currMod <- readRDS(paste0(currInp, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))

	e1 <- evaluate(currMod, p=currTestData, a=bg, x=envStack)
}
#'testsamplesfile=xxx'

######################
#test S model against N data
bg <- read.csv(paste0(outDir, "backgroundpoints.csv"))[,2:3]
for (x in 1:length(cultESlist)){
	currOutPath <- paste0(outDir, "/South model/Test NS data/", as.character(cultESlist[x]))
	dir.create(currOutPath, recursive=TRUE)
	currInp <- 	paste0(outDir, "/South model/Base run/", as.character(cultESlist[x]))
	currTestData <- markersNsub[markersNsub$species==as.character(cultESlist[x]),c("lon", "lat")]
	currMod <- readRDS(paste0(currInp, "/Maxentmodel_rds_", as.character(cultESlist[x]), ".rds"))

	e1 <- evaluate(currMod, p=currTestData, a=bg, x=envStack)
}

#sensitivity to background - use randomseed=TRUE
currArgs <- c('randomseed=TRUE', 'replicates=100', 'outputgrids=false') #check how long each run takes
alpineshp <- readOGR(paste0(wd, "Spatial data/Processed/Templates and boundaries"), "Norway_alpine")

###LAMBDAS
#helpful discussion on extracting lambdas
# https://groups.google.com/forum/#!topic/maxent/iwd8kR_J0tk
#only variables with value that is not zero in the second column were used in the model
#first column: variable and feature type
#second column: lambda value
#third column: min value
#fourth column: max value
#
  # # read lambdas file
  # rf <- read.table(file.path(curpath, 'species.lambdas'), sep=',', fill=TRUE)
  # # get variables used in model - no 0 in 2nd column)
  # rf[!is.na(rf[3]) & rf[2] != 0,]

#maxent(envStack, occurencedata, nbg=number of background points, factors=names of layers considered as categorical, args=arguments passed to maxent, removeDuplicates=FALSE, path=place to store output files)
me <- maxent(predictors, occtrain, factors='biome', path="C:/Claire/Arctic_Cultural_ES/Maxent runs/test", args=c('hinge=TRUE', 'linear=FALSE', 'quadratic=FALSE', 'product=FALSE', 'threshold=FALSE', 'autofeature=FALSE', 'writeplotdata=TRUE'))
pmap <- predict(me, predictors, path="C:/Claire/Arctic_Cultural_ES/Maxent runs/test/test.tif", progress="text", format="GTiff", args=c('outputformat=cumulative', 'perspeciesresults=true', 'threads=3'))

#Check N model 10-fold cross validation, with response curves & jackknife
currArgs <- c('responsecurves=TRUE', 'jackknife=TRUE', 'replicates=10', 'outputgrids=false')

#S model Response curves & jackknife with k-fold cross validation

#compare N-S predictions
#run through the different CultES 'species'
NmodelSdata <- lapply(1:length(cultESlist), function(x) {
		currCultES <- cultESlist[x]
		evaluate(Nmodel[x], p=markersSsub[markersSsub$species==currCultES,], x=envStack)
		})

SmodelNdata <- lapply(1:length(cultESlist), function(x) {
		currCultES <- cultESlist[x]
		evaluate(Smodel[x], p=markersNsub[markersNsub$species==currCultES,], x=envStack)
		})

#fit using S, test using N and vice versa
#http://www.inside-r.org/packages/cran/dismo/docs/maxent
# #test how well the datasets predict each other using the predictions in the eval files	Warren's D similarity statistic
# NScompare <- lapply(1:length(cultESlist), function(x) {
				# currEvalN <- readRDS(paste0(outDir, "North model/ENM eval/ENMevalofNmodel_", as.character(cultESlist[x]), ".rds"))
				# currEvalS <- readRDS(paste0(outDir, "South model/ENM eval/ENMevalofSmodel_", as.character(cultESlist[x]), ".rds"))
				# bestN <- which.min(currEvalN@results$AICc) #pick model with lowest AICc
				# bestS <- which.min(currEvalS@results$AICc) #pick model with lowest AICc
				# NmodelRast <- currEvalN@predictions[[bestN]]
				# SmodelRast <- currEvalS@predictions[[bestS]]
				# return(nicheOverlap(NmodelRast, SmodelRast, mask=FALSE, stat='D'))
			# }

###########################
#remove temp raster folder
unlink(my_tmpdir, recursive = TRUE)
#########################
#END
#########################


######################################################################################################
# CS171 BioVis Project
# Spring 2013
# Author: John Mercer
#create the datasets for PCA
# 1. alpha_and_turn_propensities (118)
# 2. beta_propensities (37)
# 3. composition (24)
# 4. hydrophobicity (149)
# 5. physicochemical_properties (46)
# 6. other_properties (28)
#######################################################################################################

###########################################################
#### first get the 544 pca dataset with all indices #######
###########################################################

biovis_data_dir = "/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/"
file <- "index_values_544_for_pca.csv"
data_path <- paste(biovis_data_dir,file,sep = "")

# this dataframe has the index values as the rows and each column is a different index
main_544_data <- read.table(data_path,header=TRUE,sep = ",")

#specify output directory
biovis_data_dir = "/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/"

# These are the file statements we will use to get the IDs each characteristic
# file <- "alpha_and_turn_propensities_from402.csv"
# file <- "beta_propensities_from402.csv"
# file <- "composition_from402.csv"
# file <- "hydrophobicity_from402.csv"
# file <- "other_properties_from402.csv"
# file <- "physicochemical_from402.csv"

# subset vars and rows help:
# http://www.ats.ucla.edu/stat/r/faq/subset_R.htm

##########################################################
########### alpha_and_turn_propensities ##################
##########################################################
file <- "alpha_and_turn_propensities_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
alpha_and_turn_index_list <- read.table(data_path,header=TRUE,sep = ",")
alpha_and_turn_index_list$ID = gsub("(^ +)|( +$)", "",alpha_and_turn_index_list$ID)
alpha_and_turn_pca_data <- subset(main_544_data, select = alpha_and_turn_index_list$ID)
write.csv(alpha_and_turn_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/alpha_and_turn_pca_data.csv")

##########################################################
###########      beta_propensities      ##################
##########################################################
file <- "beta_propensities_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
beta_index_list <- read.table(data_path,header=TRUE,sep = ",")
beta_index_list$ID = gsub("(^ +)|( +$)", "",beta_index_list$ID)
beta_propensities_pca_data <- subset(main_544_data, select = beta_index_list$ID)
write.csv(beta_propensities_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/beta_propensities_pca_data.csv")

##########################################################
###########        composition          ##################
##########################################################
file <- "composition_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
composition_index_list <- read.table(data_path,header=TRUE,sep = ",")
composition_index_list$ID = gsub("(^ +)|( +$)", "",composition_index_list$ID)
composition_pca_data <- subset(main_544_data, select = composition_index_list$ID)
write.csv(composition_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/composition_pca_data.csv")

##########################################################
###########        hydrophobicity       ##################
##########################################################
file <- "hydrophobicity_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
hydro_index_list <- read.table(data_path,header=TRUE,sep = ",")
hydro_index_list$ID = gsub("(^ +)|( +$)", "",hydro_index_list$ID)
hydro_pca_data <- subset(main_544_data, select = hydro_index_list$ID)
write.csv(hydro_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/hydrophobicity_pca_data.csv")

##########################################################
###########     Other properties        ##################
##########################################################
file <- "other_properties_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
other_index_list <- read.table(data_path,header=TRUE,sep = ",")
other_index_list$ID = gsub("(^ +)|( +$)", "",other_index_list$ID)
other_properties_pca_data <- subset(main_544_data, select = other_index_list$ID)
write.csv(other_properties_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/other_properties_pca_data.csv")

##########################################################
#######     Physicochemical properties     ###############
##########################################################
file <- "physicochemical_from402.csv"
data_path <- paste(biovis_data_dir,file,sep = "")
physico_index_list <- read.table(data_path,header=TRUE,sep = ",")
physico_index_list$ID = gsub("(^ +)|( +$)", "",physico_index_list$ID)
physico_properties_pca_data <- subset(main_544_data, select = physico_index_list$ID)
write.csv(physico_properties_pca_data,file="/Users/mercicle/Documents/python_workspace/BioVis/Codes/IndexData/physico_properties_pca_data.csv")

# example to deal with missings
#http://stackoverflow.com/questions/7731192/replace-mean-or-mode-for-missing-values-in-r
indices_with_missings_pca_data <- subset(main_544_data, select = c("AVBF000101","AVBF000102","AVBF000107","ROSM880105"))

for (var in 1:ncol(indices_with_missings_pca_data)) {
  if (class(indices_with_missings_pca_data[,var])=="numeric") {
    indices_with_missings_pca_data[is.na(indices_with_missings_pca_data[,var]),var] <- mean(indices_with_missings_pca_data[,var], na.rm = TRUE)
  }
}


#' w2v_params
#'
#' Generates the parameters list for use in \code{w2v()}.
#'
#' @param binary
#' (boolean) Save the resulting vectors in binary mode?
#' @param disk
#' (boolean) Stream text from disk during training? Otherwise the text will be loaded memory before training.
#' @param negative
#' (non-negative integer) Number of negative examples. Common values are 3 - 10 (0 = not used).
#' @param iter
#' (positive integer) Number of training iterations.
#' @param window
#' (non-negative integer) Max skip length between words.
#' @param batch_size
#' (positive integer) The batch size used for mini-batch training.
#' @param min_count
#' (non-negative integer) Discard words that appear less than \code{min_count} times.
#' @param hidden_size
#' (positive integer) Size of word vectors.
#' @param min_sync_words
#' (positive integer) Minimal number of words to be synced at each model sync.
#' @param full_sync_times
#' (non-negative integer) Enforced full model sync-up times during training.
#' @param alpha
#' (float) Initial learning rate.
#' @param sample
#' (float) Threshold for word occurrence.
#' @param model_sync_period
#' (float) Synchronize model every \code{model_sync_period} seconds.
#'
#' @return
#' A \code{w2v_params} object, for use in \code{w2v()}.
#'
#' @export
w2v_params = function(
  binary = FALSE,
  disk = FALSE,
  negative = 5,
  iter = 5,
  window = 5,
  batch_size = 11,
  min_count = 5,
  hidden_size = 100,
  min_sync_words = 1024,
  full_sync_times = 0,
  alpha = 0.1,
  sample = 1e-3,
  model_sync_period = 0.1
)
{
  check.is.flag(binary)
  check.is.flag(disk)
  check.is.natnum(negative)
  check.is.posint(iter)
  check.is.natnum(window)
  check.is.posint(batch_size)
  check.is.natnum(min_count)
  check.is.posint(hidden_size)
  check.is.posint(min_sync_words)
  check.is.natnum(full_sync_times)
  check.is.float(alpha)
  check.is.float(sample)
  check.is.float(model_sync_period)

  p = list(
    binary = as.logical(binary),
    disk = as.logical(disk),
    negative = as.integer(negative),
    iter = as.integer(iter),
    window = as.integer(window),
    batch_size = as.integer(batch_size),
    min_count = as.integer(min_count),
    hidden_size = as.integer(hidden_size),
    min_sync_words = as.integer(min_sync_words),
    full_sync_times = as.integer(full_sync_times),
    alpha = as.double(alpha),
    sample = as.double(sample),
    model_sync_period = as.double(model_sync_period)
  )

  class(p) = "w2v_params"
  p
}


pdf_file<-"pdf/barcharts_multiple_all_panel.pdf"
cairo_pdf(bg="grey98", pdf_file, width=13,height=10.5)

par(omi=c(1.25,1.25,1.25,0.25),lheight=1.15,family="Lato Light",las=1)
library(RColorBrewer)

# Import data and prepare chart

source("scripts/inc_labels_za4753.r")
source("scripts/inc_data_za4753.r")
layout(matrix(data=c(1,2,3,4,5),nrow=1,ncol=5), widths=c(2.5,1,1,1,1),heights=c(1,1))

myData1<-cbind(z[,1]+z[,2],z[,3],z[,4],z[,5],z[,6])
tmyData<-myData1
DD_pos<-c(45,45,45,45,35)
myC1<-rgb(0,208,226,maxColorValue=255)
myC2<-rgb(109,221,225,maxColorValue=255)
myC3<-rgb(255,138,238,maxColorValue=255)
myC4<-rgb(255,0,210,maxColorValue=255)
colours<-c("grey",myC1,myC2,myC3,myC4)

# Create chart

for (i in 1:5) {
if (i == 1)
{
par(mai=c(0.25,2.75,0.25,0.15))
bp1<-barplot(tmyData[ ,i],horiz=T,cex.names=1.6,names.arg=mynames,
		xlim=c(0,50),col=colours[i],border=NA,axes=F)
} else
{
par(mai=c(0.25,0.1,0.25,0.15))
bp2<-barplot(tmyData[ ,i],horiz=T,axisnames=F,xlim=c(0,50),col=colours[i],border=NA,axes=F)
}

# Other elements

rect(0,0,10,8.5,col=rgb(191,239,255,80,maxColorValue=255),border=NA)
rect(10,0,20,8.5,col=rgb(191,239,255,120,maxColorValue=255),border=NA)
rect(20,0,30,8.5,col=rgb(191,239,255,80,maxColorValue=255),border=NA)
rect(30,0,40,8.5,col=rgb(191,239,255,120,maxColorValue=255),border=NA)
rect(40,0,50,8.5,col=rgb(191,239,255,80,maxColorValue=255),border=NA)

mtext(myresponses[i],3,adj=0,line=0,cex=0.95,font=3)
mtext(c(10,20,30,40,50),at=c(10,20,30,40,50),1,line=1,cex=0.85)
mtext(0,at=0,1,line=1,cex=0.90,family="Lato Bold")
arrows(0,-0.1,0,8.6,lwd=2.5,length=0,xpd=T,col="skyblue3")
}

# Titling

mtext("It is often said that attitudes towards gender roles are changing",3,line=3.5,adj=1,cex=1.8,family="Lato Black",outer=T)
mtext("N=2,075",1,line=3,adj=0.25,cex=1.1,family="Lato",font=4,outer=T)
mtext("All values in percent",1,line=3,adj=1,cex=1.1,font=3,outer=T)
mtext("Source: European Values Study 2008 Germany, ZA4753. www.gesis.org. Design: Stefan Fichtel, ixtract",1,line=5.5,adj=1.0,cex=0.95,outer=T)
dev.off()


# Main function called for continuous fields
testContinuous <- function(varName, varType, thisdata, varlogfile)
{
    cat("CONTINUOUS MAIN || ", file=varlogfile, append=TRUE)

    pheno <- thisdata[,phenoStartIdx:ncol(thisdata)]
    # reassign values
    pheno <- reassignValue(pheno, varName, varlogfile)
    thisdata[,phenoStartIdx:ncol(thisdata)] <- pheno

    data_to_add <- testContinuous2(varName, varType, thisdata, varlogfile)
    return(data_to_add)
}

# Main code used to process continuous fields, or integer fields that have been reassigned as continuous because they have >20 distinct values.
# This is needed because we have already reassigned values for integer fields, so do this in the function above for continuous fields.
testContinuous2 <- function(varName, varType, thisdata, varlogfile)
{
    cat("CONTINUOUS || ", file=varlogfile, append=TRUE)

    pheno <- thisdata[,phenoStartIdx:ncol(thisdata)]

    if (!is.null(dim(pheno))) {
        phenoAvg <- rowMeans(pheno, na.rm = TRUE)
    } else {
        phenoAvg <- pheno
    }

    # Recode NaN to NA, which is generated if all cols of pheno are NA for a given person
    idxNan <- which(is.nan(phenoAvg))
    phenoAvg[idxNan] <- NA
    numNotNA <- length(na.omit(phenoAvg))

    # Check whether > opt$propforcontinuous examples with same value
    uniqVar <- unique(na.omit(phenoAvg))
    valid <- TRUE
    for (uniq in uniqVar) {
        numWithValue <- length(which(phenoAvg == uniq))
        if (numWithValue / numNotNA >= opt$propforcontinuous) {
            valid <- FALSE
            break
        }
    }

    if (valid == FALSE) {
        # Treat as ordinal categorical
        cat(">", opt$propforcontinuous*100, "% IN ONE CATEGORY || ", file=varlogfile, append=TRUE)

        # If >2 unique values then treat as ordered categorical
        numUniqueValues <- length(uniqVar)

        # Straightforward case that there are two (or one) values
        if (numUniqueValues <= 2) {
            # Treat as binary or skip (binary requires>=10 per category)

            # Remove categories if < 10 examples to see if this should be binary
            # or not, but if ordered categorical then we include all values when
            # generating this.
            phenoAvgMoreThan10 <- testNumExamples(phenoAvg, varlogfile)

            # Binary if 2 distinct values, else ordered categorical
            phenoFactor <- factor(phenoAvgMoreThan10)
            numLevels <- length(unique(na.omit(phenoAvgMoreThan10)))

            if (numLevels<=1) {
                cat("SKIP (number of levels: ", numLevels, ")",
                    sep="", file=varlogfile, append=TRUE)
                incrementCounter("cont.onevalue")
                return(NULL)
            } else if (numLevels == 2) {
            	# Binary so logistic regression
                incrementCounter("cont.binary")
            	thisdatanew <- cbind.data.frame(thisdata[,1:numPreceedingCols], phenoFactor)
            	data_to_add <- binaryLogisticRegression(varName, varType,
                    thisdatanew, varlogfile)
                return(data_to_add)
            }

        } else {
            # Try to treat as ordered categorical
            incrementCounter("cont.ordcattry")
            # Equal sized bins
            phenoBinned <- equalSizedBins(phenoAvg, varlogfile)

            # Check number of people in each bin
            bin0Num <- length(which(phenoBinned == 0))
            bin1Num <- length(which(phenoBinned == 1))
            bin2Num <- length(which(phenoBinned == 2))

            if (bin0Num >= 50 & bin1Num >= 50 & bin2Num >= 50) {
                # Successful binning. >=10 examples in each of the 3 bins
                incrementCounter("cont.ordcattry.ordcat")
                thisdatanew <- cbind.data.frame(thisdata[,1:numPreceedingCols], phenoBinned)
                data_to_add <- testCategoricalOrdered(varName, varType,
                    thisdatanew, varlogfile)
                return(data_to_add)
            } else {
                # Try to treat as binary because not enough examples in each bin
                if (bin0Num < 50 & bin2Num < 50) {
                    # Skip - not possible to create binary variable because first
                    # and third bins are too small ie. could merge bin1 with bin 2
                    # but then bin3 still too small etc...
                    cat("SKIP 2 bins are too small || ", file=varlogfile, append=TRUE)
                    incrementCounter("cont.ordcattry.smallbins")
                    return(NULL)
                } else if ((bin0Num < 50 | bin1Num < 50) &
                          (bin0Num + bin1Num) >= 50) {
                    # Combine first and second bin to create binary variable
                    incrementCounter("cont.ordcattry.binsbinary")
                    cat("Combine first two bins and treat as binary || ",
                        file=varlogfile, append=TRUE)
                    phenoBinned[which(phenoBinned==0)] <- 1
                    # Binary so logistic regression.
                    thisdatanew <- cbind.data.frame(thisdata[,1:numPreceedingCols], phenoBinned)
                    data_to_add <- binaryLogisticRegression(varName, varType,
                        thisdatanew, varlogfile)
                    return(data_to_add)
            	} else if ((bin2Num < 50 | bin1Num < 50) & (bin2Num+bin1Num) >= 50) {
                    # Combine second and last bin to create binary variable
                    incrementCounter("cont.ordcattry.binsbinary")
                    cat("Combine last two bins and treat as binary || ",
                        file=varlogfile, append=TRUE)
                    phenoBinned[which(phenoBinned==2)] <- 1
                    # Binary, so logistic regression.
                    thisdatanew <- cbind.data.frame(thisdata[,1:numPreceedingCols], phenoBinned)
                    data_to_add <- binaryLogisticRegression(varName, varType,
                        thisdatanew, varlogfile)
                    return(data_to_add)
                } else {
                    # Skip - not possible to create binary variable because combining
                    # bins would still be too small.
                    cat("SKIP 2 bins are too small(2) || ", file=varlogfile, append=TRUE)
                    incrementCounter("cont.ordcattry.smallbins2")
                    return(NULL)
                }
            }
        }
    } else {
        cat("IRNT || ", file=varlogfile, append=TRUE)
        incrementCounter("cont.main")

        # Check there are at least 500 examples
        numNotNA <- length(which(!is.na(phenoAvg)))
        if (numNotNA < opt$contnacutoff) {
            cat("CONTINUOUS-SKIP-", opt$contnacutoff, " (", numNotNA, ") || ",sep="",
                file=varlogfile, append=TRUE)
            incrementCounter(paste("cont.main.", opt$contnacutoff, sep=""))
            return(NULL)
        } else {
            # Inverse rank normal transformation
            phenoIRNT <- irnt(phenoAvg)

            # Do regression (use standardised geno values)
            geno <- scale(thisdata[,"geno"])
            confounders <- thisdata[,2:numPreceedingCols]

            # BEGIN TRYCATCH
            tryCatch(
            {
                incrementCounter("success.continuous")
                return(list(phenoIRNT, varName))

                # END TRYCATCH
            }, error = function(e) {
                cat("ERROR:", varName, gsub("[\r\n]", "", e))
                incrementCounter("continuous.error")
            })
        }
    }
}

irnt <- function(pheno) {
    set.seed(1234)
    numPhenos <- length(which(!is.na(pheno)))
    quantilePheno <- (rank(pheno, na.last = "keep", ties.method = "random") - 0.5) / numPhenos
    phenoIRNT <- qnorm(quantilePheno)
    return(phenoIRNT)
}


## two normalisation strategies
## 1) across all samples
## 2) within cell type
## script also performs sample filtering ## to add
args<-commandArgs(trailingOnly = TRUE)
source(args[1])

library(bigmelon)

setwd(dataDir)

gfile<-openfn.gds(gdsFile, readonly = FALSE)

## filter samples
QCSum<-read.csv(paste0(qcOutFolder,"PassQCStatusAllSamples.csv"), stringsAsFactors = FALSE, row.names = 1)

passQC<-rownames(QCSum)[which(QCSum$predLabelledCellType == "TRUE")]
QCmetrics<-read.gdsn(index.gdsn(gfile, "QCdata"))
QCmetrics<-QCmetrics[match(passQC, QCmetrics$Basename),]

normgdsFile<-sub("\\.gds", "Norm.gds", gdsFile)

# create new gfile with only samples that pass QC
if(exists(normgdsFile)){
	file.remove(normgdsFile) ## delete if already exists
}
normfile <- createfn.gds(normgdsFile)
for(node in ls.gdsn(gfile)){
	copyto.gdsn(node = normfile, source = index.gdsn(gfile, node), name = node)
}

## filter out samples that fail QC
## need to match to basename not colnames!!!
rawbetas<-betas(normfile)[,]
subSet(normfile, i=1:length(rownames(normfile)), j=match(passQC, colnames(rawbetas)))

## need to close gds file in order to open in another R session
closefn.gds(gfile)

## repeat detection p value filtering
pfilter(normfile)

## normalise all samples together
dasen(normfile, node="normbeta")

## need to extract below to run normalisation on each cell type
meth<-methylated(normfile)[,]
unmeth<-unmethylated(normfile)[,]
probeType<-fData(normfile)$Type
rawbetas<-betas(normfile)[,]

cellTypes<-unique(QCmetrics$Cell.type)

celltypeNormbeta<-matrix(NA, nrow = nrow(meth), ncol = ncol(meth))
rownames(celltypeNormbeta)<-rownames(rawbetas)
colnames(celltypeNormbeta)<-colnames(rawbetas)
for(each in cellTypes){
	index<-which(QCmetrics$Cell.type == each)
	if(length(index) > 2){
		## perform normalisation within cell type & study
		## compare effect of normalisation within cell type
		celltypeNormbeta[,index]<-dasen(meth[,index], unmeth[,index], probeType)
	}
}

add.gdsn(normfile, 'celltypenormbeta', val = celltypeNormbeta, replace = TRUE)

closefn.gds(normfile)


## BC_BASGRA_MCMC_init.R ##
   cat(file=stderr(), "Starting BC_BASGRA_MCMC_init.r", "\n")

## MCMC chain length (total number of iterations across all chains)
   # https://stats.stackexchange.com/questions/266749/mcmc-convergence
   # suggest nBurnin up to 50% of nChain
   nChains       <- 3 # independent chains for repeatability testing
   nInternal     <- 3 # internal chains in DREAMzs
   nBurnin       <- as.integer(00000 * nChains * nInternal)
   nSampling     <- as.integer(10000 * nChains * nInternal)
   nChain        <- nBurnin + nSampling # total samples

## FILE FOR PRIOR PARAMETER DISTRIBUTION
   # this file provides parameters for all the sites
   # shape <- 0 # shape parameter (0=noninformative, 4=previous method) = now included in par file

   # parameters file has 5 columns
   # name  minimum  mode  maximum  (beta distribution)  unknown
   # require
   # .	DLMXGE > DAYLB
   # .	TOPTGE > TBASE
   # .	FSMAX has a theoretical upper limit < 1.
   # .	HAGERE <= 1
   # .	SHAPE <= 1
   # .	SLAMAX > SLAMIN
   # .	TRANCO may have physical limits [a,b] where a>0 and b<infinity.
   # .	YG < 1 because it is the Growth Yield, the fraction of C allocated to growth that
   #      actually ends up in new biomass, with the remainder being lost to growth respiration.

## LIKELIHOOD FUNCTION ##
   # source("scripts/fLogL_Gauss.R")
   source("scripts/fLogL_Sivia.R")
   # source("scripts/fLogL_Cauchy.R")
   # source("scripts/fLogL_Students.R")
   source("scripts/fLogL_mm_Beta.R")

## SETTINGS FOR THE DIFFERENT CALIBRATION SITES (at least one site)
   sitenames <- c()
   sitenumbers <- c()
   sitesettings_filenames <- c()
   sitedata_filenames     <- c()

   file_prior    <- paste(parameter_location, "/parameters_BC.txt", sep="")

   sitenames <- c(sitenames, "Waikato")
   sitenumbers <- c(sitenumbers, 2)
   sitesettings_filenames <- c(sitesettings_filenames, paste(scenario, "/initialise_BASGRA_Scott_0.r", sep=""))
   sitedata_filenames     <- c(sitedata_filenames, paste(scenario, "/data_calibration_Scott_0.txt", sep=""))

   sitenames <- c(sitenames, "Waikato-10%")
   sitenumbers <- c(sitenumbers, 2)
   sitesettings_filenames <- c(sitesettings_filenames, paste(scenario, "/initialise_BASGRA_Scott-10.r", sep=""))
   sitedata_filenames     <- c(sitedata_filenames, paste(scenario, "/data_calibration_Scott_0.txt", sep=""))

   sitenames <- c(sitenames, "Waikato-20%")
   sitenumbers <- c(sitenumbers, 2)
   sitesettings_filenames <- c(sitesettings_filenames, paste(scenario, "/initialise_BASGRA_Scott-20.r", sep=""))
   sitedata_filenames     <- c(sitedata_filenames, paste(scenario, "/data_calibration_Scott_0.txt", sep=""))

   sitenames <- c(sitenames, "Waikato-30%")
   sitenumbers <- c(sitenumbers, 2)
   sitesettings_filenames <- c(sitesettings_filenames, paste(scenario, "/initialise_BASGRA_Scott-30.r", sep=""))
   sitedata_filenames     <- c(sitedata_filenames, paste(scenario, "/data_calibration_Scott_0.txt", sep=""))

   nSites                 <- length(sitedata_filenames)
   sitelist               <- list() ; length(sitelist) <- nSites

   # Specify data uncertainties (the max of: cv for relative uncertainty, sd for absolute)
   # These are used in BC_BASGRA_MCMC_init_general.r to set the data uncertainites
   # Data uncertainties are now specified in the data file
   # cv_default    <- rep( 0.2 , nSites ) # Simon change from 0.5 to 0.2
   # cv_DM         <- rep( 0.05, nSites ) ; sd_DM_min     <- rep(   0, nSites )
   # cv_LAI        <- rep( 0.1 , nSites ) ; sd_LAI_min    <- rep(   0, nSites )
   # cv_TILTOT     <- rep( 0.0 , nSites ) ; sd_TILTOT_min <- rep( 200, nSites )
   # cv_YIELD      <- rep( 0.05, nSites ) ; sd_YIELD_min  <- rep(   0, nSites )
   # sd_LT50       <- rep( 5   , nSites )
   # sd_CST        <- rep( 5   , nSites ) # need to add this because has values of 0
   # sd_CLV        <- rep( 5  , nSites )
   # sd_WCL        <- rep(  2.5  , nSites )
   # cv_mm_default <- rep( 0.2 , nSites ) # “minimum and maximum” variables
   # cv_mm_FRTILG  <- rep( 0.2 , nSites )

## PROPOSAL TUNING FACTOR
   # fPropTuning   <- 0.05 # This factor is used to modify Gelman"s suggested average step length
   #                       # (2.38^2 / np_BC) which seems too big
   fPropTuning   <- 0.1  # Simon (not used in BT)

## GENERAL INITIALISATION FOR MCMC
   source("scripts/BC_BASGRA_MCMC_init_general.R")

   #
   cat(file=stderr(), "Finished BC_BASGRA_MCMC_init.r", "\n")


#!/usr/bin/env Rscript

##measuring runtime
ptm <- proc.time()

##setup
library(sangerseqR)
mainpath<-'/home/rf/Desktop/crispr mice sequencing/'
setwd(mainpath)
folderlist<-list.files(mainpath)

##read each file
for (fileab1 in folderlist){
write("-----------------", file = "sequences1.txt", sep = "\n", append = TRUE)
write(fileab1, file = "sequences1.txt", sep = "\n", append = TRUE)
data1 <- read.abif(fileab1)
loclist <- data1@data$PLOC.2
Alist = c()
Clist = c()
Glist = c()
Tlist = c()
for (locnum in loclist){
	Glist <- append(Glist, (data1@data$DATA.9[locnum]))
	Alist <- append(Alist, (data1@data$DATA.10[locnum]))
	Tlist <- append(Tlist, (data1@data$DATA.11[locnum]))
	Clist <- append(Clist, (data1@data$DATA.12[locnum]))}
calllist = c()

##the readout for each nt is a 4 digit number
sanger1 <- sangerseq(data1)
refseq <- primarySeq(sanger1, string = TRUE)

##locating a sequence before the cutsite (should be uniform), the actual compared sequence will be 55bp downstream
library(stringr)
site1 <- str_locate(refseq, "GGACCCTATCCT")[2]
site41 <- site1 + 40
site90 <- site1 + 89

##handling long deletions, other errors
if (is.na(site1)) {
	write("notfound", file = "sequences1.txt", sep = "\n", append = TRUE)
	next
	}
if (length(Alist)<(site90+20)) {
	write("outofrange", file = "sequences1.txt", sep = "\n", append = TRUE)
	next}
site41 <- site41+55
site90 <- site90+55

##threshold maybe tuned for your needs
##autothreshold can be ran
maxA = max(Alist[site1:(site90+20)])
maxC = max(Clist[site1:(site90+20)])
maxG = max(Glist[site1:(site90+20)])
maxT = max(Tlist[site1:(site90+20)])
signal1 = c()
for (i in site1:(site90+20)){
	signal1 = rbind(signal1, (c(Alist[i]/maxA, Clist[i]/maxC, Glist[i]/maxG, Tlist[i]/maxT)))
	}
signal2 = sort(signal1)
dropmax = 0
cutnum = 0
for (i in 1:(length(signal2)-1)){
	if (signal2[i]>0.01 & signal2[i]<0.25){
		drop1 = signal2[i+1]/signal2[i]
		if (drop1 > dropmax){
			cutnum = signal2[i+1]
			dropmax = drop1
			}
		}
	}
print(cutnum)
##or manually set
##cutnum=0.05
thresh1 <- c(cutnum*maxA,cutnum*maxC,cutnum*maxG,cutnum*maxT)
print(thresh1)
for (i in 1:length(loclist)){
	call1 <- 0
	if (Alist[i] >= thresh1[1]){
		call1 <- call1 + 1000}
	if (Clist[i] >= thresh1[2]){
		call1 <- call1 + 100}
	if (Glist[i] >= thresh1[3]){
		call1 <- call1 + 10}
	if (Tlist[i] >= thresh1[4]){
		call1 <- call1 + 1}
	calllist <- append(calllist, call1)
	}
diglist <- sprintf("%04d", calllist[site41:site90])
refseqfrag <- unlist(diglist)
seqscore <- str_c(refseqfrag,collapse="")

##enter wildtype sequence around the area of interest here
(wtseq <- "GTCTACAGAGTGAGTTCTGGGAGAGCCAGAGCTACACAGAGAAACCCTGTCTTGGAGTGCGGGGAACAGAACTGTTGCAGTTGTGTTAACAGAGAACCCCTCGTGACGCTTGTTTTCTCAGATAACGGTTACTATCTGCCATACTACAAGAGAGAAAGGAATAAGCGGAGCACGCAGATCACAGTCAGGTTCCTGGACAGCCCCCACTACAGCAAGAACATCCGCAAGAAGGACCCTATCCTCCTGCTGCACTGGTGGAAGGAGATATTCGGGACGATCCTGCTTTGAATCGTGGCCACAACGTTTATCGTGCGCAGGCTTTTCCATCCTCAGCCCCACAGGGTAAGATGCTCTGTCAACCTAATGTGCTTCCAAGTGGTTGCTGTGTAGGAAACCTCTGGGAGGGAGAGTTCCGGCACTCAGACTTACCAGAGAATCATCTGCGCTTGTACTGTCTCCGTCACTTAGCCACAGAGTCACTGTACCTGAACAGCCAGTATGACATAGCCTACTGCTGCCAGGTTGCAGTCTCATACAGTGTGTGACTACAACATAGTATTTAGTATCTGTGTATCTGAAATAGATGAGGCACACTTAAAAAAAAAAAAAAAAAAAACACTAGCCACAATTGTTTTCAACTTTATTGCAATGTTACGGGTCGGCTGGAGAGATGGTTCAGTGGTTAGAGGACCTGAGTTCATTTTGCAGCACCCACATCAGACAACCCTTACCTGCCTGTAACTCCAGCTACTGGAAATCTGACA")
sitewt <- str_locate(wtseq, "GGACCCTATCCT")[2]
sitewt <- sitewt+55

##investigating the -35:+300 range of indels
wtscorelist = c()
for (i in (-35:300)){
	fragname = paste("frag",i,sep="")
	fragseq = str_c(unlist(strsplit(substr(wtseq, sitewt+40+i, sitewt+89+i),split="")),collapse="")
	wtscorelist = rbind(wtscorelist,c(fragname,fragseq))
	for (j in 1:10){
		matchval = 1
		readbase = calllist[site41+j-1]
		if ((substring(fragseq,j,j)=="A") & (readbase%/%1000%%10==1)){
		matchval = 0}
		if ((substring(fragseq,j,j)=="C") & (readbase%/%100%%10==1)){
		matchval = 0}
		if ((substring(fragseq,j,j)=="G") & (readbase%/%10%%10==1)){
		matchval = 0}
		if ((substring(fragseq,j,j)=="T") & (readbase%/%1%%10==1)){
		matchval = 0}
		if (matchval == 1){
		break}
	}
	if (matchval == 0){
	print(fragname)
	write(fragname, file = "sequences1.txt", sep = "\n", append = TRUE)
}}}
proc.time() - ptm


library(rgl)
main <- function(){
    time_series <- read.csv("assumption_checking_data/realTimeData2017_03_17_16_50_31.txt")
    a$FX <- a$JR3.MX
    a$FY <- a$JR3.FX
    a$FZ <- a$JR3.MY
    a$MX <- a$JR3.FZ
    a$MY <- a$JR3.FY
    a$MZ <- a$JR3.MZ
    plot3d(a$FX,a$FY,a$FZ)
}
main()


#' Function to make a map
#'
#' @param data Data to be plotted in the map
#' @param map The map itself
#' @param decimal_mark sign to use for decimal
#' @param big_mark sign to use for numbers over a thousand
#'
#' @return a map
#' @export
#' @importFrom magrittr "%>%"
#'
make_map <- function(data = NULL, map = NULL, decimal_mark = ",", big_mark = " ") {

  # convert from utm33 to leaflet
  map_data <- kart::utm33_to_leaflet(map = map, sf = TRUE)
  map_data <- methods::as(map_data, Class = "Spatial")
   # remove area_name from map, since we will get it from `data`
  map_data@data$area_name <- NULL
  # Only keep columns that we want to use
  simple_data <- data[c("area", "value", "area_name", "type", "numerator_name", "numerator")]
  # Join data with map
  map_data@data <- dplyr::left_join(map_data@data, simple_data, by = c("area_num" = "area"))
  pal <- leaflet::colorBin(palette = SKDEr::skde_colors(num = 5),
                           domain = map_data@data$value,
                           bins = 5,
                           pretty = FALSE)
  output <- map_data %>%
    leaflet::leaflet(options = leaflet::leafletOptions(minZoom = 5, maxZoom = 5)) %>%
    leaflet::addPolygons(weight = 1,
                         color  = "black",
                         fillColor = ~pal(value),
                         fillOpacity = 0.8,
                         label = paste0("",
                                        map_data@data$area_name,
                                        "
",
                                        map_data@data$type,
                                        ": ",
                                        format(map_data@data$value,
                                               big.mark = big_mark, decimal.mark = decimal_mark, digits = 2),
                                        "
",
                                        map_data@data$numerator_name,
                                        ": ",
                                        format(map_data@data$numerator,
                                               big.mark = big_mark, decimal.mark = decimal_mark, digits = 2)
                                        ) %>%
                                 lapply(htmltools::HTML),
                         popupOptions = leaflet::popupOptions(closeButton = TRUE),
                         stroke = TRUE,
                         smoothFactor = 1,
                         highlight = leaflet::highlightOptions(weight = 2,
                                                               color = "white",
                                                               bringToFront = TRUE
                                                               )
                         ) %>%
    leaflet::addLegend(position = "bottomright",
                       pal = pal,
                       values = ~value,
                       title = "Antall per 100 000 innbygger",
                       labFormat = leaflet::labelFormat(digits = 0)
                       ) %>%
    leaflet.extras::setMapWidgetStyle(list(background = "white"))
  return(output)
}


library(ggplot2)

args <- commandArgs(trailingOnly = TRUE)

mutations.by.id.file = args[1]
absent.aa.by.id.file = args[2]
genes = strsplit(args[3], ",")[[1]]
genes = c(genes, "")
outdir = args[4]

print("---------------- read input ----------------")

mutations.by.id = read.table(mutations.by.id.file, sep="\t", fill=T, header=T, quote="")
absent.aa.by.id = read.table(absent.aa.by.id.file, sep="\t", fill=T, header=T, quote="")

for(gene in genes){
	graph.title = paste(gene, "AA mutation frequency")
	if(gene == ""){
		mutations.by.id.gene = mutations.by.id[!grepl("unmatched", mutations.by.id$best_match),]
		absent.aa.by.id.gene = absent.aa.by.id[!grepl("unmatched", absent.aa.by.id$best_match),]

		graph.title = "AA mutation frequency all"
	} else {
		mutations.by.id.gene = mutations.by.id[grepl(paste("^", gene, sep=""), mutations.by.id$best_match),]
		absent.aa.by.id.gene = absent.aa.by.id[grepl(paste("^", gene, sep=""), absent.aa.by.id$best_match),]
	}
	print(paste("nrow", gene, nrow(absent.aa.by.id.gene)))
	if(nrow(mutations.by.id.gene) == 0){
		next
	}

	mutations.at.position = colSums(mutations.by.id.gene[,-c(1,2)])
	aa.at.position = colSums(absent.aa.by.id.gene[,-c(1,2,3,4)])

	dat_freq = mutations.at.position / aa.at.position
	dat_freq[is.na(dat_freq)] = 0
	dat_dt = data.frame(i=1:length(dat_freq), freq=dat_freq)

	print("---------------- plot ----------------")

	m = ggplot(dat_dt, aes(x=i, y=freq)) + theme(axis.text.x = element_text(angle = 90, hjust = 1), text = element_text(size=13, colour="black"))
	m = m + geom_bar(stat="identity", colour = "black", fill = "darkgrey", alpha=0.8) + scale_x_continuous(breaks=dat_dt$i, labels=dat_dt$i)
	m = m + annotate("segment", x = 0.5, y = -0.05, xend=26.5, yend=-0.05, colour="darkgreen", size=1) + annotate("text", x = 13, y = -0.1, label="FR1")
	m = m + annotate("segment", x = 26.5, y = -0.07, xend=38.5, yend=-0.07, colour="darkblue", size=1) + annotate("text", x = 32.5, y = -0.15, label="CDR1")
	m = m + annotate("segment", x = 38.5, y = -0.05, xend=55.5, yend=-0.05, colour="darkgreen", size=1) + annotate("text", x = 47, y = -0.1, label="FR2")
	m = m + annotate("segment", x = 55.5, y = -0.07, xend=65.5, yend=-0.07, colour="darkblue", size=1) + annotate("text", x = 60.5, y = -0.15, label="CDR2")
	m = m + annotate("segment", x = 65.5, y = -0.05, xend=104.5, yend=-0.05, colour="darkgreen", size=1) + annotate("text", x = 85, y = -0.1, label="FR3")
	m = m + expand_limits(y=c(-0.1,1)) + xlab("AA position") + ylab("Frequency") + ggtitle(graph.title)
	m = m + theme(panel.background = element_rect(fill = "white", colour="black"), panel.grid.major.y = element_line(colour = "black"), panel.grid.major.x = element_blank())
	#m = m + scale_colour_manual(values=c("black"))

	print("---------------- write/print ----------------")

	dat.sums = data.frame(index=1:length(mutations.at.position), mutations.at.position=mutations.at.position, aa.at.position=aa.at.position)

	write.table(dat.sums, paste(outdir, "/aa_histogram_sum_", gene, ".txt", sep=""), sep="\t",quote=F,row.names=F,col.names=T)
	write.table(mutations.by.id.gene, paste(outdir, "/aa_histogram_count_", gene, ".txt", sep=""), sep="\t",quote=F,row.names=F,col.names=T)
	write.table(absent.aa.by.id.gene, paste(outdir, "/aa_histogram_absent_", gene, ".txt", sep=""), sep="\t",quote=F,row.names=F,col.names=T)
	write.table(dat_dt, paste(outdir, "/aa_histogram_", gene, ".txt", sep=""), sep="\t",quote=F,row.names=F,col.names=T)

	png(filename=paste(outdir, "/aa_histogram_", gene, ".png", sep=""), width=1280, height=720)
	print(m)
	dev.off()

	ggsave(paste(outdir, "/aa_histogram_", gene, ".pdf", sep=""), m, width=14, height=7)
}


library(ggplot2)
library(data.table)
library(dplyr)

if(! exists("inputFile")){
  args <- commandArgs(TRUE)
  if(length(args) == 0){
    setwd("/scratch/cqs/shengq2/macrae_linton/20190517_linton_exomeseq_3321_human/GATK4_CNV_Germline_10_CNVGenesPlot/result")
    inputFile<-"linton_exomeseq_3321.position.txt.1000"
    outputPrefix<-'linton_exomeseq_3321.position'
    sizeFactorFile<-"/scratch/cqs/shengq2/macrae_linton/20190517_linton_exomeseq_3321_human/GATK4_CNV_Germline_08_SizeFactor/result/linton_exomeseq_3321.txt.sizefactor"
  }else{
    inputFile<-args[1]
    outputPrefix<-args[2]
    sizeFactorFile<-args[3]
  }
}

cat(inputFile, "\n")
rawTable<-fread(file=inputFile, sep="\t", header=T)

readSizeFactors<-function(fileName){
  sizeFactorsAll<-read.table(sizeFactorFile, sep="\t", header=T)
  sizeFactorsAll<-sizeFactorsAll[!is.na(sizeFactorsAll$SizeFactor),]
  sfMedian<-sizeFactorsAll %>% group_by(Sample) %>% summarise(Median=median(SizeFactor))

  result<-sfMedian$Median
  names(result)<-sfMedian$Sample
  return(result)
}

sizeFactors<-readSizeFactors(sizeFactorFile)

cols <- c("DEL" = "blue", "DUP" = "red", "NoIndel" = "gray")
selectedFeature<-unique(rawTable$Feature)[1]
for (selectedFeature in unique(rawTable$Feature)) {
  cat(paste0(selectedFeature, "\n"))

  dataForPlot<-rawTable[which(rawTable$Feature==selectedFeature),]
  dataForPlot$NormalizedPositionCount<-round(dataForPlot$PositionCount * sizeFactors[dataForPlot$File])
  dataForPlot$FileName<-gsub("_", " ", dataForPlot$File)

  hasCNV<-(("CNV" %in% colnames(dataForPlot)) & (length(unique(dataForPlot$CNV)) > 1))

  if(hasCNV){
    dataForPlot$CNV[dataForPlot$CNV=="FALSE"] = "DEL"
    dataForPlot$CNV[dataForPlot$CNV=="NOREAD"] = "DEL"
    dataForPlot$CNV[dataForPlot$CNV==""] = "NoIndel"
  }

  if(grepl("_\\d+_\\d+$", selectedFeature)){
    parts=unlist(strsplit(selectedFeature, "_"))
    featureName=paste0(paste0(parts[1:(length(parts)-2)]), ":", parts[length(parts)-1], "-", parts[length(parts)])
  }else{
    featureName = selectedFeature
  }

  columns<-c("Read count", "Percentage", "Normalized read count")
  names(columns)<-c("PositionCount", "Percentage", "NormalizedPositionCount")
  column<-"NormalizedPositionCount"
  for (column in names(columns)){
    outputFile = paste0(outputPrefix, ".", selectedFeature, ".", column, ".pdf")
    #if (file.exists(outputFile)){
    #  next
    #}
    pdf(outputFile, height=max(4, length(unique(dataForPlot$FileName))), width=6, onefile=TRUE)
    g <- ggplot(dataForPlot, aes_string("Position", column)) +
      ylab(columns[column]) +
      xlab(unique(dataForPlot$Locus))+
      geom_line()

    if(featureName != unique(dataForPlot$Locus)){
      g <- g + ggtitle(featureName)
    }

    if(hasCNV){
      g <- g + geom_area(aes(fill=CNV), alpha=.5)+
        scale_fill_manual(values=cols)
    }else{
      g <- g + geom_area(fill = "black")
    }

    g <- g +
      facet_grid(FileName~.) +
      theme_bw() +
      theme(strip.background=element_blank())

    print(g)
    dev.off()
  }
}