############################
# Script for performing forecast scenarios for a high number of F combinations

###

my.area<-c('North Sea','Baltic Sea')[1]   ## North Sea or Baltic Sea runs
KeyRunYear<-c(2014,2017)[2]    # Number of species depends on key run (year)

ask.me<-F  # you can be asked to confirm a run
HPC<-F   # use High Performance Computer (on DTU Unix only)

do.MSFD.indicators<-F
make.new.run<-T       # make a new run, or use the results from a previously run for plotting etc.

##########  Choose arae (above) and define scenario (e.g. HCR_1_deter_noadjust_test from keywords below .

#sce<- 'HCR_1_stoch21_noadjust_single_test'
sce<- 'HCR_1_stoch21_noadjust_test'

########## key_word for scenarios, for automatic settings, one from each group
# key words :
# _stoch1_  : recruitmentMode<-1
# _stoch2_  : recruitmentMode<-2
# _stoch3_  : recruitmentMode<-3
# _stoch21_ : recruitmentMode<-21   # Equisim
# _deter_   : recruitmentMode<-0    # deterministic recruitment
# _deter20_ : recruitmentMode<-21   # deterministic recruitment Equisim
#
#_noadjust_" : recruit.adjust.CV<-0
#_adjust_" : recruit.adjust.CV<-2   # median to mean, only deterministic recruitment
#
# HCR_1    : HCR 1 (fixed F)
# HCR_2    : HCR_2 (SSB dependent F)
#
# _single  : single species mode for creation af F combinations (just the diagonal), (default multispecies mode=all combinations)
#
# _wide1 .. _wide9    : combinations of F values
# _narrow1  .. _narow9: combinations of F values
# _test               : combinations of F values

##################################################################################
# Define combinations of F

if (grepl('_test',sce)) {
  ss<-0.5
  if (KeyRunYear==2017) rr<-matrix(c(
    0.1, 0.5, ss,     #  COD
    0.1, 0.5, ss,      # WHG
    0.1, 0.5, ss,     #  HAD
    0.1, 0.5, ss,     # POK
    0.1, 0.5, ss,     # Mackerel
    0.1, 0.3, ss,     #  HER
    0.1, 0.5, ss,     # North SAN
    0.1, 0.5, ss,     # South SAN
    0.1, 1.0, ss,     #  NOP
    0.1, 1.0, ss,     # SPR
    0.2,  0.2,  ss,  #  PLE
    0.2,  0.2,  ss )  #  SOL
    ,ncol=3,byrow=T)
  dimnames(rr)[[1]]<-sp.names[first.VPA:nsp]
}

##################################################################################
if (OS!="unix" ) HPC<-FALSE

automatic.settings<-T   # set options automaticacly from sceenario name (sce)

# Stochastic recruitment.
# 0=No,  S-R parameters from OP_config.dat
# 1=Yes  S-R parameters and variance from OP_config.dat
# 2=Yes, S-R parameters from OP_config.dat and variance-covariance matrix from file covariance_Rec.in
# 3=Yes, S-R parameters from OP_config.dat and residuals from OP_SSB_rec_residuals.in

recruitmentMode<-1
if (automatic.settings) {
 if (grepl('_stoch',sce)) {
   if (grepl('_stoch1_',sce)) recruitmentMode<-1
   if (grepl('_stoch2_',sce)) recruitmentMode<-2
   if (grepl('_stoch3_',sce)) recruitmentMode<-3
   if (grepl('_stoch21_',sce)) recruitmentMode<-21
 }
 else if (grepl('_deter_',sce)) recruitmentMode<-0
 else if (grepl('_deter20_',sce)) recruitmentMode<-20

 if (grepl('_single',sce)) justDiagonal<-T else justDiagonal<-F
}

# adjust recruitment with half of the variance (factor exp(-(CV^2)/2) option adjust.recruit.CV
# 0=no adjustment\n",
# 1=stochastic recruitment, do adjustment such that average recruitment becomes equal to the median (downward adjustment)\n",
# 2=determenistic recruitment, do ajustment such that recruitment becomes equal to the mean of the log-normal distribution (upward adjustment)\n", file=file,append=T,sep="")
recruit.adjust.CV<-0
if (automatic.settings) {
 if (grepl("_noadjust_",sce)) recruit.adjust.CV<-0
 if (grepl("_adjust_",sce)) {
   if (recruitmentMode==0 |recruitmentMode==20 ) recruit.adjust.CV<-2 else  recruit.adjust.CV<-1
 }
}

nsp.VPA<-sum(SMS.control@species.info[,'predator'] %in% c(0,1))

recruit.adjust.factor<-rep(1,nsp.VPA)  # default no adjustment factor

if (OS=='windows') op.command<-'op.exe' else if (OS=='unix') op.command<-'op'

source(file=file.path(prog.path.func,'hcr_op_batch_simulate_function.r'))

if (my.area=='North Sea') {

  if (KeyRunYear==2014) {
    #                     Cod     Whiting     Haddock      Saithe     Herring     North-Sandeel South-Sandeel   Nor. pout       Sprat      Plaice        Sole
    HCR1<-            c(    1,          1,          1,          1,          1,          1,          1,             1,          1,          1,          1)
    HCR2<-            c(    2,          2,          2,          2,          2,         22,         22,            22,         22,          2,          2)
    my.FBlim.adjust<- c(    0,          0,          0,          0,          0,          1,          1,             1,          1,          0,          0)     # used for HCR2 to simulate escapement startegy for short lived sp
    my.FBlim.adjust3<-c(    1,          1,          1,          1,          1,          1,          1,             1,          1,          0,          0)     # used for HCR3

  } else if (KeyRunYear==2017) {
    #                     Cod     Whiting     Haddock      Saithe     Mackerel Herring     North-Sandeel South-Sandeel   Nor. pout       Sprat      Plaice        Sole
    HCR1<-            c(    1,          1,          1,          1,       1,      1,          1,          1,             1,          1,          1,          1)
    HCR2<-            c(    2,          2,          2,          2,       2,      2,         22,         22,            22,         22,          2,          2)
    my.FBlim.adjust<- c(    0,          0,          0,          0,       0,      0,          1,          1,             1,          1,          0,          0)     # used for HCR2 to simulate escapement startegy for short lived sp
    my.FBlim.adjust3<-c(    1,          1,          1,          1,       1,      1,          1,          1,             1,          1,          0,          0)     # used for HCR3
  }

} else if (my.area=='Baltic Sea') {
  #                     Cod       Herring        Sprat
  HCR1<-            c(    1,          1,          1)
  HCR2<-            c(    2,          2,          2)
  my.FBlim.adjust<- c(    0,          0,          0)
  my.FBlim.adjust3<-c(    1,          1,          1)
}

local.HCR<-HCR1
if (automatic.settings) {
 if (grepl("HCR_1",sce)) local.HCR<-HCR1
 if (grepl("HCR_2",sce)) local.HCR<-HCR2
}
if (automatic.settings) {
  cat('\nscenario name:',sce,'\n')
  cat('\nrecruitmentMode is set to:',  recruitmentMode)
  cat('\nrecruit.adjust.CV is set to:', recruit.adjust.CV)
  cat('\nHCR is set to :', local.HCR,'\n')
}

# defined later on    justAboveBlim<-T     # make plots only for Fcombinations where the risk to Blim <5%
# justAboveBpa<-F

fac.other.first.year<- -1  # first year for change (must be larger than first year in prediction)
fac.other.last.year<- -1  # last year for change
my.other<-rep(1,first.VPA-1);

FBlim.adjust<-my.FBlim.adjust  # F=0 at SSB at 0 (ICES default), except for short lived species

riskLevels<-rep(5,nsp.VPA)  # Risk levels accepted for SSB below Blim, used in stepwise reduction
if (my.area=='North Sea') riskLevels<-rep(5,nsp.VPA-2) # forget plaice and sole

keepPro<-0.95  # keep F values which is factor keepPro of  MSY, used in stepwise reduction

# scenarios

##### fixed M species
if (F) {
  source(file.path(prog.path.func,"fmsy_matrix_functions_tables.r"))

 POK.PL.SOL(HCR=HCR1,stochastic.recruitment=recruitmentMode,recruit.adjust.CV=2)
  #POK.PL.SOL(HCR=HCR2,stochastic.recruitment=recruitmentMode,recruit.adjust.CV=1)
  #POK.PL.SOL(HCR=HCR2,FBlim.adjust=my.FBlim.adjust,stochastic.recruitment=recruitmentMode,recruit.adjust.CV=0)
}

# result from previous run
PLE.seq<-seq(0.20,0.20,0.05)
SOL.seq<-seq(0.35,0.35,0.05)

#%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

# defined later on    justAboveBlim<-T     # make plots only for Fcombinations where the risk to Blim <5%
# justAboveBpa<-F

FBlim.adjust<-my.FBlim.adjust  # F=0 at SSB at 0 (ICES default), except for short lived species

################

my.OP.output=14  # default=14 for condensed output for  combinations of F

##########################################

if  (my.area=='North Sea') {

  if (justDiagonal) { # single species combinations ("just the diagonal")
    targetFs<-NULL
    rr2<-rr
    for (a in (1:dim(rr)[[1]])) {
      rr<-rr2
      rr[,2]<-rr[,1]
      rr[a,]<-rr2[a,]
      if (KeyRunYear==2014) Fs<-expand.grid(COD=seq(rr['Cod',1],rr['Cod',2],rr['Cod',3]),
                      WHG=seq(rr['Whiting',1],rr['Whiting',2],rr['Whiting',3]),
                      HAD=seq(rr['Haddock',1],rr['Haddock',2],rr['Haddock',3]),
                      POK=seq(rr['Saithe',1],rr['Saithe',2],rr['Saithe',3]),
                      HER=seq(rr['Herring',1],rr['Herring',2],rr['Herring',3]),
                      NSA=seq(rr['N. sandeel',1],rr['N. sandeel',2],rr['N. sandeel',3]),
                      SSA=seq(rr['S. sandeel',1],rr['S. sandeel',2],rr['S. sandeel',3]),
                      NOR=seq(rr['Nor. pout',1],rr['Nor. pout',2],rr['Nor. pout',3]),
                      SPR=seq(rr['Sprat',1],rr['Sprat',2],rr['Sprat',3]),
                      PLE=seq(rr['Plaice',1],rr['Plaice',2],rr['Plaice',3]),
                      SOL=seq(rr['Sole',1],rr['Sole',2],rr['Sole',3]))
      if (KeyRunYear==2017) Fs<-expand.grid(COD=seq(rr['Cod',1],rr['Cod',2],rr['Cod',3]),
                                            WHG=seq(rr['Whiting',1],rr['Whiting',2],rr['Whiting',3]),
                                            HAD=seq(rr['Haddock',1],rr['Haddock',2],rr['Haddock',3]),
                                            POK=seq(rr['Saithe',1],rr['Saithe',2],rr['Saithe',3]),
                                            MAC=seq(rr['Mackerel',1],rr['Mackerel',2],rr['Mackerel',3]),
                                            HER=seq(rr['Herring',1],rr['Herring',2],rr['Herring',3]),
                                            NSA=seq(rr['N. sandeel',1],rr['N. sandeel',2],rr['N. sandeel',3]),
                                            SSA=seq(rr['S. sandeel',1],rr['S. sandeel',2],rr['S. sandeel',3]),
                                            NOR=seq(rr['Nor. pout',1],rr['Nor. pout',2],rr['Nor. pout',3]),
                                            SPR=seq(rr['Sprat',1],rr['Sprat',2],rr['Sprat',3]),
                                            PLE=seq(rr['Plaice',1],rr['Plaice',2],rr['Plaice',3]),
                                            SOL=seq(rr['Sole',1],rr['Sole',2],rr['Sole',3]))

      targetFs<-rbind(targetFs,Fs)
  }}
  if (!justDiagonal) {
    if (KeyRunYear==2014) targetFs<-expand.grid(COD=seq(rr['Cod',1],rr['Cod',2],rr['Cod',3]),
                                                WHG=seq(rr['Whiting',1],rr['Whiting',2],rr['Whiting',3]),
                                                HAD=seq(rr['Haddock',1],rr['Haddock',2],rr['Haddock',3]),
                                                POK=seq(rr['Saithe',1],rr['Saithe',2],rr['Saithe',3]),
                                                HER=seq(rr['Herring',1],rr['Herring',2],rr['Herring',3]),
                                                NSA=seq(rr['N. sandeel',1],rr['N. sandeel',2],rr['N. sandeel',3]),
                                                SSA=seq(rr['S. sandeel',1],rr['S. sandeel',2],rr['S. sandeel',3]),
                                                NOR=seq(rr['Nor. pout',1],rr['Nor. pout',2],rr['Nor. pout',3]),
                                                SPR=seq(rr['Sprat',1],rr['Sprat',2],rr['Sprat',3]),
                                                PLE=seq(rr['Plaice',1],rr['Plaice',2],rr['Plaice',3]),
                                                SOL=seq(rr['Sole',1],rr['Sole',2],rr['Sole',3]))
    if (KeyRunYear==2017) targetFs<-expand.grid(COD=seq(rr['Cod',1],rr['Cod',2],rr['Cod',3]),
                                                WHG=seq(rr['Whiting',1],rr['Whiting',2],rr['Whiting',3]),
                                                HAD=seq(rr['Haddock',1],rr['Haddock',2],rr['Haddock',3]),
                                                POK=seq(rr['Saithe',1],rr['Saithe',2],rr['Saithe',3]),
                                                MAC=seq(rr['Mackerel',1],rr['Mackerel',2],rr['Mackerel',3]),
                                                HER=seq(rr['Herring',1],rr['Herring',2],rr['Herring',3]),
                                                NSA=seq(rr['N. sandeel',1],rr['N. sandeel',2],rr['N. sandeel',3]),
                                                SSA=seq(rr['S. sandeel',1],rr['S. sandeel',2],rr['S. sandeel',3]),
                                                NOR=seq(rr['Nor. pout',1],rr['Nor. pout',2],rr['Nor. pout',3]),
                                                SPR=seq(rr['Sprat',1],rr['Sprat',2],rr['Sprat',3]),
                                                PLE=seq(rr['Plaice',1],rr['Plaice',2],rr['Plaice',3]),
                                                SOL=seq(rr['Sole',1],rr['Sole',2],rr['Sole',3]))
  }

} else if  (my.area=='Baltic Sea') {
   targetFs<-expand.grid(COD=seq(rr['Cod',1],rr['Cod',2],rr['Cod',3]),
                       HER=seq(rr['Herring',1],rr['Herring',2],rr['Herring',3]),
                       SPR=seq(rr['Sprat',1],rr['Sprat',2],rr['Sprat',3]))
}

 #just checking
cat('\nScenarium:',sce,'\n')
cat('Area:',my.area,'\n')
cat('\nrecruitmentMode:',recruitmentMode,'\n')
dim(targetFs)

 if (recruitmentMode %in% c(0,20)) { # deterministic recruitment
    my.last.year<-2043; years.in.average<-2;
    nSim<-dim(targetFs)[[1]]
    nRuns<-1 # number of  batch run
    no.iters<-rep(1,nRuns)            # number of iterations in each batch run
    first.iter<-no.iters
    seqRuns<-round(seq(0,nSim,len=nRuns+1))
    first.run.no<-seqRuns[1:(length(seqRuns)-1)]+1
    cat("First run no in each run:",first.run.no,'\n')
    fromTo<-matrix(c(first.run.no,seqRuns[-1]),nrow=2,byrow=T)
} else  if (recruitmentMode>0) {   #stochastic recruitment
  if (justDiagonal) {years.in.average<-1; my.last.year<-2045; }
  else {years.in.average<-100; my.last.year<-2014+years.in.average+10;}

  if (OS=='windows') {
       kernels<-4  # number of CPU kernels
       nrep<-2     # number of iteration for each kernel
       if (justDiagonal) nrep<-125
   } else if (OS=='unix') {
      kernels<-10
      nrep<-1
   }
   no.iters<-rep(nrep,kernels)  # number of iterations in each batch run by CPUE kernel
   no.iter.stoch<-sum(no.iters)
   nRuns<-length(no.iters)
   first.run.no<-rep(1,nRuns)
   first.iter<-no.iters
   first.iter[1]<-1
   if (nRuns>1) for (i in (2:nRuns)) first.iter[i]<-round(first.iter[i-1]+no.iters[i-1])
   cat("Number of kernels used (number og runs):",kernels,"\nNumber of iterations:",no.iter.stoch,"\nFirst iteration no in each run:", first.iter,'\n')
   fromTo<-matrix(c(rep(1,nRuns),rep(dim(targetFs)[[1]],nRuns)),nrow=2,byrow=T)
}

if (make.new.run) {
  scenario.gem<-NULL
  scenario.gemmes<-NULL

  # run in batch mode

    for (i in (1:nRuns)) {
      NO<-formatC(i,w=2,flag="0")
      OP.simulate(scenario=paste(sce,NO,sep='_'),first.no.iter.stoch=first.iter[i],first.run.no=first.run.no[i],no.iter.stoch=no.iters[i],just.batch.file=T,HCR=local.HCR,targetFs=targetFs[fromTo[1,i]:fromTo[2,i],],FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,recruit.adjust.CV=recruit.adjust.CV,recruit.adjust.factor=recruit.adjust.factor,OP.output=my.OP.output,do.indicators=do.MSFD.indicators,fac.other.first.year=fac.other.first.year,fac.other.last.year=fac.other.last.year,fac.other=my.other)
      scenario.gemmes<-c(scenario.gemmes,scenario.gem)
    }

    save(scenario.gemmes, file =file.path(data.path, paste(sce,'scenarios.RData',sep='_')))

    if (HPC) {  #make the script that runs all the sub-scripts
     cat(paste(
          "#!/bin/sh",
          "# embedded options to qsub - start with #PBS",
          "# -- Name of the job ---",
          "#PBS -N OP_BATCH",
          "# –- specify queue --",
          "#PBS -q hpc",
          "# -- estimated wall clock time (execution time): hh:mm:ss --",
          "#PBS -l walltime=00:05:00",
          "# –- user email address --",
          "#PBS -M mv@aqua.dtu.dk",
          "# –- mail notification –-",
          "#PBS -m abe", "\n", sep='\n'),
          file=file.path(data.path,'OP_batch.sh'))
       for (i in  scenario.gemmes) {
        cat(paste(
            paste("chmod +rwx",file.path(data.path,i,'run_OP.sh')),
            paste("qsub",file.path(data.path,i,'run_OP.sh')),"# --\n",
            sep='\n'),append=T,file=file.path(data.path,'OP_batch.sh'))
       }
      stop("\nPlease note: you have to run (qsub) the shell script ",file.path(data.path,'OP_batch.sh'), " and wait for its completion \n")

     } else if (OS=='windows') {
        for (i in  scenario.gemmes) {
          cat('Command:',file.path(data.path,i,"run_OP.bat"),'has been submitted, wait for its completion\n')
          system2(command=file.path(data.path,i,"run_OP.bat"), wait = F,stdout = file.path(data.path,i,"Run_OP_out.dat"))

        }
        stop('\nPlease note: you have to wait runing the rest of the script, until the bach runs have completed.\nCheck the "Windows Task Manager", "CPUE usage" for completion\n')
     }

} else { # "old" run exist already
  load(file =file.path(data.path, paste(sce,'scenarios.RData',sep='_')),verbose = T)
}
#############################

if (FALSE) { # used to subset data befor further processing. Mainly used with very large datasets
   scenario<-paste(sce,'01',paste("HCR",local.HCR[1],sep=''),FBlim.adjust[1],paste("Rec",recruitmentMode,sep=''),ifelse(recruit.adjust.CV[1]==1 | recruit.adjust.CV[1]==2,"Recadj",""),my.last.year,sep='_')
   scenario.dir<-file.path(data.path,scenario)
   load(file =file.path(scenario.dir, "condensed.RData"),verbose=T)
   # op 2 condensed<-droplevels(subset(condensed,COD>=0.5 & WHG==0.15 & HAD>=0.55 &  HER>=0.35 & NSA>=0.55 & SSA>=0.35 & NOR>=0.45 & SPR>=0-65 ))
   condensed<-droplevels(subset(condensed,COD==0.3 & WHG==0.1 & HAD>=0.55 &  HER>=0.35 & (NSA==0.50 | NSA==0.55) & SSA>=0.35  & SSA<=0.45 & NOR>=0.45 & NOR<=0.5 & SPR>=0-65 ))
   save(condensed, file =file.path(scenario.dir, "condensed.RData"))
   rm(condensed)

   load(file =file.path(scenario.dir, "a.RData"),verbose=T)
   # op2 a<-droplevels(subset(a,COD>=0.5 & WHG==0.15 & HAD>=0.55 &  HER>=0.35 & NSA>=0.55 & SSA>=0.35 & NOR>=0.45 & SPR>=0-65 ))
    a<-droplevels(subset(a,COD==0.3 & WHG==0.1 & HAD>=0.55 &  HER>=0.35 & (NSA==0.50 | NSA==0.55) & SSA>=0.35  & SSA<=0.45 & NOR>=0.45 & NOR<=0.5 & SPR>=0-65 ))
   save(a, file =file.path(scenario.dir, "a.RData"));
   rm(a)

   load(file =file.path(scenario.dir, "indicators.RData"),verbose=T)
   dim(indi)
   # op2 indi<-droplevels(subset(indi,COD>=0.5 & WHG==0.15 & HAD>=0.55 &  HER>=0.35 & NSA>=0.55 & SSA>=0.35 & NOR>=0.45 & SPR>=0-65 ))
   indi<-droplevels(subset(indi,COD==0.3 & WHG==0.1 & HAD>=0.55 &  HER>=0.35 & (NSA==0.50 | NSA==0.55) & SSA>=0.35  & SSA<=0.45 & NOR>=0.45 & NOR<=0.5 & SPR>=0-65 ))
   dim(indi)
   save(indi, file =file.path(scenario.dir, "indicators.RData"))
}

  # read results into R data sets
 for (i in (1:nRuns)) {
    NO<-formatC(i,w=2,flag="0")
    OP.simulate(scenario=paste(sce,NO,sep='_'),first.run.no=first.run.no[i],first.no.iter.stoch=first.iter[i],targetFs=targetFs[fromTo[1,i]:fromTo[2,i],],do.simulation=F, read.condense=T,read.indicators=T, do.plot.indicators=F, do.plot.condense=F,HCR=local.HCR,FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,no.iter.stoch=no.iter.stoch,recruit.adjust.CV=recruit.adjust.CV)
  }

  # combine into one run
  justAboveBlim<-F  # do also include plot for just combinations with <5% risk to Blim for any species in the combination
  justAboveBpa<-F
  OP.simulate(scenario=paste(sce,'01',sep='_'),targetFs=targetFs,do.simulation=F,read.condense=F,read.indicators=F, do.plot.indicators=F, do.plot.condense=F,combine.scenario.run=T,ReduceTable=F,riskLevels=riskLevels,keepPro=keepPro,scenario.dirs=scenario.gemmes, scenario.dirs.out=scenario.gemmes[1], HCR=local.HCR,FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,justAboveBlim=justAboveBlim,justAboveBpa=justAboveBpa,recruit.adjust.CV=recruit.adjust.CV)

   # prepare to make plots, select potential subset
   if  (my.area=='North Sea') {

      if (TRUE) { # reduction 1

        if (KeyRunYear==2014) {
                           #COD    WHG   HAD  POK  HER  NSAN   SSAN   NOR    SPR  PLE  SOL
        my.Frange<-matrix(c(0.0,   0.0, 0.0, 0.0, 0.0, 0.0,   0.0,   0.0,   0.0, 0.0, 0.0,     # min
                            2.0,  0.10, 2.0, 2.0, 2.0, 2.0,   2.0,   2.0  , 2.0, 2.0, 2.0),     # max
                            ncol=11,byrow=T)  # min and max F for inclusion in plots, remember to set option subset.out below
        }
        if (KeyRunYear==2017) {
                             #COD    WHG   HAD  POK  MAC  HER  NSAN   SSAN   NOR    SPR  PLE  SOL
          my.Frange<-matrix(c(0.0,   0.0, 0.0, 0.0, 0.0, 0.0, 0.0,   0.0,   0.0,   0.0, 0.0, 0.0,     # min
                              2.0,  0.10, 2.0, 2.0, 2.0, 2.0, 2.0,   2.0,   2.0  , 2.0, 2.0, 2.0),     # max
                            ncol=12,byrow=T)  # min and max F for inclusion in plots, remember to set option subset.out below
        }

      }

    }  else if  (my.area=='Baltic Sea') {
                         #COD   HER  SPR        DOES NOT WORK YET for the BAltic
      my.Frange<-matrix(c(0.0,  0.0, 0.0,      # min
                          2.0,  2.0, 2.0 ),     # max
                          ncol=3,byrow=T)  # min and max F for inclusion in plots, remember to set option subset.out below
    }

  source(file.path(prog.path.func,"fmsy_matrix_functions_tables.r"))

  my.dev<-'png'

  library(gtools)
  #make plots
  my.subset.out<-F  # use subset of data  specified above
  my.ReduceTable<-F
  my.do.plot.condense<-T; my.do.plot.indicators<-T & do.MSFD.indicators;
  OP.simulate(singleSp=justDiagonal, scenario=paste(sce,'01',sep='_'),subset.out=my.subset.out,subset.species=my.Frange,my.dev=my.dev,targetFs=targetFs,do.simulation=F, read.condense=F,read.indicators=F,do.plot.indicators=my.do.plot.indicators,do.plot.condense=my.do.plot.condense,ReduceTable=my.ReduceTable,riskLevels=riskLevels,keepPro=keepPro,HCR=local.HCR,FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,no.iter.stoch=no.iter.stoch,justAboveBlim=justAboveBlim,justAboveBpa=justAboveBpa,recruit.adjust.CV=recruit.adjust.CV)

  ##################

  #Reduce tables,  default output
  #OP.simulate(scenario=paste(sce,'01',sep='_'),subset.out=F,subset.species=my.Frange,my.dev=my.dev,targetFs=targetFs,do.simulation=F, read.condense=F,read.indicators=F,do.plot.indicators=FALSE,do.plot.condense=FALSE,ReduceTable=TRUE,ReduceTableDetails=F,riskLevels=riskLevels,keepPro=keepPro,HCR=local.HCR,FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,no.iter.stoch=no.iter.stoch,justAboveBlim=justAboveBlim,justAboveBpa=justAboveBpa,recruit.adjust.CV=recruit.adjust.CV)

  ##################
if (FALSE) {
  #Reduce tables option12&3 and 4
  riskLevels<-rep(0.0,nsp.VPA-2) # risk level (%), -2 to forget plaice and sole
  keepPro<-0.95  # keep F values which is factor keepPro of  MSY, used in stepwise reduction
  saveOutput<-TRUE
  OP.simulate(scenario=paste(sce,'01',sep='_'),subset.out=F,subset.species=my.Frange,my.dev=my.dev,targetFs=targetFs,do.simulation=F, read.condense=F,read.indicators=F,do.plot.indicators=FALSE,do.plot.condense=FALSE,ReduceTable=TRUE,ReduceTableDetails=T,ReduceOption='option12', saveOutput=saveOutput,riskLevels=riskLevels,keepPro=keepPro,HCR=local.HCR,FBlim.adjust=FBlim.adjust,ask.me=ask.me,my.area=my.area,my.last.year=my.last.year, years.in.average= years.in.average,stochastic.recruitment=recruitmentMode,no.iter.stoch=no.iter.stoch,justAboveBlim=justAboveBlim,justAboveBpa=justAboveBpa,recruit.adjust.CV=recruit.adjust.CV)
}

 # load(file =file.path(scenario.dir, "option4.RData"),verbose=T)
 # load(file =file.path(scenario.dir, "indicators.RData"),verbose=T)
 # load(file =file.path(scenario.dir, "indi1.RData"),verbose=T)
#  names(indiNew)

if (F) {
  load(file =file.path(scenario.dir.simul, "condensed.RData"))
  a<-condensed
  a<-a[order(paste(formatC(a$Species.n,flag="0",w=3), formatC(a$run,flag="0",w=3),formatC(a$iteration,flag="0",w=3))),]
  head(a,20)

  res<-matrix(a$yield,ncol=10,nrow=11,byrow=F)
  dimnames(res)<-list(c('start',paste('F',sp.names[17:26],sep='_')),paste('yield',sp.names[17:26],sep='_'))
  res<-t(res)
  res

  avg<-res[,1]
  res.delta<-res

  for (r in (2:11)) res.delta[,r]<-res.delta[,r]-res.delta[,1]

  res.delta<-res.delta[,-1]
  res.delta<-res.delta /delta   # denne skal ligne admodel.dep filen

  cat("\nres.delta=admodel.dep\n")
  print(round(res.delta))    # svarende til en ?gning i F p? 1.0

  change.val<-0.10   # ?ndring p? 10%
  Fval.change<-Fval *change.val

  res.delta.change<-res.delta*Fval.change

  cat("\n relativ change of yield in percentage of a ",change.val*100,"% of species F (first line)\n")
  print(round(Fval,2))
  print(round(100*res.delta.change/avg,1))      # relative ?ndring

}


# Make null model simulations
# Set up to be called by wfs_make_sims_null_sbatch.sh
# arguments: pop c1 c2 yr1 yr2 trimlowsampsize rerunlow repnum nsims
#	pop: Can or Lof
#	c1: the first sample size (# chromosomes)
#	c2: the second sample size (# chromosomes)
#	yr1: 07 or 11 (not needed for Can)
#	yr2: 11_14 (not needed for Can): 11_14 means both 1907-2011 and 1907-2014
#	trimlowsampsize: 1 to trim loci with less than half of individuals genotypes, 0 to not trim
#	rerunlow: 0 for default, 1 to only run simulations for loci with low p-values (<=4/(n+1)) in wfs_nullmodel_pos&pvals_07-11-14.rds
#	repnum: which repetition number to run
# 	nsims: usually 500k

# read arguments
args=(commandArgs(trailingOnly=TRUE))

pop <- args[1]
c1 <- as.numeric(args[2])
c2 <- as.numeric(args[3])
yr1 <- args[4]
yr2 <- args[5]
trimlowsampsize <- as.numeric(args[6])
rerunlow <- as.numeric(args[7])
repnum <- as.numeric(args[8])
nsims <- as.numeric(args[9])

print(paste('Arguments: pop=', pop, ', c1=', c1, ', c2=', c2, ', yr1=', yr1, ', yr2=', yr2, ', trimlowsampsize=', trimlowsampsize, ', repnum=', repnum, ', nsims=', nsims, sep=''))

if(!(pop %in% c('Lof', 'Can'))) stop('pop must be Lof or Can!')
if(pop=='Lof' & !(yr1 %in% c('07', '11'))) stop('yr1 must be 07 or 11 for Lof!')
if(pop=='Lof' & !(yr2 %in% c('11', '14', '11_14'))) stop('yr2 must be 11 or 14 or 11_14 for Lof!')
if(!(trimlowsampsize %in% 0:1)) stop('trimlowsampsize must be 0 or 1!')
if(!(rerunlow %in% 0:1)) stop('rerunlow must be 0 or 1!')

if(pop=='Lof' & yr2 !='11_14' & rerunlow==1) stop('ERROR: yr2 must be 11_14 for Lof if rerunlow==1 right now!')

# load functions: assume this is run on a cod or abel node
source('scripts/wfs_byf1samp.r')
require(bit, lib.loc="/projects/cees/lib/R_packages/") # for use with ff
require(ff, lib.loc="/projects/cees/lib/R_packages/") # for big objects shared across processes
require(data.table, lib.loc="/projects/cees/lib/R_packages/")

# read in Ne and frequency data (latter to get starting frequencies)
if(pop=='Lof' & yr1=='07' & yr2=='11'){
	nes <- read.table('analysis/LOF_07_to_LOF_S_11_to_LOF_S_14.w_Ne_bootstrap.txt')[,1] # the values of Ne from wfabc_1
	nchrs <- fread('data_2019_03_18/Frequency_table_Lof07_Lof11.txt', header=TRUE) # read in frequency table data
	gen <- 11
}
if(pop=='Lof' & yr1=='07' & yr2=='14'){
	nes <- read.table('analysis/LOF_07_to_LOF_S_11_to_LOF_S_14.w_Ne_bootstrap.txt')[,1] # the values of Ne from wfabc_1
	nchrs <- fread('data_2019_03_18/Frequency_table_Lof07_Lof14.txt', header=TRUE) # read in frequency table data
	gen <- 11
}
if(pop=='Lof' & yr1=='11' & yr2=='14'){
	nes <- read.table('analysis/LOF_07_to_LOF_S_11_to_LOF_S_14.w_Ne_bootstrap.txt')[,1] # the values of Ne from wfabc_1
	nchrs <- fread('data_2019_03_18/Frequency_table_Lof11_Lof14.txt', header=TRUE) # read in frequency table data
	gen <- 11
}
if(pop=='Lof' & yr1=='07' & yr2=='11_14'){
	nes <- read.table('analysis/LOF_07_to_LOF_S_11_to_LOF_S_14.w_Ne_bootstrap.txt')[,1] # the values of Ne from wfabc_1
	nchrs11 <- fread('data_2019_03_18/Frequency_table_Lof07_Lof11.txt', header=TRUE) # read in frequency table data
	nchrs14 <- fread('data_2019_03_18/Frequency_table_Lof07_Lof14.txt', header=TRUE) # read in frequency table data
	nchrs11[,yr:=11]
	nchrs14[,yr:=14]
	nchrs <- rbind(nchrs11, nchrs14) # all possible sample size combinations across both pairs of years
	gen <- 11
}
if(pop=='Can') {
	nes <- read.table('analysis/Can_40_to_Can_25kmer_dp.w_Ne_bootstrap.txt')[,1] # the values of Ne from wfabc_1
	freqfile <- paste('data_2019_03_18/Frequency_table_CAN_40_TGA.txt', sep='')
	nchrs <- fread(freqfile, header=TRUE)
	gen <- 8
}

setnames(nchrs, 3:7, c('N_CHR_1', 'Freq_1', 'N_CHR_2', 'Freq_2', 'ABS_DIFF'))

# trim out missing loci
	print(paste(nrow(nchrs), 'loci initially'))
nchrs <- nchrs[N_CHR_1>0 & N_CHR_2>0,]
	print(paste(nrow(nchrs), 'loci after trimming out missing loci'))

# trim to loci with at least half of individuals genotyped?
if(trimlowsampsize==1){
	print('Trimming to loci with at least half of the individuals genotyped')
	nrow(nchrs)
	nchrs <- nchrs[N_CHR_1>=max(N_CHR_1)/2 & N_CHR_2>=max(N_CHR_2)/2,]
	print(nrow(nchrs))
}

# trim only to loci with low p-values in wfs_nullmodel_pos&pvals_*.rds (from analysis of a previous run)?
if(rerunlow==1){
	print('Trimming to loci with p<=4/(n+1)')
	if(pop=='Lof' & yr2=='11_14'){
		pvals11 <- as.data.table(readRDS('analysis/wfs_nullmodel_pos&pvals_07-11.rds'))
		pvals14 <- as.data.table(readRDS('analysis/wfs_nullmodel_pos&pvals_07-14.rds'))
		pvals11[,yr:=11]
		pvals14[,yr:=14]
		pvals <- rbind(pvals11, pvals14)
		nchrs <- merge(nchrs, pvals[,.(CHROM, POS, n, p, yr)], by=c('CHROM', 'POS', 'yr')) # merge in p-values
	}
	if(pop=='Can') {
		pvals <- as.data.table(readRDS('analysis/wfs_nullmodel_pos&pvals_Can.rds'))
		nchrs <- merge(nchrs, pvals[,.(CHROM, POS, n, p)], by=c('CHROM', 'POS')) # merge in p-values
	}
	print(nrow(nchrs))
	nchrs <- nchrs[p<=4/(n+1),]
	print(nrow(nchrs))
}

# check that nes >0
print(summary(nes))
print(paste('Of', length(nes), 'Ne values', sum(nes > 0), 'are >0. Trimming all others out.'))
nes <- nes[nes>0]

# check where this is to monitor temp file creation in the file system /tmp/Rtmpp__
options('fftempdir')

# only run observed starting sample frequencies
nchr1s <- nchrs[c1==N_CHR_1 & c2==N_CHR_2, unique(round(Freq_1*N_CHR_1))] # number of starting copies of allele for the retained loci
f1samps=rep((nchr1s)/c1, rep(nsims, length(nchr1s)))

print(paste(length(nchr1s), 'starting frequencies to run'))
print(paste(length(f1samps), 'simulations to run'))

# make simulations
thisout <- sapply(f1samps, FUN=wfs_byf1samp, smin=0, smax=0, c1=c1, c2=c2, gen=gen, ne=nes, h=0.5, simplify=TRUE)

thisout.ff <- ff(thisout, dim=dim(thisout), dimnames=dimnames(thisout)) # create in tempdir

# save to permanent file (semi-permanent.. in my temp directory)
if(pop=='Lof') {
	outfile <- paste('analysis/temp/wfs_simsnull_ff', paste(c1, c2, sep=','), '_', repnum, sep='')
}
if(pop=='Can') {
	outfile=paste('analysis/temp/wfs_simsnullCAN_ff', paste(c1, c2, sep=','), '_', repnum, sep='')
}
ffsave(thisout.ff, file=outfile)
print(paste('saved', outfile))

# remove the temp files
delete(thisout.ff) # returns TRUE
rm(thisout.ff)


#!/usr/bin/env Rscript

# ---
# title: "process_batchcorrection"
# author: "zhong"
# date: "8/12/2021"
# output: html_document
# ---

## Clear workspace
##########{r packages, cache=FALSE}
rm(list=ls())
set.seed(123)
WORKDIR = '~/Singel-Cell-Analysis-From-Zero/Deal_With_Gene_Matrix/'
setwd(WORKDIR)
if (!dir.exists("../output")){
  dir.create(path = "../output", recursive = TRUE)
}

library(Seurat)
# library(SeuratDisk)
library(ggplot2)
library(patchwork)
library(dplyr)
library(showtext)
library(ggprism)
library(rstatix)
##########

showtext_auto(TRUE)
font_add("Times New Roman", regular = "/usr/share/fonts/truetype/msttcorefonts/Times_New_Roman.ttf")
PATH = "../Figure0_batch"
if (!dir.exists(PATH)){
  dir.create(path = PATH, recursive = TRUE)
}
theme_me <- theme(text = element_text(family="Times New Roman", size=10.5, face = "plain"),
                  axis.text = element_text(family="Times New Roman", size=10.5, face = "plain"),
                  plot.caption = element_text(family="Times New Roman", size=10.5, face = "italic"),
                  plot.title = element_text(family="Times New Roman", size=12, face = "bold"),
                  plot.subtitle = element_text(family="Times New Roman", size=11, face = "bold"),
                  legend.title = element_text(family="Times New Roman", size=11, face = "plain"),
                  strip.text = element_text(family="Times New Roman", size=10.5, face = "italic"))

theme_set(theme_me)

## Data loading
# To demonstrate mapping to this multimodal reference, we will use a dataset of 2,700 PBMCs generated by 10x Genomics and available via `SeuratData`.
# Noted that sct transformed can also be replaced with NormalizeData(), ScaleData() and FindVariableFeatures()

######{r tc.load echo=FALSE}
DATADIR = '~/Singel-Cell-Analysis-From-Zero/data_filtered_feature_bc_matrix/ASS5HD3'
WORKDIR = '~/Singel-Cell-Analysis-From-Zero/Deal_With_Gene_Matrix/'
folders = list.files(DATADIR)
library(stringr)
samples=str_split(folders,'PBMC',simplify = T)[,1]

setwd(DATADIR)
scrna.list = lapply(X = folders, FUN=function(x){
  x <- CreateSeuratObject(counts = Read10X(x), project=x, min.cells = 3, min.features = 200)
})
setwd(WORKDIR)

scrna.all <- merge(scrna.list[[1]],
                 y = c(scrna.list[[2]], scrna.list[[3]], scrna.list[[4]], scrna.list[[5]],
                       scrna.list[[6]], scrna.list[[7]], scrna.list[[8]]),
                 add.cell.ids = samples,
                 project = "LungProj")

##########{r without.batchcorr}
pbmckk <- scrna.all
tabk1 <- table(pbmckk$orig.ident)
DefaultAssay(pbmckk) <- 'RNA'
pbmckk[["percent.mt"]] <- PercentageFeatureSet(pbmckk, pattern = "^MT-")
pbmckk <- subset(pbmckk, subset = nFeature_RNA > 500 & nFeature_RNA < 3000 & percent.mt < 10)
tabk2 <- table(pbmckk$orig.ident)
write.table(tabk1, paste(PATH, "cell_count_wob.csv", sep='/'))
write.table(tabk2, paste(PATH, "cell_count_wob_qc.csv", sep='/'))

##########
pbmckk <- NormalizeData(pbmckk, verbose = FALSE)
pbmckk <- FindVariableFeatures(pbmckk, nfeatures = 3000, verbose = FALSE)
pbmckk <- ScaleData(pbmckk, verbose = FALSE)
pbmckk <- RunPCA(object = pbmckk, verbose = FALSE)
pbmckk <- RunUMAP(object = pbmckk, reduction = "pca", dims = 1:10)
plot.without.batch <- DimPlot(pbmckk, reduction = "umap", group.by = "orig.ident") + theme_me
rm(pbmckk)
# print(tabk1, tabk2)
print("Done with scrna without batch correction!")
#######################################

if (!file.exists("../output/pbmc_ASSHD_pbmc3k.orig.rds")) {
  scrna.list <- SplitObject(scrna.all, split.by = "orig.ident")
  rm(scrna.all)
  scrna.list <- lapply(X=scrna.list, FUN = SCTransform)
  # select features that are repeatedly variable across datasets for integration
  features <- SelectIntegrationFeatures(object.list = scrna.list, nfeatures=3000)
  scrna.list <- PrepSCTIntegration(object.list = scrna.list, anchor.features = features)
  # Perform integration
  scrna.list <- lapply(X=scrna.list, FUN = RunPCA, verbose = FALSE, features = features)
  pbmc3k.anchors <- FindIntegrationAnchors(object.list = scrna.list, anchor.features = features, normalization.method = "SCT", reduction = "rpca", dim = 1:10)
  # this command creates an 'integrated' data assay
  rm(scrna.list)
  pbmc3k <- IntegrateData(anchorset = pbmc3k.anchors, normalization.method = "SCT", dim = 1:10)

  ######
  #### We then save primary integration of pbmc3k and save it to files.
  ######{r tc.save.orig}
  WORKDIR = '~/Singel-Cell-Analysis-From-Zero/Deal_With_Gene_Matrix/'
  setwd(WORKDIR)
  # saveRDS(pbmc3k, file = "../output/pbmc_TC_orig.rds")
  # saveRDS(pbmc3k, file = "../output/pbmc_ASS_ILD_HD_orig.rds")
  saveRDS(pbmc3k, file = "../output/pbmc_ASSHD_pbmc3k.orig.rds")
  print("Saved data!")
  ######
} else{
  print("Loading data: pbmc_ASSHD_pbmc3k.orig.rds")
  pbmc3k <- readRDS("../output/pbmc_ASSHD_pbmc3k.orig.rds")
  print("Loaded")
}

## Quality Control
##########{r tc.quality.contrl fig.width=10}
DefaultAssay(pbmc3k) <- 'RNA'
pbmc3k[["percent.mt"]] <- PercentageFeatureSet(pbmc3k, pattern = "^MT-")
plot.qc.vlnplot <- VlnPlot(pbmc3k, features = c("nFeature_RNA", "nCount_RNA", "percent.mt"), ncol = 3)
plot1 <- FeatureScatter(pbmc3k, feature1 = "nCount_RNA", feature2 = "percent.mt") + NoLegend() + theme_me
plot2 <- FeatureScatter(pbmc3k, feature1 = "nCount_RNA", feature2 = "nFeature_RNA") + theme_me
plot.qc.scatter <- plot1 + plot2

##########

###### We then choose subset of pbmc3k and save it to files.
##########{r tc.save.qc}
tab1 <- table(pbmc3k$orig.ident)
write.table(tab1, paste(PATH, "cell_count_wb.csv", sep='/'))
##########

##########{r with.batchcorr}
DefaultAssay(pbmc3k) <- 'RNA'
pbmc3k[["percent.mt"]] <- PercentageFeatureSet(pbmc3k, pattern = "^MT-")
pbmc <- subset(pbmc3k, subset = nFeature_RNA > 500 & nFeature_RNA < 3000 & percent.mt < 10)
rm(pbmc3k)
DefaultAssay(pbmc) <- "integrated"
pbmc <- ScaleData(object = pbmc, verbose = FALSE)
pbmc <- RunPCA(object = pbmc, verbose = FALSE)
pbmc <- RunUMAP(object = pbmc, reduction = "pca", dims = 1:10)
plot.with.batch <- DimPlot(pbmc, reduction = "umap", group.by = "orig.ident")  + NoLegend() + theme_me

tab2 <- table(pbmc$orig.ident)
write.table(tab2, paste(PATH, "cell_count_wb_qc.csv", sep='/'))
rm(pbmc)
# print(tab1, tab2)
print("Done with scrna with batch correction!")
######################################
plot.batch <- plot.with.batch + plot.without.batch
ggsave("batch_correction.pdf", plot.batch, path = PATH, width = 15, height = 7)
ggsave("quality_control.pdf", plot.qc.scatter, path = PATH, width = 15, height = 7)
##########


#Creating CRAFTY input data - region.csv
#edited from create_CRAFTY_regionCSV2.0_JM_2018-04-27.r

#The following map data are needed as inputs
# - map of municipality IDs to be simulated
# - land cover from LandCoverMap.r (or ClassifyDisaggregateMap.r)
# - land prices from LandPriceMap.r
# - moisture and growing season capital maps from moistureMap.r
# - infrastrucutre capital from infrastructureMap.r

#script now assumes all these munis files are latlong with identical headers (but not that the spatial extent is identical)

#Also required:
# Municipality_area_and_IBGE_code_number.csv
# A list of states in which Double Cropping is possible

#Captials derived within this script are:
# - Moisture
# - Nature
# - Human
# - Development
# - Economic
# - Other
# - Other Agriculture
# - Land Protection

rm(list=ls())

library(raster)
library(tidyverse)

######
#FUNCTIONS
#raster to xyz  (with help from https://stackoverflow.com/a/19847419)
#sepcify input raster, whether nodata cells should be output, whether a unique cell ID should be added
#return is a matrix. note format is row (Y) then col (X)
extractXYZ <- function(raster, nodata = FALSE, addCellID = TRUE){

  vals <- raster::extract(raster, 1:ncell(raster))   #specify raster otherwise dplyr used
  xys <- rowColFromCell(raster,1:ncell(raster))
  combine <- cbind(xys,vals)

  if(addCellID){
    combine <- cbind(1:length(combine[,1]), combine)
  }

  if(!nodata){
    combine <- combine[!rowSums(!is.finite(combine)),]  #from https://stackoverflow.com/a/15773560
  }

  return(combine)
}

#read raster data function
readMapXYZ <- function(mapz)
{
  #map <- raster(path)     #read raster
  map <-flip(mapz, 'y')    #flip maps as CRAFTY read values from the bottom of the map
  map <- extractXYZ(map)  #convert from map to xyz (as tibble)
  return(as_tibble(map))  #return xyz as tibble
}

recode_if <- function(x, condition, ...){
  if_else(condition, recode(x, ...), x)
}
######

#land cover map provided should have 5 landcover classes:
#1 = Nature
#2 = Other Agri
#3 = Agri
#4 = Other
#5 = Pasture

ofname <- "region2001_2020-04-18.csv"  #output filename

#unzip if needed
#unzip(zipfile="Data/sim10_BRmunis_latlon_5km_2018-04-27.zip",exdir="Data")

#read required files
munis <- raster("Data/sim10_BRmunis_latlon_5km.asc") #map of municipality IDs to be simulated
#LC <- raster("Data/ObservedLCmaps/LandCover2001_PastureB_Disagg.asc")  #land cover from LandCoverMap.r (or ClassifyDisaggregateMap.r)
LC <- raster("Data/ObservedLCmaps/LandCover2001_PastureB_Disagg.asc")  #land cover from LandCoverMap.r (or ClassifyDisaggregateMap.r)
#Lpr <- raster('Data/LandPrice2001_Capital_nat1.asc')  #land prices from LandPriceMap.r
Lpr <- raster('Data/LandPrice/LandPrice_Capital_08_2001.asc')  #land prices from LandPriceMap.r

#mois <- raster('Data/Moisture/MoistureCap_OctNovDecJanFebMar_S_2001.asc')   #moisture capital from moistureMap.r
mois <- raster('Data/Moisture/MoistureCap_OctNovDecJanFebMar_S_2001.asc')   #moisture capital from moistureMap.r
#GS <- raster('Data/Moisture/GSCap_JanFebMarAprMayJun_S_2001.asc')   #growing season (double cropper) capital from moistureMap.r
GS <- raster('Data/Moisture/GSCap_JanFebMarAprMayJun_S_2001.asc')   #growing season (double cropper) capital from moistureMap.r

#infra <- raster('Data/PortAccessCapital/PortAccessCap2000.asc') #infrastrucutre capital from infrastructureMap.r
infra <- raster('Data/PortAccessCapital/PortAccessCap2020_2000.asc') #infrastrucutre capital from infrastructureMap.r
#SoyProtect <- raster('Data/landProtection/All_ProtectionMap.asc') #land protection is intially identical for all services
#MaizeProtect <- raster('Data/landProtection/All_ProtectionMap.asc') #land protection is intially identical for all services
#OAgriProtect <- raster('Data/landProtection/All_ProtectionMap.asc') #land protection is intially identical for all services
#PasProtect <- raster('Data/landProtection/All_ProtectionMap.asc') #land protection is intially identical for all services
SoyProtect <- raster('Data/landProtection/All_ProtectionMap_025.asc')
MaizeProtect <- raster('Data/landProtection/All_ProtectionMap_025.asc')
OAgriProtect <- raster('Data/landProtection/All_ProtectionMap_025.asc')
PasProtect <- raster('Data/landProtection/All_ProtectionMap_025.asc')

Economic <- raster('Data/AgriLocations2001.asc')

OAslope <- raster('Data/OAgri-slope_2018-08-16.asc')  #other agriculture cap set to slope
OAslope <- round(OAslope, 1)  #round because of long dp

NatAccess <- raster('Data/NatureAccess_2001_PastureB_Disagg_075.asc')
AgriAccess <- raster('Data/AgriAccess_2001_PastureB_Disagg_005.asc')
OAgriAccess <- raster('Data/OAgriAccess_2001_PastureB_Disagg_005.asc')

agriHarvest <- read_csv("Data/muni2001_harvestAreas.csv", col_types = ("iiiid")) #from DoubleCropping.rmd
DC <- F   #NO double cropping exists in 2001

#agriHarvest <- read_csv("Data/muni2018_plantedAreas_newDC.csv", col_types = ("iiiiiidddd")) #from DoubleCropping.rmd
#DC <- T   #double cropping exists in 2018

VH <- F
#specify csv containing spatially uniform capital values (as used in createUpdateFiles.r)
uniform_caps <- read_csv("Data/UniformCapitals_2020-02-10b.csv")
unis <- uniform_caps %>%
  filter(Year == 2001)   #select the uniform capital values for start year (edit if starting from a different year)

#if we want to see input maps
plt <- F
if(plt) {
  plot(munis, main = "munis")
  plot(LC, main = "LC")
  plot(Lpr, main = "Lpr")
  plot(mois, main = "moisture")
  plot(GS, main = "growing season")
  plot(infra, main = "Port Access")
  plot(SoyProtect, main = "SoyProtect")
  plot(MaizeProtect, main = "MaizeProtect")
  plot(OAgriProtect, main = "OAgriProtect")
  plot(PasProtect, main = "PasProtect")
}

#convert raster to xyz
munis.xy <- readMapXYZ(munis)
LC.xy <- readMapXYZ(LC)
Lpr.xy <- readMapXYZ(Lpr)
mois.xy <- readMapXYZ(mois)
GS.xy <- readMapXYZ(GS)
infra.xy <- readMapXYZ(infra)
SoyProtect.xy <- readMapXYZ(SoyProtect)
MaizeProtect.xy <- readMapXYZ(MaizeProtect)
OAgriProtect.xy <- readMapXYZ(OAgriProtect)
PasProtect.xy <- readMapXYZ(PasProtect)
OAslope.xy <- readMapXYZ(OAslope)
NatAccess.xy <- readMapXYZ(NatAccess)
AgriAccess.xy <- readMapXYZ(AgriAccess)
OAgriAccess.xy <- readMapXYZ(OAgriAccess)
Economic.xy <- readMapXYZ(Economic)

#create a list of unique municipality id values
u.mids <- unique(munis)

#joins (because Infrastructure, Moisture, Land Price maps are not perfectly aligned with munis)
#four service land protections use the same initial conditions
join.xy <- left_join(munis.xy, infra.xy, by = c("row", "col")) %>%
  dplyr::select(-V1.y) %>%
  rename(muniID = vals.x, `Port Access` = vals.y) %>%
  left_join(., round(Lpr.xy,3), by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Land Price" = vals) %>%
  left_join(., round(mois.xy,3), by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Moisture" = vals) %>%
  left_join(., round(GS.xy,3), by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Growing Season" = vals) %>%
  left_join(., SoyProtect.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Soy Protection" = vals) %>%
  left_join(., MaizeProtect.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Maize Protection" = vals) %>%
  left_join(., PasProtect.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Pasture Protection" = vals) %>%
  left_join(., OAgriProtect.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("OAgri Protection" = vals) %>%
  left_join(., OAslope.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Other Agriculture" = vals) %>%
  left_join(., NatAccess.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Nature Access" = vals) %>%
  left_join(., AgriAccess.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Agri Infrastructure" = vals) %>%
  left_join(., OAgriAccess.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("OAgri Infrastructure" = vals) %>%
  left_join(., round(Economic.xy,3), by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Economic" = vals)

if(VH){
  human <- raster('Data/HumanDev/HumanCapital2001.asc')
  human.xy <- readMapXYZ(human)

  join.xy <- left_join(munis.xy, human.xy, by = c("row", "col")) %>%
  dplyr::select(-V1) %>%
  rename("Human" = vals) %>%
  filter_all(all_vars(!is.na(.)))      #filter any rows missing data NA values
}

if(DC){
#add harvest areas and LC to the table then use to create FR column (see below for logic)
join.xy <- agriHarvest %>%
  dplyr::select(muniID, maize_prop, cum_ms_prop, dc_prop) %>%
  full_join(., join.xy, by = c("muniID"))

join.xy <- left_join(join.xy, LC.xy, by = c("row","col")) %>%
  dplyr::select(-V1) %>%
  rename(LC = vals) %>%
  mutate(rand = runif(n(),0,1)) %>%
  mutate(FR =
    if_else(LC == 1, "FR5",          #Nature
    if_else(LC == 2, "FR6",           #Other Agri
    if_else(LC == 3,                 #Agri #proportions calculated in DoubleCropping.rmd
      if_else(rand <= maize_prop, "FR2",
          if_else(rand <= cum_ms_prop,  "FR1",
              if_else(dc_prop > 0, "FR3",
                if_else(rand < 0.5, "FR2", "FR1")))),  #because there are some munis mapped that for some reason are not in the IBGE data...
    if_else(LC == 4, "FR7", "FR8")   #Other or Pasture
    ))))
}

if(!DC){

  join.xy <- agriHarvest %>%
  dplyr::select(muniID, maize_prop) %>%
  full_join(., join.xy, by = c("muniID"))

  join.xy <- left_join(join.xy, LC.xy, by = c("row","col")) %>%
  dplyr::select(-V1) %>%
  rename(LC = vals) %>%
  mutate(rand = runif(n(),0,1)) %>%
  mutate(FR =
    if_else(LC == 1, "FR5",          #Nature
    if_else(LC == 2, "FR6",           #Other Agri
    if_else(LC == 3,                 #Agri #proportions calculated in DoubleCropping.rmd
      if_else(rand <= maize_prop, "FR2", "FR1"),
    if_else(LC == 4, "FR7", "FR8")   #Other or Pasture
    ))))
}

#FR1 = Soy
#FR2 = Maize
#FR3 = Double Crop
#FR4 = Nature (LC1) [does not exist at model initialisation]
#FR5 = Nature (LC1)
#FR6 = Other Agri (LC2)
#FR7 = Other (LC4)
#FR8 = Pasture (LC5)

#add nature
join.xy <- join.xy %>%
  mutate(Nature = if_else(FR == "FR5", 1,       #virgin nature (FR4 not in initial state)
    if_else(FR == "FR8", 0.4,0)                     #pasture = 0.4, else 0
    ))

#decrease Nature Capital at edge (Nature Access as guide)
join.xy <- join.xy %>%
  mutate(Nature = if_else(Nature == 1 & `Nature Access` == 0.75, 0.75, Nature))

#write.table(join.xy, file=paste0("Data/","joinxy-",ofname), sep=",",row.names=F) #use write.table to wrap 'Cognitor' and column headers in quotes

#add columns that are either uniform or simple row number
region.xy <- join.xy %>%
  rename(Y = row, X = col) %>%
  mutate(agentid = row_number()) %>%  #add dummy agent_ID
  mutate(BT = "Cognitor") %>%
  mutate(Development = unis$Development) %>%
  #mutate(Economic = 1.0) %>%
  mutate(Other = if_else(FR == "FR7", 1,0)) %>%                #if Other LC set Capital to 1
  mutate(Climate = Moisture) %>%
  mutate(`Port Access` = round(`Port Access`, 3))  #added to prevent many dps (unknown why)

if(!VH){
  region.xy <- mutate(region.xy, Human = unis$Human)
}

region <- region.xy %>%
  dplyr::select(Y,X,muniID,BT,FR,Moisture,Nature,Human,Development,`Port Access`,Economic,`Nature Access`,`Land Price`,`Growing Season`,`Other Agriculture`,Other,`Soy Protection`,`Maize Protection`,`Pasture Protection`,`OAgri Protection`,`Agri Infrastructure`,`OAgri Infrastructure`) %>%
  filter_all(., all_vars(!is.na(.))) #%>%
  #rename(" " = V1.x)

write.table(region, file=paste0("Data/",ofname), sep=",",row.names=F) #use write.table to wrap 'Cognitor' and column headers in quotes


###############################################################
# oml4spark_function_build_all_classification_models.r
#
# Function to Build all classification Models available in
# OML4Spark and export their predictions against a given TEST data
# provided
#
# Input can be HDFS ID, HIVE, IMPALA, Spark DF or R dataframe
#
# Usage: buildAllClassificationModels( INPUT_DATA ,
#                                      TEST_DATA ,
#                                      formula_class
#                                      prop = NULL ,
#                                      legend = '' ,
#                                      feedback = FALSE ,
#                                     )
#
#
# Copyright (c) 2020 Oracle Corporation
# The Universal Permissive License (UPL), Version 1.0
#
# https://oss.oracle.com/licenses/upl/
#
###############################################################

##################################################################
### BUILDING ALL POSSIBLE BINARY CLASSIFICATION MODELS WITH OML4Spark
##################################################################
# INPUT_DATA=buildFolds
# TEST_DATA=testFold
# formula_class=formula_xval
# feedback=feedback
# formula_class <- CANCELLED ~ DISTANCE + as.factor(MONTH) + as.factor(YEAR) + as.factor(DAYOFWEEK) + as.factor(DAYOFMONTH) + as.factor(FLIGHTNUM)

buildAllClassificationModels <- function(INPUT_DATA,TEST_DATA, formula_class,
                                         prop=NULL, legend='', feedback=FALSE) {

  if (grepl(feedback, "FULL", fixed = TRUE))
  {verbose_user <- TRUE
  } else {verbose_user <- FALSE}
  targetFromFormula <- strsplit(deparse(formula_class), " ")[[1]][1] # Extract the Target variable from the formula
  if (startsWith(targetFromFormula,"as.factor(")) { targetFromFormula <- regmatches(targetFromFormula,
                                                                                gregexpr( "(?<=\\().+?(?=\\))",
                                                                                targetFromFormula,
                                                                                perl = T))[[1]]}
  # target for softmax requires a factor target
  aux.formula <- unlist(strsplit(deparse(formula_class), " "))
  aux.formula[1] <- paste0("as.factor(",targetFromFormula,")")
  formula_softmax <- as.formula(paste(aux.formula,collapse=" "))

  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('OML4Spark GLM2...')
  timing_glm2 <- list()
  start.time <- Sys.time()
  model_glm2 <- orch.glm2(formula = formula_class, data = INPUT_DATA, verbose=verbose_user)
  build_timing <- Sys.time()-start.time
  timing_glm2[[1]] <- as.numeric(build_timing,units = "secs")
  timing_glm2[[2]] <- format(build_timing,digits=4)
  start.time <- Sys.time()
  pred_glm2  <- predict(model_glm2, newdata = TEST_DATA, supplemental = targetFromFormula, type='response', verbose=verbose_user)
  score_timing <- Sys.time()-start.time
  timing_glm2[[3]] <- as.numeric(score_timing,units = "secs")
  timing_glm2[[4]] <- format(score_timing,digits=4)
  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_glm2[[1]]+timing_glm2[[3]] ,'secs \n')

  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('OML4Spark Neural with Entropy Output Activation...')
  timing_neu_ent <- list()
  start.time <- Sys.time()
  model_neu_ent <- orch.neural2(formula = formula_class, data = INPUT_DATA,
                                hiddenSizes    = c(20, 20),
                                activations    = c("sigmoid", "sigmoid", "entropy"),
                                seed           = 0, verbose=verbose_user)
  build_timing <- Sys.time()-start.time
  timing_neu_ent[[1]] <- as.numeric(build_timing,units = "secs")
  timing_neu_ent[[2]] <- format(build_timing,digits=4)
  start.time <- Sys.time()
  pred_neu_ent  <- predict(model_neu_ent, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
  score_timing <- Sys.time()-start.time
  timing_neu_ent[[3]] <- as.numeric(score_timing,units = "secs")
  timing_neu_ent[[4]] <- format(score_timing,digits=4)
  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_neu_ent[[1]]+timing_neu_ent[[3]] ,'secs \n')

  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('OML4Spark Neural with SoftMax Output Activation...')
  timing_neu_sof <- list()
  start.time <- Sys.time()
  model_neu_sof <- orch.neural2(formula = formula_softmax, data = INPUT_DATA,
                                hiddenSizes    = c(20, 20),
                                activations    = c("sigmoid", "sigmoid", "softmax"),
                                seed           = 0, verbose=verbose_user)
  build_timing <- Sys.time()-start.time
  timing_neu_sof[[1]] <- as.numeric(build_timing,units = "secs")
  timing_neu_sof[[2]] <- format(build_timing,digits=4)
  start.time <- Sys.time()
  pred_neu_sof  <- predict(model_neu_sof, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
  score_timing <- Sys.time()-start.time
  timing_neu_sof[[3]] <- as.numeric(score_timing,units = "secs")
  timing_neu_sof[[4]] <- format(score_timing,digits=4)
  if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_neu_sof[[1]]+timing_neu_sof[[3]] ,'secs \n')

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('OML4Spark ELM...')
    timing_elm <- list()
    start.time <- Sys.time()
    model_elm <- orch.elm(formula = formula_class, data = INPUT_DATA,zScoreX = TRUE,
                          l = 10, lambda = 1e-12 , verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_elm[[1]] <- as.numeric(build_timing,units = "secs")
    timing_elm[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_elm  <- predict(model_elm, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_elm[[3]] <- as.numeric(score_timing,units = "secs")
    timing_elm[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_elm[[1]]+timing_elm[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('ELM Model skipped due to factor interactions in formula \n')
  }

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('OML4Spark H-ELM...')
    timing_helm <- list()
    start.time <- Sys.time()
    model_helm <- orch.helm(formula = formula_class, data = INPUT_DATA,zScoreX = TRUE,
                            l = c(10L, 50L), lambdaAEnc = 1e-3, lambdaELM = 1e-9, verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_helm[[1]] <- as.numeric(build_timing,units = "secs")
    timing_helm[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_helm  <- predict(model_helm, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_helm[[3]] <- as.numeric(score_timing,units = "secs")
    timing_helm[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_helm[[1]]+timing_helm[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('H-ELM Model skipped due to factor interactions in formula \n')
  }

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Logistic...')
    timing_logistic <- list()
    start.time <- Sys.time()
    model_logistic <- orch.ml.logistic(formula = formula_class, data = INPUT_DATA, threshold = prop, verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_logistic[[1]] <- as.numeric(build_timing,units = "secs")
    timing_logistic[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_logistic <- predict(model_logistic, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_logistic[[3]] <- as.numeric(score_timing,units = "secs")
    timing_logistic[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_logistic[[1]]+timing_logistic[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Logistic Model skipped due to factor interactions in formula \n')
  }

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Decision Trees...')
    timing_dt <- list()
    start.time <- Sys.time()
    model_dt <- orch.ml.dt(formula = formula_class, data = INPUT_DATA,
                           threshold = prop, type="classification", verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_dt[[1]] <- as.numeric(build_timing,units = "secs")
    timing_dt[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_dt <- predict(model_dt, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_dt[[3]] <- as.numeric(score_timing,units = "secs")
    timing_dt[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_dt[[1]]+timing_dt[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Decision Trees Model skipped due to factor interactions in formula \n')
  }

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Random Forest...')
    timing_rf <- list()
    start.time <- Sys.time()
    model_rf <- orch.ml.random.forest(formula = formula_class, data = INPUT_DATA, nTrees = 100,
                                      threshold = prop, type="classification", verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_rf[[1]] <- as.numeric(build_timing,units = "secs")
    timing_rf[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_rf <- predict(model_rf, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_rf[[3]] <- as.numeric(score_timing,units = "secs")
    timing_rf[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_rf[[1]]+timing_rf[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib Random Forest Model skipped due to factor interactions in formula \n')
  }

  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib SVM...')
    timing_svm <- list()
    start.time <- Sys.time()
    model_svm <- orch.ml.svm(formula = formula_class, data = INPUT_DATA, threshold = prop, verbose=verbose_user)
    build_timing <- Sys.time()-start.time
    timing_svm[[1]] <- as.numeric(build_timing,units = "secs")
    timing_svm[[2]] <- format(build_timing,digits=4)
    start.time <- Sys.time()
    pred_svm <- predict(model_svm, newdata = TEST_DATA, supplemental = targetFromFormula, verbose=verbose_user)
    score_timing <- Sys.time()-start.time
    timing_svm[[3]] <- as.numeric(score_timing,units = "secs")
    timing_svm[[4]] <- format(score_timing,digits=4)
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('done in ',timing_svm[[1]]+timing_svm[[3]] ,'secs \n')
  } else {
    if (grepl(feedback, "FULL|TRUE", fixed = TRUE)) cat('Spark MLlib SVM Model skipped due to factor interactions in formula \n')
  }

  # Returns a list with all Models and their
  # Predictions Vector (which contains the Actual plus Predictions)
  out <- list()
  if (sum(grepl(':',formula_class,fixed=TRUE))==0) {
    out <- list(list(model_glm2,pred_glm2,paste0('OML4Spark GLM2',legend),timing_glm2),
                list(model_neu_ent,pred_neu_ent, paste0('OML4Spark Neural Nets - Entropy Activation',legend),timing_neu_ent),
                list(model_neu_sof,pred_neu_sof, paste0('OML4Spark Neural Nets - SoftMax Activation',legend),timing_neu_sof),
                list(model_elm,pred_elm, paste0('OML4Spark Extreme Learning Machines',legend),timing_elm),
                list(model_helm,pred_helm, paste0('OML4Spark Hybrid-Extreme Learning Machines',legend),timing_helm),
                list(model_logistic,pred_logistic, paste0('Spark MLlib Logistic',legend),timing_logistic),
                list(model_dt,pred_dt, paste0('Spark MLlib Decision Trees',legend),timing_dt),
                list(model_rf,pred_rf, paste0('Spark MLlib Random Forest',legend),timing_rf),
                list(model_svm,pred_svm, paste0('Spark MLlib Support Vector Machines',legend),timing_svm)
                #              ,
                #              list(model_lasso,pred_lasso, paste0('Spark MLlib LASSO',legend),timing_lasso)
    )
  } else {
    out <- list(list(model_glm2,pred_glm2,paste0('OML4Spark GLM2',legend),timing_glm2),
                list(model_neu_ent,pred_neu_ent, paste0('OML4Spark Neural Nets - Entropy Activation',legend),timing_neu_ent),
                list(model_neu_sof,pred_neu_sof, paste0('OML4Spark Neural Nets - SoftMax Activation',legend),timing_neu_sof))
  }

  return(out)

}  # END OF CLASSIFICATION MODELS


analysis.root.dir="/tmp/slastic-20110627-101440982-UTC/"
source(file="r-scripts/performance-logger/common.r")
analysis.logger.rel.dir="PerformanceEvaluator/PerformanceLogger/"

# TODO: read time window / output interval
#str="# winTimeMin=5; outputIntervalMin=2"
# sub("^.*winTimeMin=(\\d*).*$", "\\1", str, perl=TRUE) # returns 5

pdf(file="plots.pdf")

## CPU Utilization
analysis.component.rel.dir="ExecutionContainerCPUUtilizationLogger-winTimeSec_60-outputIntervalSec_15/"
#analysis.data.rel.fn="dbsrv-0-0--cpu0-1-.csv"
#analysis.data.rel.fn="appsrv-0-1--cpu0-3-.csv"
analysis.data.rel.fn="appsrv-0-1--cpu1-4-.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotExecutionContainerCPUUtilization(data.df)

## Memory/Swap Usage
analysis.component.rel.dir="ExecutionContainerMemSwapUsageLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="appsrv-0-1--memSwap-5-.csv"
#analysis.data.rel.fn="dbsrv-0-0--memSwap-2-.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotMemSwapUsage(data.df)

## Assembly Component Invocation Count Logger
analysis.component.rel.dir="AssemblyComponentInvocationCountLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="0--org.compiere.Wstore-org.compiere.Wstore__T.csv"
data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotAssemblyComponentInvocationCount(data.df)

analysis.component.rel.dir="AssemblyComponentInvocationCountLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="1--org.posterita.Businesslogic-org.posterita.Businesslogic__T.csv"
data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotAssemblyComponentInvocationCount(data.df)

## Deployment Component Invocation Count Logger
analysis.component.rel.dir="DeploymentComponentInvocationCountLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="0--appsrv-0-1--org.compiere.Wstore.csv"
#analysis.data.rel.fn="1--appsrv-0-1--org.posterita.Businesslogic.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotDeploymentComponentInvocationCount(data.df)

## Assembly Component Operation Execution Count Logger
analysis.component.rel.dir="AssemblyComponentOperationExecutionCountLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="0--org.compiere.Wstore-org.compiere.Wstore__T_loginServlet__doPost_35.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotAssemblyComponentOpExecCount(data.df)

## Deployment Component Operation Execution Count Logger
analysis.component.rel.dir="DeploymentComponentOperationExecutionCountLogger-winTimeSec_60-outputIntervalSec_15/"
#analysis.data.rel.fn="1--appsrv-0-1--org.posterita.Businesslogic.menuManager__buildMenuTree_55.csv"
#analysis.data.rel.fn="0--appsrv-0-1--org.compiere.Wstore.loginServlet__doGet_32.csv"
analysis.data.rel.fn="0--appsrv-0-1--org.compiere.Wstore.loginServlet__doPost_35.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotDeploymentComponentOpExecCount(data.df)

## Assembly Component Average Operation Execution RTs
analysis.component.rel.dir="AssemblyComponentAvgRTsLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="org.compiere.Wstore-org.compiere.Wstore__T.basketServlet__doPost_48.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotAssemblyComponentAvgOpExecRT(data.df)

## Deployment Component Average Operation Execution RTs
analysis.component.rel.dir="DeploymentComponentAvgRTsLogger-winTimeSec_60-outputIntervalSec_15/"
analysis.data.rel.fn="0--appsrv-0-1--org.compiere.Wstore.loginServlet__doPost_35.csv"

data.df=readTSFile(buildFN(analysis.root.dir,analysis.logger.rel.dir,analysis.component.rel.dir,analysis.data.rel.fn));
plotDeploymentComponentAvgOpExecRT(data.df)

#
dev.off()


## Need to generate a heatmap that has all 4 celltypes
## at the 1, 2,and 18hr marks. Need to generate one
## heatmap per timepoint and readout. The readouts are
## Smad23, and CTNNB1.
library(Cairo)

timepoints = c('1','2')

### NOTE: The 1hr HCEC pSmad158 timepoint is super-noisy
###       in the TI04.60 dataset. The kind of noisy that is
###       completely useless.
datafiles  = c('data/TI04.60_summaryStats.csv',
               'data/TI04.120_summaryStats.csv',
               'data/TI16.18_TB_summaryStats.csv')
reps       = 3
pValCutoff = 0.05
readouts   = c('pSmad158')
colors     = c('darkgreen')
colnames(colors)=readouts
channels   = c(2,2)
celltypes  = c('HCEC','SKMEL2','MALME3M','IEC6')
allTreatments = list(Smad23=c('ctrl+ctrl','ctrl+BMP4',
                           'TGFB3+ctrl','TGFB3+BMP4'),
                     pSmad158=c('ctrl+ctrl','TGFB3+ctrl',
                           'ctrl+BMP4','TGFB3+BMP4'))

setToOne   = c('ctrl+BMP4','ctrl+BMP4')
feature    = 'sum_nucleus'
allYlims   = list(HCEC   = c(-.15, 2 ),
                  SKMEL2 = c(-.15, 2 ),
                  MALME3M= c(-.15, 2 ),
                  IEC6   = c(-.15, 8 ) )

CairoSVG( 'bmpTgfbInsulationSmad158',width=8,height=2)
par(mfrow=c(1,4),family='sans')

for( rIdx in 1:length(readouts) ){

  treatments = allTreatments[[readouts[rIdx]]]

  for( cIdx in 1:length(celltypes) ){
    ylims      = allYlims[[celltypes[cIdx]]]
    readout   = readouts[  rIdx ]
    channel   = channels[  rIdx ]
    marker    = paste('marker',channel,sep='.')
    celltype  = celltypes[ cIdx ]

    plot(NA,NA,ylim=ylims, xlim=c(.75,length(treatments)*length(timepoints)+.25),
         main=celltype,ylab=readouts[rIdx],xaxt='n',xlab='',yaxt='n')
    abline(h=0,col='gray',lty=3)
    abline(h=1,col='gray',lty=3)
    axis(2, at=c(0,1))

    for( treatIdx in 1:length(treatments)){

      for( tIdx in 1:length(timepoints)){

        data = read.table( datafiles[tIdx], sep=',', header=T,
                           stringsAsFactors=F )

        treatment = treatments[treatIdx]

        subdata  = data[data$celltype==celltype &
                        data$marker.2==readouts[1]  &
                        !(data$experiment %in% c(1,4,5)) &
                        data$treatment %in% treatments,]

        # Convert to 0-1 scale
        thisFeature = paste(feature,'median',channel,sep='.')
        ctrls   = subdata[subdata$treatment == 'ctrl+ctrl',
                          thisFeature]
        subdata[,thisFeature] = subdata[,thisFeature]-mean(ctrls)
        subdata[,thisFeature] = subdata[,thisFeature]/
          mean(subdata[subdata$treatment==setToOne[rIdx],thisFeature])

        # Collect statistics
        isEqCtrl = t.test(subdata[subdata$treatment == 'ctrl+ctrl',thisFeature],
                          subdata[subdata$treatment == treatment,thisFeature])$p.value
        isEqOne  = t.test(subdata[subdata$treatment == setToOne[rIdx],thisFeature],
                          subdata[subdata$treatment == treatment,thisFeature])$p.value

        if(isEqCtrl<pValCutoff){
          isEqCtrl = '*'
        }else{isEqCtrl = ' '}

        if(isEqOne<pValCutoff){
          isEqOne = '*'
        }else{isEqOne = ' '}

        mu = mean(subdata[subdata$treatment == treatment,thisFeature])
        s  = sd(  subdata[subdata$treatment == treatment,thisFeature])

        # Plot with first two treatment error bars up, second two down
        x  = (treatIdx-1)*length(timepoints)+tIdx
        errorY = mu+s*c(1,-1)
        points( x, mu, pch=16, col=c('gray',colors[rIdx],'gray',colors[rIdx])[treatIdx])
        lines( c(x,x), errorY, lwd=1,
               col=c('darkgray',colors[rIdx],'darkgray',colors[rIdx])[treatIdx])
        par(family='mono')
        axis(1,at=x,
             paste(isEqOne,isEqCtrl,
                   sep=''), tick=F, las =3, cex.axis=1.5,
             col.axis=c('darkgray',colors[rIdx],'darkgray',colors[rIdx])[treatIdx])
        par(family='sans')
      }
    }
  }
}

dev.off()


#############################################################################
##
#W  sml512.r               GAP library of groups           Hans Ulrich Besche
##                                               Bettina Eick, Eamonn O'Brien
##

SMALL_GROUP_LIB[ 512 ][ 18 ] := [
"%%%%%//Yh0&,,8%&D8,,%E%%&%8,%%!]Vq5<!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^u\
h(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!J<q.c",
"%%%%%/8Yh09,,8%&D8,,%E%%(%8,%%!U.KV<V!U.TV(Y!L7K8(Y!L7T8(Y!L7KV(Y!L7TV(Y!U7\
K8(Y!U7T8(Y!U7KV(Y!U7TV(Y!.L,8(Y!.L58(Y!.L,V(Y!.L5V(Y!7L,8(Y!7L58(X",
"%%%%%/%Yh00,,8%&D8,,%E%%%%8,%%!SWe-b!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^t\
h(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]=h(}",
"%%%%%/%Yh00,,8%&D8,,%E%%(%8,%%!]Zd(}!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bxg(Y!^w\
d(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!J9k-b",
"%%%%%/%Yh00,,8%&D8,,%E%%&%8,%%!]uh53!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^t\
h(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(W",
"%%%%%/8Yh00,,8%&D8,,%E%%(%8,%%!46<b.c!7BYb(Y!7B=b(Y!7BZb(Y!.^<b(Y!.^Yb(Y!.^\
=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!U,8b({",
"%%%%%//Yh00,,8%&D8,,%E%%%%8,%%!]th<]!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Bu\
q(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y",
"%%%%%//Yh00,,8%&D8,,%E%%(%8,%%!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bx\
g(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(V",
"%%%%%//Yh00,,8%&D8,,%E%%&%8,%%!AWq5<!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Bt\
q(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!,=q.c",
"%%%%%/8Yh09,,8%(D8,,%E%b(%8,%%!LLR4<V!ULO1(Y!ULR1(Y!ULO4(Y!ULR4(Y!LUO1(Y!LU\
R1(Y!LUO4(Y!LUR4(Y!UUO1(Y!UUR1(Y!UUO4(Y!UUR4(Y!..(F(Y!..+F(Y!..(I(X",
"%%%%%/%Yh0&,,8%%D8,,%E%D%%8,%%!5Vn-b!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Bu\
h(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!A<q(}",
"%%%%%/%Yh0&,,8%%D8,,%E%D(%8,%%!AYg(}!Axg(Y!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bx\
d(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!,8n-b",
"%%%%%/%Yh0&,,8%%D8,,%E%D&%8,%%!Atq53!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Bu\
h(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(W",
"%%%%%/8Yh0&,,8%%D8,,%E%D(%8,%%!+6Yb.c!.B=b(Y!.BZb(Y!7B<b(Y!7BYb(Y!7B=b(Y!7B\
Zb(Y!.^<b(Y!.^Yb(Y!.^=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!L,Vb({",
"%%%%%//Yh0&,,8%%D8,,%E%D%%8,%%!Auh<]!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bt\
h(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y",
"%%%%%//Yh0&,,8%%D8,,%E%D(%8,%%!Axd(Y!Awg(Y!Axg(Y!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bw\
d(Y!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(V",
"%%%%%//Yh0&,,8%%D8,,%E%D&%8,%%!AVh5<!Auh(Y!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]u\
q(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!,<h.c",
"%%%%%//Yh0&,,8%%D/,,%E%D&%8,%%!Av8<V!Ay8(Y!AvV(Y!AyV(Y!]v8(Y!]y8(Y!]vV(Y!]y\
V(Y!Bv8(Y!By8(Y!BvV(Y!ByV(Y!^v8(Y!^y8(Y!^vV(Y!^yV(X",
"%%%%%/8Yh09,,8%%D8,,%E%D(%8,%%!R4TM-b!.L,8(Y!.L58(Y!.L,V(Y!.L5V(Y!7L,8(Y!7L\
58(Y!7L,V(Y!7L5V(Y!.U,8(Y!.U58(Y!.U,V(Y!.U5V(Y!7U,8(Y!7U58(Y!7U,V(Y!7O5V(}",
"%%%%%/%Yh00,,8%%D8,,%E%D%%8,%%!^Wq(}!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]w\
q(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^xh(Y!^wq(Y!K=k-b",
"%%%%%/%Yh00,,8%%D8,,%E%D(%8,%%!^xg53!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]t\
p(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(Y!^tm(Y!^um(Y!^tp(W",
"%%%%%/%Yh00,,8%%D8,,%E%D&%8,%%!TVq.c!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]x\
h(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^xh(Y!^<q({",
"%%%%%/8Yh00,,8%%D8,,%E%D(%8,%%!7^=b<]!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA\
8b(Y!UAVb(Y!UA9b(Y!UAWb(Y!L]8b(Y!L]Vb(Y!L]9b(Y!L]Wb(Y!U]8b(Y!U]Vb(Y",
"%%%%%//Yh00,,8%%D8,,%E%D%%8,%%!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]x\
h(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^xh(V",
"%%%%%//Yh00,,8%%D8,,%E%D(%8,%%!^Zd5<!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]t\
m(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(Y!^tm(Y!K9m.c",
"%%%%%//Yh00,,8%%D8,,%E%D&%8,%%!^uh<V!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]w\
h(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(X",
"%%%%%/8Yh09,,8%(D8,,%E%D(%8,%%!RR1G-b!UU4J(Y!UU1S(Y!UU4S(Y!..F,(Y!..I,(Y!..\
F5(Y!..I5(Y!7.F,(Y!7.I,(Y!7.F5(Y!7.I5(Y!.7F,(Y!.7I,(Y!.7F5(Y!.7I5(Y!71F,(}",
"%%%%%/%Yh0&,,8%&D8,,%E%D%%8,%%!^Vh(}!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Ax\
q(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!K<b-b",
"%%%%%/%Yh0&,,8%&D8,,%E%D(%8,%%!^wd53!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Au\
p(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(W",
"%%%%%/%Yh0&,,8%&D8,,%E%D&%8,%%!6Wq.c!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Aw\
q(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!B=q({",
"%%%%%/8Yh0&,,8%&D8,,%E%D(%8,%%!.^Zb<]!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LA\
Vb(Y!LA9b(Y!LAWb(Y!UA8b(Y!UAVb(Y!UA9b(Y!UAWb(Y!L]8b(Y!L]Vb(Y!L]9b(Y",
"%%%%%//Yh0&,,8%&D8,,%E%D%%8,%%!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Aw\
q(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(V",
"%%%%%//Yh0&,,8%&D8,,%E%D(%8,%%!BYg5<!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Au\
m(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!-8p.c",
"%%%%%//Yh0&,,8%&D8,,%E%D&%8,%%!Btq<V!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Ax\
h(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(X",
"%%%%%/8Yh09,,8%&D8,,%E%D(%8,%%!I4T/-b!L7KV(Y!L7TV(Y!U7K8(Y!U7T8(Y!U7KV(Y!U7\
TV(Y!.L,8(Y!.L58(Y!.L,V(Y!.L5V(Y!7L,8(Y!7L58(Y!7L,V(Y!7L5V(Y!.U,8(Y!.O58(}",
"%%%%%/%Yh00,,8%&D8,,%E%D%%8,%%!BWh(}!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Aw\
h(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!-=b-b",
"%%%%%/%Yh00,,8%&D8,,%E%D(%8,%%!Bxd53!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!At\
m(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(W",
"%%%%%/%Yh00,,8%&D8,,%E%D&%8,%%!6Vh.c!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^u\
q(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!B<h({",
"%%%%%/8Yh00,,8%&D8,,%E%D(%8,%%!.^<b<]!.^Yb(Y!.^=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^\
=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA8b(Y!UAVb(Y!UA9b(Y!UAWb(Y",
"%%%%%//Yh00,,8%&D8,,%E%D%%8,%%!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^u\
q(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(V",
"%%%%%//Yh00,,8%&D8,,%E%D(%8,%%!]Zg5<!Bwd(Y!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^w\
g(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!J9p.c",
"%%%%%//Yh00,,8%&D8,,%E%D&%8,%%!]uq<V!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^t\
q(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(X",
"%%%%%k8Yh09,,8%(D8,,%E%b(%8,%%!3lnoD-b!4moob(Y!1pnnb(Y!1ponb(Y!1pnob(Y!1poo\
b(Y!4pnnb(Y!4ponb(Y!4pnob(Y!4poob(Y!Fdbbb(Y!Fdcbb(Y!Fdbcb(Y!Fdccb(Y!Idbbb(Y!\
Idcbb(Y!Ibbcb(}",
"%%%%%M%Yh0&,,8%%D8,,%E%%%%8,%%!]Vq(}!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^u\
h(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!J<k-b",
"%%%%%M%Yh0&,,8%%D8,,%E%%(%8,%%!]wg53!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^x\
d(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(W",
"%%%%%M%Yh0&,,8%%D8,,%E%%&%8,%%!SWh.c!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^t\
h(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]=h({",
"%%%%%M8Yh0&,,8%%D8,,%E%%(%8,%%!7BYb<]!7B=b(Y!7BZb(Y!.^<b(Y!.^Yb(Y!.^=b(Y!.^\
Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA8b(Y",
"%%%%%M/Yh0&,,8%%D8,,%E%%%%8,%%!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^t\
h(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(V",
"%%%%%M/Yh0&,,8%%D8,,%E%%(%8,%%!]Yd5<!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bx\
g(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!J8m.c",
"%%%%%M/Yh0&,,8%%D8,,%E%%&%8,%%!]th<V!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Bu\
q(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(X",
"%%%%%M8Yh09,,8%%D8,,%E%%(%8,%%!I+TM-b!U.K8(Y!U.T8(Y!U.KV(Y!U.TV(Y!L7K8(Y!L7\
T8(Y!L7KV(Y!L7TV(Y!U7K8(Y!U7T8(Y!U7KV(Y!U7TV(Y!.L,8(Y!.L58(Y!.L,V(Y!.F5V(}",
"%%%%%M%Yh00,,8%%D8,,%E%%%%8,%%!AWq(}!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Bt\
q(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!,=k-b",
"%%%%%M%Yh00,,8%%D8,,%E%%(%8,%%!Axg53!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bw\
g(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(W",
"%%%%%M%Yh00,,8%%D8,,%E%%&%8,%%!5Vq.c!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Bu\
h(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!A<q({",
"%%%%%M8Yh00,,8%%D8,,%E%%(%8,%%!.B=b<]!.BZb(Y!7B<b(Y!7BYb(Y!7B=b(Y!7BZb(Y!.^\
<b(Y!.^Yb(Y!.^=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y",
"%%%%%M/Yh00,,8%%D8,,%E%%%%8,%%!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Bu\
h(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(V",
"%%%%%M/Yh00,,8%%D8,,%E%%(%8,%%!AZd5<!Awg(Y!Axg(Y!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bw\
d(Y!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!,9m.c",
"%%%%%M/Yh00,,8%%D8,,%E%%&%8,%%!Auh<V!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bt\
h(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(X",
"%%%%%M8Yh09,,8%(D8,,%E%%(%8,%%!II1G-b!LL4J(Y!LL1S(Y!LL4S(Y!UL1J(Y!UL4J(Y!UL\
1S(Y!UL4S(Y!LU1J(Y!LU4J(Y!LU1S(Y!LU4S(Y!UU1J(Y!UU4J(Y!UU1S(Y!UU4S(Y!.(F,(}",
"%%%%%M%Yh0&,,8%&D8,,%E%%%%8,%%!AVh(}!Auh(Y!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]u\
q(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!,<b-b",
"%%%%%M%Yh0&,,8%&D/,,%E%%%%8,%%!Av853!Ay8(Y!AvV(Y!AyV(Y!]v8(Y!]y8(Y!]vV(Y!]y\
V(Y!Bv8(Y!By8(Y!BvV(Y!ByV(Y!^v8(Y!^y8(Y!^vV(Y!^yV(W",
"%%%%%M%Yh0&,,8%&D8,,%E%%(%8,%%!TZg.c!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]t\
p(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(Y!^tm(Y!^um(Y!^tp(Y!^9p({",
"%%%%%M%Yh0&,,8%&D8,,%E%%&%8,%%!^uq<]!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]w\
q(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^xh(Y!^wq(Y",
"%%%%%M8Yh0&,,8%&D8,,%E%%(%8,%%!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA8b(Y!UA\
Vb(Y!UA9b(Y!UAWb(Y!L]8b(Y!L]Vb(Y!L]9b(Y!L]Wb(Y!U]8b(Y!U]Vb(Y!U]9b(V",
"%%%%%M/Yh0&,,8%&D8,,%E%%%%8,%%!^Vq5<!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]x\
h(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^xh(Y!K<q.c",
"%%%%%M/Yh0&,,8%&D8,,%E%%(%8,%%!^wg<V!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]u\
m(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(Y!^tm(Y!^um(X",
"%%%%%M/Yh0&,,8%&D8,,%E%%&%8,%%!TWe-b!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]w\
h(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(Y!^=h(}",
"%%%%%M8Yh09,,8%&D8,,%E%%(%8,%%!U4T8(}!U7KV(Y!U7TV(Y!.L,8(Y!.L58(Y!.L,V(Y!.L\
5V(Y!7L,8(Y!7L58(Y!7L,V(Y!7L5V(Y!.U,8(Y!.U58(Y!.U,V(Y!.U5V(Y!7U,8(Y!1O5%-b",
"%%%%%M%Yh00,,8%&D8,,%E%%%%8,%%!^uh53!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]w\
h(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y!^wh(W",
"%%%%%M%Yh00,,8%&D8,,%E%%(%8,%%!TYd.c!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Au\
p(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!Btp(Y!Bup(Y!^8m({",
"%%%%%M%Yh00,,8%&D8,,%E%%&%8,%%!^th<]!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Ax\
q(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!Bxq(Y",
"%%%%%M8Yh00,,8%&D8,,%E%%(%8,%%!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA\
9b(Y!LAWb(Y!UA8b(Y!UAVb(Y!UA9b(Y!UAWb(Y!L]8b(Y!L]Vb(Y!L]9b(Y!L]Wb(V",
"%%%%%M/Yh00,,8%&D8,,%E%%%%8,%%!BWq5<!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Aw\
q(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!Bwq(Y!-=q.c",
"%%%%%M/Yh00,,8%&D8,,%E%%(%8,%%!Bxg<V!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!At\
p(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!Bum(Y!Btp(X",
"%%%%%M/Yh00,,8%&D8,,%E%%&%8,%%!6Vn-b!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Ax\
h(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(Y!B<q(}",
"%%%%%M8Yh09,,8%(D8,,%E%b(%8,%%!LRO4(}!LUR4(Y!UUO1(Y!UUR1(Y!UUO4(Y!UUR4(Y!..\
(F(Y!..+F(Y!..(I(Y!..+I(Y!7.(F(Y!7.+F(Y!7.(I(Y!7.+I(Y!.7(F(Y!.7+F(Y!(1(G-b",
"%%%%%M%Yh0&,,8%%D8,,%E%D%%8,%%!Btq53!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Ax\
h(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y!Bxh(W",
"%%%%%M%Yh0&,,8%%D8,,%E%D(%8,%%!6Zd.c!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!At\
m(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!]up(Y!Btm(Y!B9m({",
"%%%%%M%Yh0&,,8%%D8,,%E%D&%8,%%!Buh<]!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Aw\
h(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!Bwh(Y",
"%%%%%M8Yh0&,,8%%D8,,%E%D(%8,%%!.^Yb(Y!.^=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^\
Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA8b(Y!UAVb(Y!UA9b(Y!UAWb(Y!L]8b(V",
"%%%%%M/Yh0&,,8%%D8,,%E%D%%8,%%!BVh5<!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^u\
q(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]xq(Y!-<h.c",
"%%%%%M/Yh0&,,8%%D8,,%E%D(%8,%%!Bwd<V!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^x\
g(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]tp(Y!]up(X",
"%%%%%M/Yh0&,,8%%D8,,%E%D&%8,%%!SWn-b!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^t\
q(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(Y!]=q(}",
"%%%%%M8Yh09,,8%%D8,,%E%D(%8,%%!U+TV(}!L7K8(Y!L7T8(Y!L7KV(Y!L7TV(Y!U7K8(Y!U7\
T8(Y!U7KV(Y!U7TV(Y!.L,8(Y!.L58(Y!.L,V(Y!.L5V(Y!7L,8(Y!7L58(Y!7L,V(Y!1F5D-b",
"%%%%%M%Yh00,,8%%D8,,%E%D%%8,%%!]uq53!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^t\
q(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y!]wq(W",
"%%%%%M%Yh00,,8%%D8,,%E%D(%8,%%!SYg.c!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bxg(Y!^wd(Y!^x\
d(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(Y!]um(Y!]8p({",
"%%%%%M%Yh00,,8%%D8,,%E%D&%8,%%!]tq<]!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^u\
h(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!]xh(Y",
"%%%%%M8Yh00,,8%%D8,,%E%D(%8,%%!7B=b(Y!7BZb(Y!.^<b(Y!.^Yb(Y!.^=b(Y!.^Zb(Y!7^\
<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(Y!LAWb(Y!UA8b(Y!UAVb(V",
"%%%%%M/Yh00,,8%%D8,,%E%D%%8,%%!]Wh5<!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Buq(Y!^t\
h(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]wh(Y!J=h.c",
"%%%%%M/Yh00,,8%%D8,,%E%D(%8,%%!]xd<V!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bwg(Y!Bxg(Y!^w\
d(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!Aup(Y!]tm(X",
"%%%%%M/Yh00,,8%%D8,,%E%D&%8,%%!SVe-b!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Bu\
q(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(Y!]<h(}",
"%%%%%M8Yh09,,8%(D8,,%E%D(%8,%%!UI1J(}!UL4J(Y!UL1S(Y!UL4S(Y!LU1J(Y!LU4J(Y!LU\
1S(Y!LU4S(Y!UU1J(Y!UU4J(Y!UU1S(Y!UU4S(Y!..F,(Y!..I,(Y!..F5(Y!..I5(Y!1(F%-b",
"%%%%%M%Yh0&,,8%&D8,,%E%D%%8,%%!]th53!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Btq(Y!Bu\
q(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y!Axq(W",
"%%%%%M%Yh0&,,8%&D8,,%E%D(%8,%%!5Zg.c!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bxd(Y!Bw\
g(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(Y!Atp(Y!A9p({",
"%%%%%M%Yh0&,,8%&D8,,%E%D&%8,%%!Auq<]!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Buh(Y!Bt\
q(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!Awq(Y",
"%%%%%M8Yh0&,,8%&D8,,%E%D(%8,%%!.BZb(Y!7B<b(Y!7BYb(Y!7B=b(Y!7BZb(Y!.^<b(Y!.^\
Yb(Y!.^=b(Y!.^Zb(Y!7^<b(Y!7^Yb(Y!7^=b(Y!7^Zb(Y!LA8b(Y!LAVb(Y!LA9b(V",
"%%%%%M/Yh0&,,8%&D8,,%E%D%%8,%%!AVq5<!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bth(Y!Bu\
h(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!Axh(Y!,<q.c",
"%%%%%M/Yh0&,,8%&D8,,%E%D(%8,%%!Awg<V!Axg(Y!]wd(Y!]xd(Y!]wg(Y!]xg(Y!Bwd(Y!Bx\
d(Y!Bwg(Y!Bxg(Y!^wd(Y!^xd(Y!^wg(Y!^xg(Y!Atm(Y!Aum(X",
"%%%%%M/Yh0&,,8%&D8,,%E%D&%8,%%!5We-b!Atq(Y!Auq(Y!]th(Y!]uh(Y!]tq(Y!]uq(Y!Bt\
h(Y!Buh(Y!Btq(Y!Buq(Y!^th(Y!^uh(Y!^tq(Y!^uq(Y!Awh(Y!A=h(}",
"%%%%%M8Yh09,,8%&D8,,%E%D(%8,%%!L+T8(}!L.KV(Y!L.TV(Y!U.K8(Y!U.T8(Y!U.KV(Y!U.\
TV(Y!L7K8(Y!L7T8(Y!L7KV(Y!L7TV(Y!U7K8(Y!U7T8(Y!U7KV(Y!U7TV(Y!.L,8(Y!(F5%-b" ];


#Aggregate cosmpolitan taxa in Bahram.
#clear environment, source paths, packages and functions.
rm(list=ls())
source('paths.r')
source('NEFI_functions/common_group_quantification.r')
library(data.table)

#set output paths.----
output.path <- bahram_16S_common_phylo_fg_abun.path

#load data.----
map <- as.data.frame(readRDS(bahram_metadata.path))
otu <- readRDS(bahram_dada2_SV_table_rare.path)
tax <- readRDS(bahram_tax_fg_16S.path)

#subset otu file to match mapping file and vice versa.----
map <- map[,c('Mapping.ID', 'Run','pC','cn','NPP','forest','conifer','relEM',"map", "map_sd", "mat", "mat_sd", "map_CV", "mat_CV")]
map <- map[complete.cases(map),]
map <- map[as.character(map$Run) %in% rownames(otu),]
otu <- otu[rownames(otu) %in% map$Run,]
otu <- otu[,colnames(otu) %in% rownames(tax)]
tax <- tax[rownames(tax) %in% colnames(otu),]

#get each level of taxonomy output.----
of_interest <- c('phylum','class','order','family','genus',colnames(tax)[8:19])
all_taxa_out <- list()
for(i in 1:length(of_interest)){
  all_taxa_out[[i]] <- common_group_quantification(otu,
                                                   tax,
                                                   unique(tax[,colnames(tax) == of_interest[i]]),
                                                   of_interest[i],
                                                   samp_freq = .95)
}
names(all_taxa_out) <- of_interest

#save output.----
saveRDS(all_taxa_out,output.path)