import BIDMat.{CMat,CSMat,DMat,Dict,IDict,FMat,FND,GDMat,GMat,GIMat,GSDMat,GSMat,HMat,Image,IMat,Mat,SMat,SBMat,SDMat}
import BIDMat.MatFunctions._
import BIDMat.SciFunctions._
import BIDMat.Solvers._
import BIDMat.JPlotting._
import BIDMach.Learner
import BIDMach.models.{FM,GLM,KMeans,KMeansw,ICA,LDA,LDAgibbs,NMF,RandomForest,SFA}
import BIDMach.datasources.{MatSource,FileSource,SFileSource}
import BIDMach.mixins.{CosineSim,Perplexity,Top,L1Regularizer,L2Regularizer}
import BIDMach.updaters.{ADAGrad,Batch,BatchNorm,IncMult,IncNorm,Telescoping}
import BIDMach.causal.{IPTW}

Mat.checkMKL
Mat.checkCUDA
Mat.setInline
if (Mat.hasCUDA > 0) GPUmem

"""
Explanation: BIDMach: parameter tuning
In this notebook we'll explore automated parameter exploration by grid search.
End of explanation
"""


var dir = "../data/rcv1/"             // adjust to point to the BIDMach/data/rcv1 directory
tic
val train = loadSMat(dir+"docs.smat.lz4")
val cats = loadFMat(dir+"cats.fmat.lz4")
val test = loadSMat(dir+"testdocs.smat.lz4")
val tcats = loadFMat(dir+"testcats.fmat.lz4")
toc

"""
Explanation: Dataset: Reuters RCV1 V2
The dataset is the widely used Reuters news article dataset RCV1 V2. This dataset and several others are loaded by running the script <code>getdata.sh</code> from the BIDMach/scripts directory. The data include both train and test subsets, and train and test labels (cats).
End of explanation
"""


val lrates = col(0.03f, 0.1f, 0.3f, 1f)        // 4 values
val texps = col(0.3f, 0.4f, 0.5f, 0.6f, 0.7f)  // 5 values

"""
Explanation: First lets enumerate some parameter combinations for learning rate and time exponent of the optimizer (texp)
End of explanation
"""


val lrateparams = ones(texps.nrows, 1) ⊗ lrates
val texpparams = texps ⊗ ones(lrates.nrows,1)
lrateparams \ texpparams

"""
Explanation: The next step is to enumerate all pairs of parameters. We can do this using the kron operator for now, this will eventually be a custom function:
End of explanation
"""


val (mm, opts) = GLM.learner(train, cats, GLM.logistic)

"""
Explanation: Here's the learner again:
End of explanation
"""


val nparams = lrateparams.length
val targmap = zeros(nparams, 103)
targmap(?,6) = 1

opts.targmap = targmap
opts.lrate = lrateparams
opts.texp = texpparams

mm.train

val (pp, popts) = GLM.predictor(mm.model, test)

"""
Explanation: To keep things simple, we'll focus on just one category and train many models for it. The "targmap" option specifies a mapping from the actual base categories to the model categories. We'll map from category six to all our models:
End of explanation
"""


pp.predict
val preds = FMat(pp.preds(0))

pp.model.asInstanceOf[GLM].mats.length

"""
Explanation: And invoke the predict method on the predictor:
End of explanation
"""


val vcats = targmap * tcats                                          // create some virtual cats
val lls = mean(ln(1e-7f + vcats ∘ preds + (1-vcats) ∘ (1-preds)),2)  // actual logistic likelihood
mean(lls)

"""
Explanation: Although ll values are printed above, they are not meaningful (there is no target to compare the prediction with). 
We can now compare the accuracy of predictions (preds matrix) with ground truth (the tcats matrix).
End of explanation
"""


val rocs = roc2(preds, vcats, 1-vcats, 100)   // Compute ROC curves for all categories

plot(rocs)

val aucs = mean(rocs)

"""
Explanation: A more thorough measure is ROC area:
End of explanation
"""


val (bestv, besti) = maxi2(aucs)

"""
Explanation: The maxi2 function will find the max value and its index.
End of explanation
"""


texpparams(besti) \ lrateparams(besti)

"""
Explanation: And using the best index we can find the optimal parameters:
End of explanation
"""


