/* 
 * This file is part of the UEA Time Series Machine Learning (TSML) toolbox.
 *
 * The UEA TSML toolbox is free software: you can redistribute it and/or 
 * modify it under the terms of the GNU General Public License as published 
 * by the Free Software Foundation, either version 3 of the License, or 
 * (at your option) any later version.
 *
 * The UEA TSML toolbox is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with the UEA TSML toolbox. If not, see <https://www.gnu.org/licenses/>.
 */
 
package utilities;
/*
 * Created on Dec 4, 2005

 */

import java.util.ArrayList;
import java.util.Random;

import evaluation.evaluators.SingleTestSetEvaluator;
import evaluation.storage.ClassifierResults;
import experiments.data.DatasetLoading;
import fileIO.OutFile;
import machine_learning.classifiers.kNN;
import statistics.distributions.NormalDistribution;
import tsml.classifiers.TSClassifier;
import tsml.data_containers.TimeSeriesInstance;
import tsml.data_containers.TimeSeriesInstances;
import weka.classifiers.Classifier;
import weka.classifiers.bayes.NaiveBayes;
import java.util.logging.Level;

import tsml.classifiers.EnhancedAbstractClassifier;
import tsml.classifiers.distance_based.utils.classifiers.contracting.TimedTest;
import tsml.classifiers.distance_based.utils.classifiers.results.ResultUtils;
import tsml.classifiers.distance_based.utils.system.logging.Loggable;
import tsml.classifiers.distance_based.utils.system.memory.MemoryWatcher;
import tsml.classifiers.distance_based.utils.system.timing.StopWatch;

import weka.classifiers.evaluation.EvaluationUtils;
import weka.classifiers.evaluation.NominalPrediction;
import weka.classifiers.functions.SMO;
import weka.classifiers.functions.supportVector.PolyKernel;
import weka.classifiers.lazy.IBk;
import weka.classifiers.meta.RotationForest;
import weka.classifiers.trees.J48;
import weka.classifiers.trees.RandomForest;
import weka.core.*;
import weka.filters.supervised.attribute.NominalToBinary;
import weka.filters.unsupervised.attribute.ReplaceMissingValues;

/**
 * @author ajb
 *
 *Methods to perform Classification tasks with Weka which I cant seem to
 *do in Weka 
 */
public class ClassifierTools {
	

/**
 * 	Simple util to find the accuracy of a trained classifier on a test set. Probably is a built in method for this! 
 * @param test
 * @param c
 * @return accuracy of classifier c on Instances test
 */
	public static double accuracy(Instances test, Classifier c){
		double a=0;
		int size=test.numInstances();
		Instance d;
		double predictedClass,trueClass;
		for(int i=0;i<size;i++)
		{
			d=test.instance(i);
			try{
				predictedClass=c.classifyInstance(d);
				trueClass=d.classValue();
				if(trueClass==predictedClass)
					a++;
//				System.out.println("True = "+trueClass+" Predicted = "+predictedClass);
			}catch(Exception e){
                            System.out.println(" Error with instance "+i+" with Classifier "+c.getClass().getName()+" Exception ="+e);
                            e.printStackTrace();
                            System.exit(0);
                        }
		}
		return a/size;
    }	
    
    public static double accuracy(TimeSeriesInstances test, TSClassifier c){
		double a=0;
		int size=test.numInstances();
		TimeSeriesInstance d;
		double predictedClass,trueClass;
		for(int i=0;i<size;i++)
		{
			d=test.get(i);
			try{
				predictedClass=c.classifyInstance(d);
				trueClass=d.getLabelIndex();
				if(trueClass==predictedClass)
					a++;
//				System.out.println("True = "+trueClass+" Predicted = "+predictedClass);
			}catch(Exception e){
                            System.out.println(" Error with instance "+i+" with Classifier "+c.getClass().getName()+" Exception ="+e);
                            e.printStackTrace();
                            System.exit(0);
                        }
		}
		return a/size;
	}	
	
	public static Classifier[] setDefaultSingleClassifiers(ArrayList<String> names){
		ArrayList<Classifier> sc2=new ArrayList<>();
		sc2.add(new kNN(1));
		names.add("NN");
		Classifier c;
		sc2.add(new NaiveBayes());
		names.add("NB");
		sc2.add(new J48());
		names.add("C45");
		c=new SMO();
		PolyKernel kernel = new PolyKernel();
		kernel.setExponent(1);
		((SMO)c).setKernel(kernel);
		sc2.add(c);
		names.add("SVML");
		c=new SMO();
		kernel = new PolyKernel();
		kernel.setExponent(2);
		((SMO)c).setKernel(kernel);
		sc2.add(c);
		names.add("SVMQ");
		c=new RandomForest();
		((RandomForest)c).setNumTrees(100);
		sc2.add(c);
		names.add("RandF100");
		c=new RotationForest();
		sc2.add(c);
		names.add("RotF30");
	
		Classifier[] sc=new Classifier[sc2.size()];
		for(int i=0;i<sc.length;i++)
			sc[i]=sc2.get(i);

		return sc;
	}
         
        
/**
 * This method returns the data in the same order it was given, and returns probability distributions for each test data
 * Assume data is randomised already
 * @param trainData
 * @param testData
 * @param c
 * @return distributionForInstance for each Instance in testData
 */
	public static double[][] predict(Instances trainData,Instances testData, Classifier c){
		double[][] results=new double[testData.numInstances()][];
		try{
			c.buildClassifier(trainData);
			for(int i=0;i<testData.numInstances();i++)
				results[i]=c.distributionForInstance(testData.instance(i));
		}catch(Exception e){
			System.out.println(" Error in manual cross val");
		}
		return results;
	}
/**
 * This method does a cross validation using the EvaluationUtils and stores the predicted and actual values.
 * I implemented this because I saw no way of using the built in cross vals to get the actual predictions,
 * useful for e.g McNemar's test (and cv variance). Note that the use of FastVector has been depreciated 
 * @param c
 * @param allData
 * @param m
 * @return
 */
    @SuppressWarnings({ "deprecation", "rawtypes" })
    public static double[][] crossValidation(Classifier c, Instances allData, int m){
            EvaluationUtils evalU;
            double[][] preds=new double[2][allData.numInstances()];
            Object[] p;
            FastVector f;
            NominalPrediction nom;
            try{
                    evalU=new EvaluationUtils();
                    evalU.setSeed(10);
                    f=evalU.getCVPredictions(c,allData,m);
                    p=f.toArray(); 
                    for(int i=0;i<p.length;i++)
                    {
                            nom=((NominalPrediction)p[i]);
                            preds[1][i]=nom.predicted();
                            preds[0][i]=nom.actual();
                    }
            }catch(Exception e){
                    System.out.println(" Error ="+e+" in method Cross Validate Experiment");
                    e.printStackTrace();
                    System.out.println(allData.relationName());
                    System.exit(0);

            }
            return preds;

    }
	
	
/**
* 	This method does a cross validation using the EvaluationUtils and stores t
* he predicted and actual values.
* Accuracy is stored in preds[0][0], StdDev of accuracy between folds SHOULD BE 
* stored in preds[1][0].
* TO IMPLEMENT!
* Could do with some testing, there is some uncertainty over the last fold.
* @param allData
* @param m
* @return
*/
    @SuppressWarnings({ "deprecation", "rawtypes" })
    public static double[][] crossValidationWithStats(Classifier c, Instances allData, int m)
    {
            EvaluationUtils evalU;
            double[][] preds=new double[2][allData.numInstances()+1];
            int foldSize=allData.numInstances()/m;  //Last fold may have fewer cases than this
            FastVector f;
            Object[] p;
            NominalPrediction nom;
            double acc=0,sum=0,sumsq=0;
            try{
                    evalU=new EvaluationUtils();
//				evalU.setSeed(10);
                    f=evalU.getCVPredictions(c,allData,m);
                    p=f.toArray(); 
                    for(int i=0;i<p.length;i++)
                    {
                        nom=((NominalPrediction)p[i]);
                        preds[1][i+1]=nom.predicted();
                        preds[0][i+1]=nom.actual();
//					System.out.println(" pred = "+preds[i+1]);
                        if(preds[0][i+1]==preds[1][i+1]){
                            preds[0][0]++;
                            acc++;
                        }
                        if((i>0 && i%foldSize==0)){
//Sum Squares                                        
                            sumsq+=(acc/foldSize)*(acc/foldSize);
//Sum                                                                           
                            sum+=(acc/foldSize);
                            acc=0;                                        
                        }
                    }
                    //Accuracy stored in preds[0][0]
                    preds[0][0]=preds[0][0]/p.length;
                    preds[1][0]=(sumsq-sum*sum/m)/m;
                    preds[1][0]=Math.sqrt(preds[1][0]);
            }catch(Exception e)
            {
                    System.out.println(" Error ="+e+" in method Cross Validate Experiment");
                    e.printStackTrace();
                    System.out.println(allData.relationName());
                    System.exit(0);

            }
            return preds;

    }
		
    public static double stratifiedCrossValidation(Instances data, Classifier c, int folds, int seed) throws Exception{
        Random rand = new Random(seed);   // create seeded number generator
        Instances randData = new Instances(data);   // create copy of original data
        randData.randomize(rand);         // randomize data with number generator
        randData.stratify(folds);
        int correct=0;
        int total=data.numInstances();
        for (int n = 0; n < folds; n++) {
           Instances train = randData.trainCV(folds, n);
           Instances test = randData.testCV(folds, n);
           c.buildClassifier(train);
            for(Instance ins:test){
                int pred=(int)c.classifyInstance(ins);
                if(pred==ins.classValue())
                    correct++;
            }
//                System.out.println("Finished fold "+n+" acc ="+((double)correct/((n+1)*test.numInstances())));
  
        }            
        return ((double)correct)/total;
    }

/**
 * This does a manual cross validation (i.e. without EvalUtils) and rather confusingly returns 
 * the distribution for each Instance (as opposed to the predicted/actual in method performCrossValidation
 * @param data
 * @param c
 * @param numFolds
 * @return distribution for each Instance 
 */
    public static double[][] performManualCrossValidation(Instances data, Classifier c, int numFolds)
    {
        double[][] results=new double[data.numInstances()][data.numClasses()];
        Instances train;
        Instances test;
        int interval = data.numInstances()/numFolds;
        int start=0;		
        int end=interval;
        int testCount=0;
        try{
            for(int f=0;f<numFolds;f++){
                //Split Data
                train=new Instances(data,0);
                test=new Instances(data,0);
                for(int i=0;i<data.numInstances();i++){
                    if(i>=start && i<end)
                            test.add(data.instance(i));
                    else
                            train.add(data.instance(i));
                }
                //Classify on training
                c.buildClassifier(data);
                //Predict
                for(int i=0;i<interval;i++){
                    results[testCount]=c.distributionForInstance(test.instance(i));
                    testCount++;
                }
                //Increment
                start=end;
                end=end+interval;
            }
        }catch(Exception e){
            System.out.println(" Error in manual cross val");
        }
        return results;
    }
	
/**
 * Writes the predictions vs actual of a pre trained classifier to a file
 * @param model
 * @param data
 * @param path
 */
    public static void makePredictions(Classifier model,Instances data, String path){
        OutFile f1 = new OutFile(path+".csv");
        double actual,pred;
        Instance t;
        try{
            for(int i=0;i<data.numInstances();i++){
                t=data.instance(i);
                actual=t.classValue();
                pred=model.classifyInstance(t);
                f1.writeLine(i+","+actual+","+pred);
            }
        }catch(Exception e){
            System.out.println("Exception in makePredictions"+e);
        }
    }

    public static Classifier[] setSingleClassifiers(ArrayList<String> names){
        ArrayList<Classifier> sc2=new ArrayList<Classifier>();
        IBk k=new IBk(50);
        k.setCrossValidate(true);
        sc2.add(k);
        names.add("kNN");
        Classifier c;
        sc2.add(new NaiveBayes());
        names.add("NB");
        sc2.add(new J48());
        names.add("C45");
        c=new SMO();
        PolyKernel kernel = new PolyKernel();
        kernel.setExponent(1);
        ((SMO)c).setKernel(kernel);
        sc2.add(c);
        names.add("SVML");
        c=new SMO();
        kernel = new PolyKernel();
        kernel.setExponent(2);
        ((SMO)c).setKernel(kernel);
        sc2.add(c);
        names.add("SVMQ");
        c=new RandomForest();
        ((RandomForest)c).setNumTrees(100);
        sc2.add(c);
        names.add("RandF100");
        c=new RotationForest();
        sc2.add(c);
        names.add("RotF30");

        Classifier[] sc=new Classifier[sc2.size()];
        for(int i=0;i<sc.length;i++)
                sc[i]=sc2.get(i);

        return sc;
    }

    public static double singleTrainTestSplitAccuracy(Classifier c, Instances train, Instances test){
        //Perform a simple experiment,
        double acc=0;
        try{
            c.buildClassifier(train);
            int correct=0;
            for(Instance ins:test){
                int pred=(int)c.classifyInstance(ins);
//                System.out.println((int)ins.classValue()+","+pred);
                if(pred==(int)ins.classValue())
                    correct++;
            }
            acc=correct/(double)test.numInstances();
        }catch(Exception e)
        {
            System.out.println(" Error ="+e+" in method singleTrainTestSplitAccuracy"+e);
            e.printStackTrace();
            System.exit(0);
        }
        return acc;
    }	
    
 /* Returns probability distribution for each instance with no randomisation  
    */
    public static double[][] crossValidate(Classifier c,Instances data,  int numFolds)
    {
        double[][] results=new double[data.numInstances()][data.numClasses()];
        Instances train;
        Instances test;
        int interval = data.numInstances()/numFolds;
        int start=0;		
        int end=interval;
        int testCount=0;
        try{
            for(int f=0;f<numFolds;f++){
                if(f==numFolds-1)
                    end=data.numInstances();
                //Split Data
                train=new Instances(data,0);
                test=new Instances(data,0);
                for(int i=0;i<data.numInstances();i++){
                    if(i>=start && i<end)
                            test.add(data.instance(i));
                    else
                            train.add(data.instance(i));
                }
                //Classify on training
                c.buildClassifier(train);
                //Predict
                for(int i=0;i<test.numInstances();i++){
                    results[testCount]=c.distributionForInstance(test.instance(i));
                    testCount++;
                }
                //Increment
                start=end;
                end=end+interval;
            }
        }catch(Exception e){
            System.out.println(" Error in manual cross val");
        }
        return results;
    }
    
    
    public static ClassifierResults constructClassifierResults(Classifier classifier, Instances test) throws Exception{
        //jamesl: no usages reported 19/02/2019, but this function header left in and refactored to
        //use up to date method in case external usages exist
        return new SingleTestSetEvaluator().evaluate(classifier, test);
    }

    /**
     * Conducts a full run of a classifier on some train and test data with a set seed, printing off the stats / results. This is intended for reducing boilerplate main method code in each classifier.
     * @param classifier
     * @param trainAndTestData
     * @param seed
     * @throws Exception
     */
    public static void trainTestPrint(Classifier classifier, Instances[] trainAndTestData, int seed) throws Exception {
        if(classifier instanceof Randomizable) {
            ((Randomizable) classifier).setSeed(seed);
        }
        Random random = new Random(seed);
        MemoryWatcher overallMemoryWatcher = new MemoryWatcher();
        StopWatch overallTimer = new StopWatch();
        overallMemoryWatcher.resetAndStart();
        overallTimer.resetAndStart();
        MemoryWatcher memoryWatcher = new MemoryWatcher();
        StopWatch timer = new StopWatch();
        if(classifier instanceof Loggable) {
            ((Loggable) classifier).setLogLevel(Level.ALL);
        }
        final Instances trainData = trainAndTestData[0];
        final Instances testData = trainAndTestData[1];
        timer.resetAndStart();
        memoryWatcher.resetAndStart();
        classifier.buildClassifier(trainData);
        timer.stop();
        memoryWatcher.stop();
        System.out.println();
        System.out.println("train time: " + timer.elapsedTime());
        System.out.println("train mem: " + memoryWatcher.getMaxMemoryUsage());
        System.out.println();
//        GcFinalization.awaitFullGc();
        if(classifier instanceof EnhancedAbstractClassifier) {
            if(((EnhancedAbstractClassifier) classifier).getEstimateOwnPerformance()) {
                ClassifierResults trainResults = ((EnhancedAbstractClassifier) classifier).getTrainResults();
                ResultUtils.setInfo(trainResults, classifier, trainData);
                System.out.println("train results:");
                System.out.println(trainResults.writeFullResultsToString());
                System.out.println("train acc: " + trainResults.getAcc());
                System.out.println();
            }
        }
        timer.resetAndStart();
        memoryWatcher.resetAndStart();
        ClassifierResults testResults = new ClassifierResults();
        for(Instance instance : testData) {
            addPrediction(classifier, instance, testResults, random);
        }
        memoryWatcher.stop();
        timer.stop();
        ResultUtils.setInfo(testResults, classifier, trainData);
        System.out.println("test time: " + timer.elapsedTime());
        System.out.println("test mem: " + memoryWatcher.getMaxMemoryUsage());
        System.out.println("test results:");
        System.out.println(testResults.writeFullResultsToString());
        System.out.println("test acc: " + testResults.getAcc());
        overallMemoryWatcher.stop();
        overallTimer.stop();
        System.out.println();
        System.out.println("overall time: " + overallTimer.elapsedTime());
        System.out.println("overall mem: " + overallMemoryWatcher.getMaxMemoryUsage());
    }

    /**
     * Add the prediction of several test cases to a results object.
     * @param classifier
     * @param testData
     * @param results
     * @param random
     * @throws Exception
     */
    public static void addPredictions(Classifier classifier, Instances testData, ClassifierResults results, Random random)
            throws Exception {
        for(Instance test : testData) {
            addPrediction(classifier, test, results, random);
        }
    }
    
    public static void addPredictions(TSClassifier classifier, TimeSeriesInstances testData, ClassifierResults results, Random random)
            throws Exception {
        for(TimeSeriesInstance test : testData) {
            addPrediction(classifier, test, results, random);
        }
    }

    /**
     * Add a prediction of a single test case to a results obj.
     * @param classifier
     * @param test
     * @param results
     * @param random
     * @throws Exception
     */
    public static void addPrediction(Classifier classifier, Instance test, ClassifierResults results, Random random) throws Exception {
        final double classValue = test.classValue();
        test.setClassMissing();
        long timestamp = System.nanoTime();
        final double[] distribution = classifier.distributionForInstance(test);
        long testTime = System.nanoTime() - timestamp;
        if(classifier instanceof TimedTest) {
            testTime = ((TimedTest) classifier).getTestTime();
        }
        final double prediction = Utilities.argMax(distribution, random);
        results.addPrediction(classValue, distribution, prediction, testTime, null);
        test.setClassValue(classValue);
    }

    public static void addPrediction(TSClassifier classifier, TimeSeriesInstance test, ClassifierResults results, Random random)
            throws Exception {
        final double classValue = test.getLabelIndex();
        test = new TimeSeriesInstance(test, -1);
        long timestamp = System.nanoTime();
        final double[] distribution = classifier.distributionForInstance(test);
        long testTime = System.nanoTime() - timestamp;
        if(classifier instanceof TimedTest) {
            testTime = ((TimedTest) classifier).getTestTime();
        }
        final double prediction = Utilities.argMax(distribution, random);
        results.addPrediction(classValue, distribution, prediction, testTime, null);
    }

    public static class ResultsStats{
        public double accuracy;
        public double sd;
        public double min;
        public double max;

        public ResultsStats(){
            accuracy=0;
            sd=0;
        }
        public ResultsStats(double[][] preds, int folds){
            findCVMeanSD(preds,folds);
        }
        public static ResultsStats find(double[][] preds, int folds){
            ResultsStats f=new ResultsStats();
            f.findCVMeanSD(preds,folds);
            return f;
        }
        public void findCVMeanSD(double[][] preds, int folds){
            double[] acc= new double[folds];
            //System.out.println("No. of folds = "+acc.length); // Test output
            int count=0; // Changed from 1
            int window=(preds[0].length-1)/folds;	//Put any excess in the last fold 
            window=(preds[0].length)/folds; //Changed from length-1
            //System.out.println("Window = "+window+" readings; excess goes in last fold"); // Test output
            for(int i=0;i<folds-1;i++){
                acc[i]=0;
                for(int j=0;j<window;j++){
                        if(preds[0][count]==preds[1][count])
                                acc[i]++;
                        count++;
                }
            }
            //Last fold is the remainder 
            int lastSize=preds[0].length-count;
            //System.out.println("Last fold has " + lastSize + " instances.");//Test output
            for(int j=count;j<preds[0].length;j++){
                if(preds[0][count]==preds[1][count])
                        acc[folds-1]++;
                count++;
            }
            //System.out.println("Final fold has accuracy = " + acc[folds-1]);//Test outputs
//Find mean, min and max		
            accuracy=acc[0];
            //System.out.println("First fold accuracy = "+accuracy);//Test output
            //min=1.0; // Should be acc[0];
            min=acc[0];
            max=0;
            for(int i=1;i<folds;i++){
                accuracy+=acc[i];
                //System.out.println("Sum of accuracies = " + accuracy);//Test output
                if(acc[i]<min)
                        min=acc[i];
                if(acc[i]>max)
                        max=acc[i];
            }
            //System.out.println(accuracy+"/"+(preds[0].length));//Test output
            accuracy/=preds[0].length;//Changed from length -1.
            //System.out.println(accuracy);//Test output
//Find SD
            sd=0;
            for(int i=0;i<folds-1;i++)// Changed from int i=1
            {
                sd+=(acc[i]/window-accuracy)*(acc[i]/window-accuracy);
             //System.out.println("Accuracy used here = " +acc[i]); //Test output, added braces.
            }
            sd+=(acc[folds-1]/lastSize-accuracy)*(acc[folds-1]/lastSize-accuracy);//Last fold
            sd/=folds;
            sd=Math.sqrt(sd);
        }
        public String toString(){
                return "Accuracy = "+accuracy+" SD = "+sd+" Min = "+min+" Max = "+max; //Added some spaces
        }
    }
	
//	public static void main(String[] args) // Test harness.
//	{
//		
//            double[][] preds = {
//                                {1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0},
//                                {1.0, 5.0, 3.0,4.0,5.0,6.0,3.0,8.0,9.0}
//                               };
//            
//            double[][] preds2 = {
//                                {1.0,2.0,3.0,4.0,5.0,6.0,7.0,8.0,9.0},
//                                {1.4, 5.0, 3.0,4.0,5.0,6.0,3.0,8.0,9.0}
//                               };
//            int folds = 3;
//            
//            //folds = preds[0].length;
//            
//            //System.out.println(folds);
////            
////            
//            
//            System.out.println("preds");
//            ResultsStats rs = new ResultsStats(preds,folds);
//            
//            System.out.println(rs);
//            
//            System.out.println("preds2");
//            rs = new ResultsStats(preds2,folds);
//            
//            System.out.println(rs);
//		
//	}
		

/********** Some other random methods *****************/
	//If the folds is 1, do a simple test/train split. Otherwise, do a cross validation by first combining the sets	
        public static ResultsStats[] evalClassifiers(Instances test, Instances train, int folds,Classifier[] sc) throws Exception{
                int nosClassifiers=sc.length;
                double[][]  preds;
                ResultsStats[] mean=new ResultsStats[nosClassifiers];
                int seed=100;

                for(int i=0;i<nosClassifiers;i++){
//				String[] settings=sc[i].getOptions();
//				for(String s:settings)
//					System.out.print(","+s);

//				System.out.print("\t folds ="+folds);
                        if(folds>1){	// Combine the two files
                                Instances full=new Instances(train);//Instances.mergeInstances(train, test);
                                for(int j=0;j<test.numInstances();j++)
                                        full.add(test.instance(j));
                    Random rand = new Random(seed);
//					System.out.print("\t cases ="+full.numInstances());
                    full.randomize(rand);
                                preds=crossValidation(sc[i],full,folds);
                                mean[i]= ResultsStats.find(preds,full.numInstances());
//					System.out.println("\t : "+mean[i].accuracy);
                        }
                        else{
                                sc[i].buildClassifier(train);
                                mean[i]=new ResultsStats();
                                mean[i].accuracy=accuracy(test,sc[i]);
//					System.out.println("\t : "+mean[i].accuracy);
                        }
                }
 		return mean;
	}		

       
        
	public static Instances estimateMissing(Instances data){

		ReplaceMissingValues nb = new ReplaceMissingValues();
		Instances nd=null;
		try{
			nb.setInputFormat(data);
			Instance temp;
			int n = data.numInstances();
			for(int i=0;i<n;i++)
				nb.input(data.instance(i));
			System.out.println(" Instances input");
			System.out.println(" Output format retrieved");
//			nd=Filter.useFilter(data,nb);
//			System.out.println(" Filtered? num atts = "+nd.numAttributes()+" num inst = "+nd.numInstances()+" filter = "+nb);
			if(nb.batchFinished())
				System.out.println(" batch finished ");
			nd=nb.getOutputFormat();
			for(int i=0;i<n;i++)
			{
				temp=nb.output();
//				System.out.println(temp); 
				nd.add(temp);
			}
		}catch(Exception e)
		{
			System.out.println("Error in estimateMissing  = "+e.toString());
			nd=data;
			System.exit(0);
			
		}
		return nd;
		
		}
	
/**
 * Converts all the categorical variables to binary
 * 
 * NOTE dummy created for all values, so the matrix is not full rank
 * If a regression formulation required (e.g. 6 binarys for 7 attribute values)
 * call makeBinaryFullRank
 * @param data
 */
    public static Instances makeBinary(Instances data){
        NominalToBinary nb = new NominalToBinary();

        Instances nd;
        try{
            Instance temp;
            nb.setInputFormat(data);
            int n = data.numInstances();
            for(int i=0;i<n;i++)
                    nb.input(data.instance(i));
            nd=nb.getOutputFormat();
            for(int i=0;i<n;i++)
            {
                    temp=nb.output();
//				System.out.println(temp); 
                    nd.add(temp);
            }
        }catch(Exception e)
        {
            System.out.println("Error in NominalToBinary  = "+e.toString());
            nd=data;
            System.exit(0);
        }
        return nd;
    }
/**
 * generates white noise attributes and random classes
 * @param numAtts
 * @param numCases
 * @param numClasses
 * @return 
 */        
    public static Instances generateRandomProblem(int numAtts,int numCases, int numClasses){
        String name="Random"+numAtts+"_"+numCases+"_"+numClasses;
        ArrayList<Attribute> atts=new ArrayList<>(numAtts);
        for(int i=0;i<numAtts;i++){
            Attribute at=new Attribute("Rand"+i);//Assume defaults to numeric?
            atts.add(at);
        }
//Add class value
        ArrayList<String> vals=new ArrayList<>(numClasses);
        for(int i=0;i<numClasses;i++)
                vals.add(i+"");
        atts.add(new Attribute("Response",vals));
 //Add instances
        NormalDistribution norm=new NormalDistribution(0,1);
        Random rng=new Random();
        Instances data=new Instances(name,atts,numCases);
        data.setClassIndex(numAtts);
        for(int i=0;i<numCases;i++){
            Instance in= new DenseInstance(data.numAttributes());
           
            for(int j=0;j<numAtts;j++){
                double v=norm.simulate();
                in.setValue(j, v);
            }            
            //Class value
            double classV=rng.nextInt(numClasses);
            in.setValue(numAtts,classV);
            data.add(in);
        }
        return data;
    }
        
    
    /**
     * Simple utility method to evaluate the given classifier on the ItalyPowerDemand dataset (fold 0)
     * and return the results. 
     */
    public static ClassifierResults testUtils_evalOnIPD(Classifier c) throws Exception { 
        int seed = 0;
        
        Instances[] data = DatasetLoading.sampleItalyPowerDemand(seed);
        SingleTestSetEvaluator eval = new SingleTestSetEvaluator(seed, true, true);
        
        return eval.evaluate(c, data[0], data[1]);
    }
    
    /**
     * Simple utility method to evaluate the classifier on the ItalyPowerDemand dataset (fold 0),
     * in order to get the expected accuracy in the first place. It is up to the human 
     * that the value returned is in fact 'correct' to the best of their knowledge, 
     * tests of this nature will only confirm reproducability, but the classifier
     * could e.g. be consistently WRONG.
     */
    public static double testUtils_getIPDAcc(Classifier c) throws Exception { 
        return testUtils_evalOnIPD(c).getAcc();
    }
    
    /**
     * Simple utility method to evaluate the classifier on the ItalyPowerDemand dataset (fold 0),
     * and compare the test accuracy to a given expected value (defined by prior 
     * experimentation/confirmation by human). 
     */
    public static boolean testUtils_confirmIPDReproduction(Classifier c, double expectedTestAccuracy, String dateOfExpectedAcc) throws Exception { 
        ClassifierResults res = testUtils_evalOnIPD(c);
        System.out.println("Expected accuracy generated " + dateOfExpectedAcc);
        System.out.println("Expected accuracy: " + expectedTestAccuracy + " Actual accuracy: " + res.getAcc());
        return res.getAcc() == expectedTestAccuracy;
    }

}
