package dotnet54.tscore.io;

import java.io.*;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;

import dotnet54.tscore.data.*;
import dotnet54.tscore.exceptions.NotSupportedException;
import dotnet54.util.PrintUtilities;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DurationFormatUtils;

/**
 * 
 * DEV
 * 
 * add support for skip top n rows, exclude columns, 
 * 
 * @author shifaz
 * @email ahmed.shifaz@monash.edu
 *
 */

public class TSReader {
	protected boolean hasHeader = false;
	protected int labelColumn = -1;	// only supported {0,-1} == first or last column
	protected String valSeparator = ",";
	protected String dimSeparator = ":";
	protected String commentStartChar = "#";
	protected String sectionStartChar = "@";
	protected String extension = ".ts";
	protected String missingValueChar = "?";
	protected Double missingValueReplaceChar = Double.NaN;
	
	//DEV: HACKY: some stats of the last loaded file, this  gets overwritten every time, just for a quick printing if needed
	public TSFileInfo _lastLoadedFile;
	protected int verbosity = 0;

	public class TSFileInfo{
		public String fileFullName;
		public String fileName; //without path
		public String datasetName;  //without extension
		List<String> tags = new ArrayList<>();

		public int rows;
		public int columns ;
		// column  types
		
		public String problemName;
		public boolean timeStamps;
		public boolean missing;
		public boolean univariate;
		public int dimensions;
		public boolean equalLength;
		public int seriesLength;
		public boolean classLabel;
		public String[] classLabels;

		//extra -- for file verification
		public boolean isMultiLabel;
		public int minNumLabels; //per series
		public int maxNumLabels;
		public int minLength;
		public int maxLength;
		public int minDimensions; //to verify consistency
		public int maxDimensions;
		public NonContinuousLabelEncoder labelEncoder;
		public ClassDistribution classDistribution;


		//simple stats
		public long memBeforeLoad; //in bytes
		public long memAfterLoad; //in bytes
		public long loadTime; //in nanoseconds

		public TSFileInfo() {
			labelEncoder = new NonContinuousLabelEncoder();
			classDistribution = new ClassDistribution();
		}

	}
	
	public TSReader() {}
	
	public TSReader(boolean hasHeader, int labelColumn) {
		this.hasHeader = hasHeader;
		this.labelColumn = labelColumn;
	}

	public TSFileInfo verifyFile(String fileName,
								 boolean allowMultivariate, boolean allowVariableLength,
								 boolean allowMultipleLabels, boolean checkIfNormalized,
								 boolean allowMissingValues) throws IOException, RuntimeException{

		FileReader fileReader = new FileReader(fileName);
		LineNumberReader lineNumberReader = new LineNumberReader(fileReader);
		TSReader.TSFileInfo fileInfo = new TSFileInfo();
		String line = null;
		String[] lineDimensions = null;
		String[] lineValues = null;
		boolean isDataSection = false;

		try {

			while ((line = lineNumberReader.readLine()) != null){

				if(line.startsWith(commentStartChar)){
					//skip
				} else if (line.startsWith("@problemName")) {
					fileInfo.problemName = line.split(" ")[1].trim();
				}else if (line.startsWith("@timeStamps")) {
					fileInfo.timeStamps = Boolean.parseBoolean(line.split(" ")[1].trim());
				}else if (line.startsWith("@missing")) {
					fileInfo.missing = Boolean.parseBoolean(line.split(" ")[1].trim());
					if (! allowMissingValues & fileInfo.missing){
						throw new NotSupportedException("The file "+fileName+" contains missing values, which is not supported");
					}
				}else if (line.startsWith("@univariate")) {
					fileInfo.univariate = Boolean.parseBoolean(line.split(" ")[1].trim());
					fileInfo.dimensions = 1;
					if (! allowMultivariate & ! fileInfo.univariate){
						throw new NotSupportedException("The file "+fileName+" is a multivariate dataset, multivariate support id currently disabled.");
					}
				}else if (line.startsWith("@dimensions")) {
					fileInfo.dimensions = Integer.parseInt(line.split(" ")[1].trim());
				}else if (line.startsWith("@equalLength")) {
					fileInfo.equalLength = Boolean.parseBoolean(line.split(" ")[1].trim());
					if (! allowVariableLength & ! fileInfo.equalLength){
						throw new NotSupportedException("The file "+fileName+" contains variable length time series, which is not supported");
					}
				}else if (line.startsWith("@seriesLength")) {
					fileInfo.seriesLength = Integer.parseInt(line.split(" ")[1].trim());
					fileInfo.minLength = fileInfo.seriesLength;
					fileInfo.maxLength = fileInfo.maxLength;
				}else if (line.startsWith("@classLabel")) {
					String [] tmp = line.split(" ");
					fileInfo.classLabel = Boolean.parseBoolean(tmp[1].trim());
					line = line.substring("@classLabel true ".length(),line.length());  // remove @classLabel from String
					fileInfo.classLabels = line.split(" ");
					fileInfo.classLabels = Arrays.stream(fileInfo.classLabels).map(String::trim).toArray(String[]::new);
				}else if (line.startsWith("@")) {
					isDataSection = true;
					continue; // from next line is assume that it is
				}else if (!isDataSection){
					continue; //skip or throw error...
				}else if(isDataSection){
					//if inside the data section do extra verification
					lineDimensions = line.split(dimSeparator);
					for (int dimension = 0; dimension < lineDimensions.length; dimension++) {
						if (dimension < fileInfo.dimensions){
							//data
							lineValues = lineDimensions[dimension].split(valSeparator);

							//update min and max length
							if (fileInfo.minLength == 0|| fileInfo.minLength > lineValues.length){
								fileInfo.minLength = lineValues.length;
							}
							if (fileInfo.maxLength < lineValues.length){
								fileInfo.maxLength = lineValues.length;
							}

							assert fileInfo.minLength <= fileInfo.maxLength;

							//TODO check for missing values

							//TODO verify normalization

						}else if (dimension == fileInfo.dimensions){
							//last element should be class the label,
							//if this element is not a label,
							//then there is a variable number of dimensions -- throw file inconsistency error
							lineValues = lineDimensions[dimension].split(valSeparator);

							if (lineValues.length > 1){
								//then there are comma separated values in the label,
								// throw an error or treat them as multiple labels?
								throw new NotSupportedException("Number of dimensions in the header does not match " +
										"the number of dimensions found in the  section." +
										" If this is a multilabel dataset, they are not supported currently");
							}

							String label = lineValues[0].trim();
							//TODO update label encoder and class distribution

						}else{
							throw new NotSupportedException("Inconsistencies found in the TS file. Number of dimensions" +
									"found in the  section does not match header information.");
						}
					}

				}else{
					continue; //skip
				}

			}

		}finally{
			if (fileReader != null) fileReader.close();
		}

		//this output array contains file information
		if (hasHeader) {
			//number of rows;
			fileInfo.rows = lineNumberReader.getLineNumber() == 0 ? lineNumberReader.getLineNumber(): lineNumberReader.getLineNumber() - 1;
		}else {
			//number of rows;
			fileInfo.rows =lineNumberReader.getLineNumber();
		}

//		fileInfo.columns = lineDimensions.length;  //number of columns;

		if (!allowMultivariate && fileInfo.dimensions > 1 ){
			throw new NotSupportedException("Support for multivariate datasets is not enabled");
		}

		if (!allowVariableLength && (fileInfo.minLength != fileInfo.maxLength ||
				fileInfo.minLength != fileInfo.seriesLength || fileInfo.maxLength != fileInfo.seriesLength)){
			throw new NotSupportedException("Support for variable length datasets is not enabled");
		}

		if (!allowMultipleLabels && fileInfo.minNumLabels != fileInfo.maxNumLabels ){
			throw new NotSupportedException("Dataset contains multiple labels");
		}

		if (!allowMissingValues && fileInfo.missing){
			// only a simple header information check
			throw new NotSupportedException("Dataset contains missing values");
		}

		//record file name and infer dataset name
		Path p = Paths.get(fileName);
		fileInfo.fileFullName = fileName;
		fileInfo.fileName = p.getFileName().toString();
		fileInfo.datasetName = fileInfo.problemName;
		//fileInfo.fileName.substring(0, fileInfo.fileName.indexOf('.'));

		if (fileInfo.fileName.contains("_TRAIN")){
			fileInfo.tags.add("train");
		}

		if (fileInfo.fileName.contains("_TEST")){
			fileInfo.tags.add("test");
		}

		return fileInfo;

	}

	public TSFileInfo getFileInfo(String fileName) throws IOException {
		FileReader input = new FileReader(fileName);
		LineNumberReader lineNumberReader = new LineNumberReader(input);		
		String line = null;
        String[] line_array = null;		
        TSReader.TSFileInfo file_info = new TSFileInfo();
		boolean isDataSection = false;

		try {
			boolean length_check = true;
			
	        while ((line = lineNumberReader.readLine()) != null){
	        	
	        	if (line.startsWith("@problemName")) {
	        		file_info.problemName = line.split(" ")[1].trim();
	        	}else if (line.startsWith("@timeStamps")) {
	        		file_info.timeStamps = Boolean.parseBoolean(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@missing")) {
	        		file_info.missing = Boolean.parseBoolean(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@univariate")) {
	        		file_info.univariate = Boolean.parseBoolean(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@dimensions")) {
	        		file_info.dimensions = Integer.parseInt(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@equalLength")) {
	        		file_info.equalLength = Boolean.parseBoolean(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@seriesLength")) {
	        		file_info.seriesLength = Integer.parseInt(line.split(" ")[1].trim());
	        	}else if (line.startsWith("@classLabel")) {
	        		String [] tmp = line.split(" ");
	        		file_info.classLabel = Boolean.parseBoolean(tmp[1].trim());
	        		line = line.substring("@classLabel true ".length(),line.length());  // remove @classLabel from String
					file_info.classLabels = line.split(" ");
					file_info.classLabels = Arrays.stream(file_info.classLabels).map(String::trim).toArray(String[]::new);
	        	}else if (line.startsWith("@")) {
	        		isDataSection = true;
	        		continue;
	        	}else if (!isDataSection){
	        		continue;
	        	}
	        	
//	        	if (length_check) {
//	        		length_check = false;
//	        		line_array = line.split(valSeparator);
//	        	}
//	            System.out.println("Line " + lineNumberReader.getLineNumber() + ": " + line);
	        }			
	        
		}finally{
			if (input != null) input.close();
		}
		
		//this output array contains file information
		if (hasHeader) {
			//number of rows;
			file_info.rows = lineNumberReader.getLineNumber() == 0 ? lineNumberReader.getLineNumber(): lineNumberReader.getLineNumber() - 1; 
		}else {
			//number of rows;
			file_info.rows =lineNumberReader.getLineNumber();  
		}
		
//		file_info.columns = line_array.length;  //number of columns;

		
		//record file name and infer dataset name
		Path p = Paths.get(fileName);
		file_info.fileFullName = fileName;
		file_info.fileName = p.getFileName().toString();
		file_info.datasetName = file_info.fileName.substring(0, file_info.fileName.indexOf('.'));

		
		return file_info;
	}

	public MTSDataset readFile(String fileName){
		LabelEncoder labelEncoder = new NonContinuousLabelEncoder();
		return readFile(fileName, labelEncoder);
	}

	public MTSDataset readFile(String fileName, LabelEncoder labelEncoder){
        BufferedReader br = null;
        String line = "";
        int i = 0;
        long start, end, elapsed;
        TSFileInfo file_info;
        MTSDataset dataset = null;
        long used_mem;
        String[] line_array = null;
        double[][] features;
        String label = null;
        Integer encodedLabel = null;
        File f = new File(fileName);
		boolean isDataSection = false;
//		NonContinuousLabelEncoder labelEncoder = new NonContinuousLabelEncoder();
 
        try {
        	if (verbosity > 0) {
        		System.out.print("Reading csv file ["+ f.getName() +"]:");
        	}
         	start  = System.nanoTime();

//        	file_info = verifyFile(fileName, true, false,
//					false, false);
			// allowing variable length and missing data loading here
			file_info = verifyFile(fileName, true, false,
					false, false, false);

        	dataset = new MTSDataset(file_info.rows);
			dataset.setIsVariableLength(! file_info.equalLength);

            br = new BufferedReader(new FileReader(fileName));

            //set up label encoder
			//NOTE refit should not clear the existing list
			//TODO handle labels not in training  but present in test
//			if (! labelEncoder.isFitted()){
				labelEncoder.fit(file_info.classLabels); 	//assumes that the header has correct info, does not read all labels line by line
//			}
			dataset.setLabelEncoder(labelEncoder);

            while ((line = br.readLine()) != null) {
            	
	        	if (line.startsWith("@")) {
	        		isDataSection = true;
	        		continue; //skip the tag line starting with @ - start reading actual  from the next line
	        	}else if (!isDataSection){
	        		continue; //skip non- rows which was not read earlier
	        	}
            	
	        	//TODO check need?
            	if (hasHeader && i == 0) {
            		i++;
            		continue; //skip header row
            	}
            	
                // use comma as separator
                String[] dimensions = line.split(dimSeparator);
                features = new double [file_info.dimensions][];
                boolean variableLengthFound = false;
                int previousLength = 0;
                int currentLength;

				for (int j = 0; j < dimensions.length; j++) {
					line_array = dimensions[j].split(valSeparator);
					currentLength = line_array.length;

					if (j==0){
						previousLength = currentLength;
					}

					if (j != dimensions.length-1 && currentLength != previousLength){		//skip label
						previousLength = currentLength;
						variableLengthFound = true;
					}

					if (j == dimensions.length - 1){		//last dimension is the class label
						label = line_array[0].trim();
					}else{
						features[j] = new double[line_array.length];
						for (int k = 0; k < line_array.length; k++){
							if (line_array[k].equals(missingValueChar)){
								if (!dataset.hasMissingValues()){
									dataset.setHasMissing(true);
									file_info.missing = true;
								}
								features[j][k] = missingValueReplaceChar;
							}else{
								features[j][k] = Double.parseDouble(line_array[k]);
							}
						}
					}
				}

				TimeSeries series = new MTimeSeries(features, dataset.getLabelEncoder().getLabel(label));
				dataset.setIsVariableLength(variableLengthFound);
                dataset.add(series);
                
                i++;
            }
            
            
            end = System.nanoTime();
            elapsed = end - start;
    		String time_duration = DurationFormatUtils.formatDuration((long) (elapsed/1e6), "H:m:s.SSS");
        	if (verbosity > 0) {
        		System.out.println("finished in " + time_duration);
        	}

        	dataset.setName(file_info.datasetName);
        	dataset.setTags(StringUtils.join(file_info.tags, ','));
        	_lastLoadedFile = file_info; //hacky, not thread-safe
        	
        } catch (FileNotFoundException e) {
//            e.printStackTrace();
        	PrintUtilities.abort(e);
        } catch (IOException e) {
//            e.printStackTrace();
        	PrintUtilities.abort(e);
        }catch (Exception e) {
            e.printStackTrace();
		} finally {
            if (br != null) {
                try {
                    br.close();
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        }
        
        return dataset;
	}

	public static void writeFile(Dataset dataset, String fileName, boolean overwrite){
		BufferedWriter bw = null;
		try {
			File file = new File(fileName);

			if (overwrite == false && file.exists()){
				return;
			}

			FileWriter fw = new FileWriter(file);
			bw = new BufferedWriter(fw);

			//write header
			bw.write("# This .ts file was saved from TSReader.java class in TS-CHIEF package. @dotnet54 \n");
			bw.write( "# classLabel contaisn encodedLabels, not actual labels" + dataset.getName() + ": " +dataset.getTags() + "\n");

			bw.write("\n");
			bw.write("@problemName " + dataset.getName() + "\n");
			bw.write("@timeStamps false\n");
			bw.write("@missing false\n");
			bw.write("@univariate " + (dataset.dimensions() == 1 ? "true\n": "false\n"));
			bw.write("@dimensions " + dataset.dimensions() + "\n");
			bw.write("@equalLength " + (dataset.isVariableLength() ? "false\n": "true\n"));
			bw.write("@seriesLength " + dataset.length() + "\n");
			// TODO fix classList -> getUniqueClasses
			String classList = String.join(" ",
					Arrays.stream(dataset.getUniqueClasses()).mapToObj(String::valueOf).toArray(String[]::new));
			bw.write("@classLabel true " + classList + "\n");

			bw.write("\n@\n");

			StringBuilder sb = new StringBuilder();
			int datasetSize = dataset.size();
			for (int i = 0; i < datasetSize; i++) {
				TimeSeries series = dataset.getSeries(i);
				double[][] data = series.data();

				sb.setLength(0);
				for (int j = 0; j < data.length; j++) {
					for (int k = 0; k < data[j].length - 1; k++) { // -1 to prevent adding final "," before ":"
						sb.append(data[j][k]);
						sb.append(",");
					}
					// add last item
					if (data[j].length > 0){
						sb.append(data[j][data[j].length-1]);
					}
					sb.append(":");
				}
				sb.append(series.label());
				sb.append("\n");

				bw.write(sb.toString());
			}

		} catch (IOException ioe) {
			ioe.printStackTrace();
		} finally{
			try{
				if(bw!=null)
					bw.close();
			}catch(Exception ex){
				ex.printStackTrace();
			}
		}
	}

	public static void main(String[] args) {
		try {

			System.out.println("Testing MTSDataset");
			String datasetName = "BasicMotions";
			String archiveName = "Multivariate2018_ts";
			String trainFile = "E://" + archiveName + "/" + datasetName + "/" + datasetName + "_TRAIN.ts";
			String testFile = "E://" + archiveName + "/" + datasetName + "/" + datasetName + "_TEST.ts";

			TSReader reader = new TSReader();
			MTSDataset train = reader.readFile(trainFile);

		} catch (Exception e) {
			System.out.println(e);
		}
	}


}
