package survey;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.TreeMap;

import clickStream.Attributes;
import clickStream.LectureMapping;

public class GettingToKnowYouMinerFall13 {
	private String filename;
	private final int COMPFEEDBACKLEN_2013FALL = 59;
	private final int MATLABQ1 = 51;
	private final int MATLABQ2 = 52;
	private final int MATLABQ3 = 53;
	private final int TITLE_LEN = 311;
	//private final int UNCOMPFEEDBACKLEN_2013FALL = 48 
	private final int STUDENT_MODE = 1;
	private final int NONSTUDENT_MODE = 2;
	private final int MALE_MODE = 3;
	private final int FEMALE_MODE = 4;
	private final int EUROPE_MODE = 5;
	private final int N_AMERICAN_MODE = 6;
	private final int OTHER_AREA_MODE = 7;
	private final int INDIA_MODE = 8;
	private final int PROF_MODE = 9;
	private final int ACADEMIA_MODE = 10;
	private final int AGE_UNDER30_MODE = 11;
	private final int AGE_ABOVE30_MODE = 12;
	
	private ArrayList<String> compFeedbackLines;
	private ArrayList<String []> compFeedbackList;
	//private ArrayList<String []> nCompFeedbackList;
	//private HashMap<String, String []> compFeedbackMap;
	private HashSet<String> users;
	private HashSet<String> studentsSubset;
	private HashSet<String> nonStudentsSubset;
	private HashSet<String> euroLearnersSubset;
	private HashSet<String> namerLearnersSubset;
	private HashSet<String> indianLearnersSubset;
	//private HashSet<String> otherAreaLearnersSubset;
	private HashSet<String> chineseLearnersSubset;
	private HashSet<String> maleLearnersSubset;
	private HashSet<String> femaleLearnersSubset;
	private HashSet<String> profLearnersSubset;
	private HashSet<String> academiaLearnersSubset;
	private HashSet<String> ageUnder30LearnersSubset;
	private HashSet<String> ageAbove30LearnersSubset;
	
	private String [] titles;
	
	public String dirName;
	private LectureMapping lectureMapping;
	private String userName = "taolee";
	private String password = "123456";
	
    private long day1HWId = 0;
	private long day2HWId = 0;
	private long day3HWId = 0;
	private long day4HWId = 0;
	private long day5HWId = 0;
	private long day6HWId = 0;
	private long day7HWId = 0;
	private long day8HWId = 0;
	private long day9HWId = 0;
	private long day10HWId = 0;
	private long day11HWId = 0;
	private long day12HWId = 0;
	private long day13HWId = 0;
	private long day14HWId = 0;
	private long day15HWId = 0;
	private long day16HWId = 0;
	private long day17HWId = 0;
	private long day18HWId = 0;
	
	private HashMap<String, FinalGrade> userFinalGradeMap;
	private HashMap<String, HashMap<Long, Double>> userQuizGradeMap;
	private HashMap<String, Long> idMapping;
	private HashMap<String, Attributes> studentAttributeMap;
	
	private HashMap<String, String> anonToUserMapping; // quiz and final grades
	private HashMap<String, String> forumToUserMapping; // forum 
	private HashMap<String, String> sessionToUserMapping; // survey ... but mapping for clickstream is missing
	
	private class FinalGrade {
		public double normal_grade;
        //public double distinction_grade;
        //public String achievement_level;
	};
	
	private class Pair {
		public double signal1;
	    public double signal2;
	}
	
	int occ_student = 0;
	int occ_industry = 0;
	int occ_research = 0;
	int occ_academia = 0;
	int occ_student_idx = 0;
	int occ_industry_idx = 0;
	int occ_research_idx = 0;
	int occ_academia_idx = 0;
	// Why did you enroll in this class?
	int mot_prof = 0;
	int mot_academia = 0;
	int mot_personal = 0;
	int mot_other = 0;
	HashSet<Integer> mot_prof_idx = new HashSet<Integer>();
	HashSet<Integer> mot_academia_idx = new HashSet<Integer>();
	HashSet<Integer> mot_personal_idx = new HashSet<Integer>();
	// How many hours per week do you anticipate spending on this class?
	
	// What is your current country of residence?
	int geo_n_america = 0;
	int geo_europe = 0;
	int geo_india = 0;
	int geo_s_america = 0;
	int geo_africa = 0;
	int geo_china = 0;
	int geo_asia = 0;
	int geo_australia = 0;
	HashSet<Integer> geo_n_america_idx = new HashSet<Integer>();
	HashSet<Integer> geo_europe_idx = new HashSet<Integer>();
	int geo_india_idx = 0;
	HashSet<Integer> geo_s_america_idx = new HashSet<Integer>();
	HashSet<Integer> geo_africa_idx = new HashSet<Integer>();
	HashSet<Integer> geo_asia_idx = new HashSet<Integer>();
	HashSet<Integer> geo_australia_idx = new HashSet<Integer>();
	int geo_china_idx = 0;
	// How old are you?
	int age_20 = 0;
	int age_20_30 = 0;
	int age_30_40 = 0;
	int age_40_50 = 0;
	int age_50_60 = 0;
	int age_60 = 0;
	HashSet<Integer> age_20_idx = new HashSet<Integer>();
	HashSet<Integer> age_20_30_idx = new HashSet<Integer>();
	HashSet<Integer> age_30_40_idx = new HashSet<Integer>();
	HashSet<Integer> age_40_50_idx = new HashSet<Integer>();
	HashSet<Integer> age_50_60_idx = new HashSet<Integer>();
	HashSet<Integer> age_60_idx = new HashSet<Integer>();
	// What is your gender?
	int gender_male = 0;
	int gender_female = 0;
	int gender_male_idx = 0;
	int gender_female_idx = 0;
	// How would you rate your fluency in English?
	
	// What is your education level and what kind of degree do you hold?
	int edu_numHighSchool = 0;
	int edu_numBS = 0;
	int edu_numCollege = 0;
	int edu_numMS = 0;
	int edu_numPHD = 0;
	int edu_numHighSchool_idx = 0;
	int edu_numCollege_idx = 0;
	int edu_numBS_idx = 0;
	int edu_numMS_idx = 0;
	int edu_numPHD_idx = 0;
	
	// get hash mapping of user names (ridiculous different usernames in differnt data dimensions) ;(
	private void getHashMapping(Connection conn) {
		anonToUserMapping = new HashMap<String, String>();
		forumToUserMapping = new HashMap<String, String>();
		sessionToUserMapping = new HashMap<String, String>();
		Statement stmt = null;
		// get id mapping
		String query =
				"select user_id, anon_user_id, forum_user_id, session_user_id " +
				"from " + "dspcourseraspring2013_forum" + "." + "hash_mapping";
		try {
			stmt = conn.createStatement();
			ResultSet rs = stmt.executeQuery(query);
			while (rs.next()) {
				String user_id = rs.getString(1);
				String anon_user_id = rs.getString(2);
				String forum_user_id = rs.getString(3);
				String session_user_id = rs.getString(4);
				anonToUserMapping.put(anon_user_id, user_id);
				forumToUserMapping.put(forum_user_id, user_id);
                //sessionToUserMapping.put(session_user_id, user_id);
                sessionToUserMapping.put(user_id, session_user_id);
                
 			}
		} catch (SQLException e ) {

		} finally {
			if (stmt != null) { 
				try {
					stmt.close();
				} catch (SQLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} 
			}
		}
	}
	
	// 312 data fields that contain all the background learner information:
	// spr '13:  ~ 10,000 learners submitted the initial survey
	// fall'13:    
	
	public void getDBData() {
		String url = "jdbc:mysql://localhost:3306/";
		String driver = "com.mysql.jdbc.Driver";
		//
		try {
			Class.forName(driver).newInstance();
			Connection conn = DriverManager.getConnection(url,userName, password);
			// dump general tables
			getGrades(conn);
			getHashMapping(conn);
			conn.close();
		} catch (Exception e) {
			e.printStackTrace();
		}
	} 
	
	public void getGrades(Connection conn) {
		Statement stmt = null;
		userQuizGradeMap = new HashMap<String, HashMap<Long, Double>>();
		userFinalGradeMap = new HashMap<String, FinalGrade>();
		idMapping = new HashMap<String, Long>(); 

		// get id mapping
		String query =
				"select id, parent_id, title " +
				"from " + "dspcourserafall2013" + "." + "quiz_metadata";
		try {
			stmt = conn.createStatement();
			ResultSet rs = stmt.executeQuery(query);
			while (rs.next()) {
				// FIXME: remove the redundant variables later
				String id = rs.getString(1);
				long parent_id = Long.parseLong(rs.getString(2));
                long item_id = Long.parseLong(id);
                String title = rs.getString(3); 
                if (title != null) {
                    idMapping.put(title, item_id);
                    if (parent_id == -1) {
                    	if (title.equals("Homework for Day 1")) {
                    		day1HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 2")) {
                    		day2HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 3")) {
                    		day3HWId = item_id;
                    	}
                    	else if (title.contains("Homework for Day 4")) { // "Homework for Day 4 "
                    		day4HWId = item_id;
                    	}
                    	else if (title.contains("Homework for Day 5")) { // 
                    		day5HWId = item_id;
                    	}
                    	else if (title.contains("Homework for Day 6")) { // 
                    		day6HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 7")) {
                    		day7HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 8")) {
                    		day8HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 9")) {
                    		day9HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 10")) {
                    		day10HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 11")) {
                    		day11HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 12")) {
                    		day12HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 13")) {
                    		day13HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 14")) {
                    		day14HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 15")) {
                    		day15HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 16")) {
                    		day16HWId = item_id;
                    	}
                    	else if (title.equals("Homework for Day 17")) {
                    		day17HWId = item_id;
                    	}
                    }
                }
 			}
		} catch (SQLException e ) {

		} finally {
			if (stmt != null) { 
				try {
					stmt.close();
				} catch (SQLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} 
			}
		}
		// get quiz grades
		query =
				"select item_id, session_user_id, submission_number, raw_score " +
				"from " + "dspcourserafall2013" + "." + "quiz_submission_metadata";
		try {
			stmt = conn.createStatement();
			ResultSet rs = stmt.executeQuery(query);
			while (rs.next()) {
				// FIXME: remove the redundant variables later
				Long item_id = Long.parseLong(rs.getString(1));
				String session_user_id = rs.getString(2);
                long submission_number = Long.parseLong(rs.getString(3));
                String scoreStr = rs.getString(4); // consider null
                double raw_score = 0.0;
                /*
                int t = 0;
                if (item_id == day1HWId) {
                	t = 1;
                }
                else if (item_id == day2HWId) {
                	t = 1;
                }
                else if (item_id == day3HWId) {
                	t = 1;
                }
                else if (item_id == day4HWId) {
                	t = 1;
                }
                else if (item_id == day5HWId) {
                	t = 1;
                }
                else if (item_id == day6HWId) {
                	t = 1;
                }
                else if (item_id == day7HWId) {
                	t = 1;
                }
                else if (item_id == day8HWId) {
                	t = 1;
                }
                else if (item_id == day9HWId) {
                	t = 1;
                }
                else if (item_id == day10HWId) {
                	t = 1;
                }
                else if (item_id == day11HWId) {
                	t = 1;
                }
                else if (item_id == day12HWId) {
                	t = 1;
                }
                else if (item_id == day13HWId) {
                	t = 1;
                }
                else if (item_id == day14HWId) {
                	t = 1;
                }
                else if (item_id == day15HWId) {
                	t = 1;
                }
                else if (item_id == day16HWId) {
                	t = 1;
                }
                else if (item_id == day17HWId) {
                	t = 1;
                }
                */
                if (scoreStr != null)
                    raw_score = Double.parseDouble(scoreStr);
                HashMap<Long, Double> quizGradeMap = userQuizGradeMap.get(session_user_id);
                if (quizGradeMap != null) {
                	//quizGradeMap.put(submission_number, raw_score);
                	quizGradeMap.put(item_id, raw_score);
                }
                else {
                	quizGradeMap = new HashMap<Long, Double>();
                	quizGradeMap.put(item_id, raw_score);
                	userQuizGradeMap.put(session_user_id, quizGradeMap);
                }
 			}
		} catch (SQLException e ) {

		} finally {
			if (stmt != null) { 
				try {
					stmt.close();
				} catch (SQLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} 
			}
		}
		// get final grades
		query =
				"select session_user_id, normal_grade, distinction_grade, achievement_level " +
				"from " + "dspcourserafall2013" + "." + "course_grades";
		try {
			stmt = conn.createStatement();
			ResultSet rs = stmt.executeQuery(query);
			while (rs.next()) {
				String anon_user_id = rs.getString(1);
				String normalGrade = rs.getString(2);
				double normal_grade = 0.0;
				if (normalGrade != null)
                    normal_grade = Double.parseDouble(normalGrade);
				String distinctionGrade = rs.getString(3);
                double distinction_grade = 0.0;
                if (distinctionGrade != null)
                	distinction_grade = Double.parseDouble(distinctionGrade);
                String achievement_level = rs.getString(4);
                if (achievement_level ==  null)
                	achievement_level = "";
                FinalGrade finalGrade = new FinalGrade();
                finalGrade.normal_grade = normal_grade;
                userFinalGradeMap.put(anon_user_id, finalGrade);
			}
		} catch (SQLException e ) {

		} finally {
			if (stmt != null) { 
				try {
					stmt.close();
				} catch (SQLException e) {
					// TODO Auto-generated catch block
					e.printStackTrace();
				} 
			}
		}
	}
	
	// get individual grades of the homeworks
	public double getHWGrades(String username, String hwTitle) {
		Double grade = 0.0;
		HashMap<Long, Double> quizGrades = userQuizGradeMap.get(username);
		if (quizGrades == null)
			return 0.0;
		if (hwTitle.equals("Day-1-hw")) {
			grade = quizGrades.get(day1HWId);
		}
		else if (hwTitle.equals("Day-2-hw")) {
			grade = quizGrades.get(day2HWId);
		}
		else if (hwTitle.equals("Day-3-hw")) {
			grade = quizGrades.get(day3HWId);
		}
		else if (hwTitle.equals("Day-4-hw")) {
			grade = quizGrades.get(day4HWId);
		}
		else if (hwTitle.equals("Day-5-hw")) {
			grade = quizGrades.get(day5HWId);
		}
		else if (hwTitle.equals("Day-6-hw")) {
			grade = quizGrades.get(day6HWId);
		}
		else if (hwTitle.equals("Day-7-hw")) {
			grade = quizGrades.get(day7HWId);
		}
		else if (hwTitle.equals("Day-8-hw")) {
			grade = quizGrades.get(day8HWId);
		}
		else if (hwTitle.equals("Day-9-hw")) {
			grade = quizGrades.get(day9HWId);
		}
		else if (hwTitle.equals("Day-10-hw")) {
			grade = quizGrades.get(day10HWId);
		}
		else if (hwTitle.equals("Day-11-hw")) {
			grade = quizGrades.get(day11HWId);
		}
		else if (hwTitle.equals("Day-12-hw")) {
			grade = quizGrades.get(day12HWId);
		}
		else if (hwTitle.equals("Day-13-hw")) {
			grade = quizGrades.get(day13HWId);
		}
		else if (hwTitle.equals("Day-14-hw")) {
			grade = quizGrades.get(day14HWId);
		}
		else if (hwTitle.equals("Day-15-hw")) {
			grade = quizGrades.get(day15HWId);
		}
		else if (hwTitle.equals("Day-16-hw")) {
			grade = quizGrades.get(day16HWId);
		}
		else if (hwTitle.equals("Day-17-hw")) {
			grade = quizGrades.get(day17HWId);
		}
		else if (hwTitle.equals("Day-18-hw")) {
			grade = quizGrades.get(day18HWId);
		}
		// no submit is equal to grade 0.0
		if (grade == null)
			grade = 0.0;
		assert (grade != null);
	    return grade;
	}
	
	private void writeCCDF(int mode) throws IOException {
		// 
	    TreeMap<Double, Long> day1HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day2HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day3HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day4HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day5HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day6HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day7HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day8HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day9HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day10HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day11HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day12HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day13HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day14HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day15HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day16HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> day17HWGradesHist = new TreeMap<Double, Long>();
	    TreeMap<Double, Long> finalGradesHist = new TreeMap<Double, Long>();
	    HashSet<String> users_subset = new HashSet<String>();
	    String outputFileName = "final-grades-fall13-histogram.txt";
	    
	    /*
		private final int MALE_MODE = 3;
		private final int FEMALE_MODE = 4;
		private final int EUROPE_MODE = 5;
		private final int N_AMERICAN_MODE = 6;
		private final int OTHER_AREA_MODE = 7;
		private final int INDIA_MODE = 8;
		private final int PROF_MODE = 9;
		private final int ACADEMIA_MODE = 10;
		private final int AGE_UNDER30_MODE = 11;
		private final int AGE_ABOVE30_MODE = 12;
		private HashSet<String> euroLearnersSubset;
	private HashSet<String> namerLearnersSubset;
	private HashSet<String> indianLearnersSubset;
	//private HashSet<String> otherAreaLearnersSubset;
	private HashSet<String> chineseLearnersSubset;
	private HashSet<String> maleLearnersSubset;
	private HashSet<String> femaleLearnersSubset;
	private HashSet<String> profLearnersSubset;
	private HashSet<String> academiaLearnersSubset;
	private HashSet<String> ageUnder30LearnersSubset;
	private HashSet<String> ageAbove30LearnersSubset;
	
		*/
    	if (mode == STUDENT_MODE) {
    		users_subset.addAll(studentsSubset);
    		outputFileName = "student-only-final-grades-fall13-histogram.csv";
    	}
    	else if (mode == NONSTUDENT_MODE) {
    		users_subset.addAll(nonStudentsSubset);
    		outputFileName = "nonstudent-only-final-grades-fall13-histogram.csv";
    	}
    	else if (mode == MALE_MODE) {
    		users_subset.addAll(maleLearnersSubset);
    		outputFileName = "male-only-final-grades-fall13-histogram.csv";
    	}
    	else if (mode == FEMALE_MODE) {
    		users_subset.addAll(femaleLearnersSubset);
    		outputFileName = "female-only-final-grades-fall13-histogram.csv";
    	}
	    else if (mode == EUROPE_MODE) {
		    users_subset.addAll(euroLearnersSubset);
		    outputFileName = "europe-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == N_AMERICAN_MODE) {
		    users_subset.addAll(namerLearnersSubset);
		    outputFileName = "n_american-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == INDIA_MODE) {
	    	users_subset.addAll(indianLearnersSubset);
		    outputFileName = "indian-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == PROF_MODE) {
		    users_subset.addAll(profLearnersSubset);
		    outputFileName = "prof-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == ACADEMIA_MODE) {
		    users_subset.addAll(academiaLearnersSubset);
		    outputFileName = "academia-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == AGE_UNDER30_MODE) {
	    	users_subset.addAll(ageUnder30LearnersSubset);
		    outputFileName = "ageunder30-only-final-grades-fall13-histogram.csv";
	    }
	    else if (mode == AGE_ABOVE30_MODE) {
	    	users_subset.addAll(ageAbove30LearnersSubset);
		    outputFileName = "ageabove30-only-final-grades-fall13-histogram.csv";
	    }
    	else { // all learners
    		users_subset.addAll(userFinalGradeMap.keySet());
    	}
    	
		// build histograms
	    for (String username: userFinalGradeMap.keySet()) {
            // add the user group under investigation

	    	// stupid mapping from anon_user_id to session_user_id
	    	//String session_user_name = sessionToUserMapping.get(anonToUserMapping.get(username));
	    	if (users_subset.contains(username)) {
/*
	    		// Day-1 HW grades
	    		double grade = getHWGrades(username, "Day-1-hw");
	    		Long l = day1HWGradesHist.get(grade);
	    		if (l != null) {
	    			day1HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day1HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-2 HW grades
	    		grade = getHWGrades(username, "Day-2-hw");
	    		l = day2HWGradesHist.get(grade);
	    		if (l != null) {
	    			day2HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day2HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-3 HW grades
	    		grade = getHWGrades(username, "Day-3-hw");
	    		l = day3HWGradesHist.get(grade);
	    		if (l != null) {
	    			day3HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day3HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-4 HW grades
	    		grade = getHWGrades(username, "Day-4-hw");
	    		l = day4HWGradesHist.get(grade);
	    		if (l != null) {
	    			day4HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day4HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-5 HW grades
	    		grade = getHWGrades(username, "Day-5-hw");
	    		l = day5HWGradesHist.get(grade);
	    		if (l != null) {
	    			day5HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day5HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-6 HW grades
	    		grade = getHWGrades(username, "Day-6-hw");
	    		l = day6HWGradesHist.get(grade);
	    		if (l != null) {
	    			day6HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day6HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-7 HW grades
	    		grade = getHWGrades(username, "Day-7-hw");
	    		l = day7HWGradesHist.get(grade);
	    		if (l != null) {
	    			day7HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day7HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-8 HW grades
	    		grade = getHWGrades(username, "Day-8-hw");
	    		l = day8HWGradesHist.get(grade);
	    		if (l != null) {
	    			day8HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day8HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-9 HW grades
	    		grade = getHWGrades(username, "Day-9-hw");
	    		l = day9HWGradesHist.get(grade);
	    		if (l != null) {
	    			day9HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day9HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-10 HW grades
	    		grade = getHWGrades(username, "Day-10-hw");
	    		l = day10HWGradesHist.get(grade);
	    		if (l != null) {
	    			day10HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day10HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-11 HW grades
	    		grade = getHWGrades(username, "Day-11-hw");
	    		l = day11HWGradesHist.get(grade);
	    		if (l != null) {
	    			day11HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day11HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-12 HW grades
	    		grade = getHWGrades(username, "Day-12-hw");
	    		l = day12HWGradesHist.get(grade);
	    		if (l != null) {
	    			day12HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day12HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-13 HW grades
	    		grade = getHWGrades(username, "Day-13-hw");
	    		l = day13HWGradesHist.get(grade);
	    		if (l != null) {
	    			day13HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day13HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-14 HW grades
	    		grade = getHWGrades(username, "Day-14-hw");
	    		l = day14HWGradesHist.get(grade);
	    		if (l != null) {
	    			day14HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day14HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-15 HW grades
	    		grade = getHWGrades(username, "Day-15-hw");
	    		l = day15HWGradesHist.get(grade);
	    		if (l != null) {
	    			day15HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day15HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-16 HW grades
	    		grade = getHWGrades(username, "Day-16-hw");
	    		l = day16HWGradesHist.get(grade);
	    		if (l != null) {
	    			day16HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day16HWGradesHist.put(grade, 1L);
	    		}

	    		// Day-17 HW grades
	    		grade = getHWGrades(username, "Day-17-hw");
	    		l = day17HWGradesHist.get(grade);
	    		if (l != null) {
	    			day17HWGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			day17HWGradesHist.put(grade, 1L);
	    		}
*/
	    		// final grades
	    		double grade = userFinalGradeMap.get(username).normal_grade;
	    		Long l = finalGradesHist.get(grade);
	    		if (l != null) {
	    			finalGradesHist.put(grade, l+1);
	    		}
	    		else {
	    			finalGradesHist.put(grade, 1L);
	    		}
	    	}
	    } 
		
		// write histograms for further visualization
		
		/*
		// Day1 HW grades
		BufferedWriter fileWriter = new BufferedWriter(new FileWriter("day1HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY1HWGRADES SIZE\n");
		for (Double grades: day1HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day1HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day2 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day2HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY2HWGRADES SIZE\n");
		for (Double grades: day2HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day2HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day3 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day3HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY3HWGRADES SIZE\n");
		for (Double grades: day3HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day3HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day4 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day4HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY4HWGRADES SIZE\n");
		for (Double grades: day4HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day4HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day5 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day5HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY5HWGRADES SIZE\n");
		for (Double grades: day5HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day5HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day6 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day6HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY6HWGRADES SIZE\n");
		for (Double grades: day6HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day6HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day7 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day7HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY7HWGRADES SIZE\n");
		for (Double grades: day7HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day7HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day8 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day8HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY8HWGRADES SIZE\n");
		for (Double grades: day8HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day8HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day9 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day9HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY9HWGRADES SIZE\n");
		for (Double grades: day9HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day9HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day10 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day10HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY10HWGRADES SIZE\n");
		for (Double grades: day10HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day10HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day11 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day11HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY11HWGRADES SIZE\n");
		for (Double grades: day11HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day11HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day12 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day12HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY12HWGRADES SIZE\n");
		for (Double grades: day12HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day12HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day13 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day13HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY13HWGRADES SIZE\n");
		for (Double grades: day13HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day13HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day14 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day14HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY14HWGRADES SIZE\n");
		for (Double grades: day14HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day14HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day15 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day15HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY15HWGRADES SIZE\n");
		for (Double grades: day15HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day15HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day16 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day16HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY16HWGRADES SIZE\n");
		for (Double grades: day16HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day16HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		
		// Day17 HW grades
		fileWriter = new BufferedWriter(new FileWriter("day17HW-grades-spr13-histogram.txt"));
		fileWriter.write("VALUE-DAY17HWGRADES SIZE\n");
		for (Double grades: day17HWGradesHist.keySet()) {
			fileWriter.write(grades + " " + day17HWGradesHist.get(grades) + "\n");
		}
		fileWriter.flush();
		fileWriter.close();
		*/
	    
	    // Can we compute CCDF directly?
	    // Include normalization, linear-linear, linear-log, log-linear, log-log plots => we need to make this professional!
	    // 
	    
	    
		// final grades
	    ArrayList<Long> ccdf = new ArrayList<Long>();
		BufferedWriter fileWriter = new BufferedWriter(new FileWriter(outputFileName));
		fileWriter.write("VALUE-FINALGRADES, SIZE\n");
		for (Double grades: finalGradesHist.keySet()) {
			fileWriter.write(grades + ", " + finalGradesHist.get(grades) + "\n");
			ccdf.add(finalGradesHist.get(grades));
		}
		fileWriter.flush();
		fileWriter.close();
		
		// compute ccdf
		//for (double grades)
		
		// find min, max and steps for each histogram for uniform quantization
	}
	
	// use the right data structure
	public void computeCCDF() {
		
	}
	
	// Is this really necessary?
	public void computeLinearLogPlots() {
		
	}
	
	public void setFilename(String filename) {
		this.filename = filename;
	}
	
	public void setUsers(HashSet<String> users) {
	    this.users = new HashSet<String>(users);	
	}
	
	public HashMap<String, String []> getUsers() {
	    HashMap<String, String []> m = new HashMap<String, String []>();
	    for (String [] strs: compFeedbackList) {
	    	if (users.contains(strs[0])) {
	    		m.put(strs[0], strs);
	    	}
	    }
	    // Funny that the users have no correspondence ;(
	    System.out.println("Common users in the forum and survey activities: " + m.size());
	    return m;
	}
	
	private void parseSurveyLine(String line) {
		compFeedbackLines.add(line); // to be processed in the 2nd round
	}

	private boolean isHexNumber (String cadena) {
		try {
			Long.parseLong(cadena, 16);
			return true;
		}
		catch (NumberFormatException ex) {
			// Error handling code...
			return false;
		}
	}
	
	private boolean isHexadecimal(String text) {
	    text = text.trim();

	    char[] hexDigits = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
	            'a', 'b', 'c', 'd', 'e', 'f', 'A', 'B', 'C', 'D', 'E', 'F' };

	    int hexDigitsCount = 0;

	    for (char symbol : text.toCharArray()) {
	        for (char hexDigit : hexDigits) {
	            if (symbol == hexDigit) {
	                hexDigitsCount++;
	                break;
	            }
	        }
	    }

	    return hexDigitsCount == text.length() ? true : false;
	}
	
	private void regroupFeedback() {
		String [] strArray;
		String cstr = null;
		boolean combine = false;
		for (String line: compFeedbackLines) {
			String [] strs = line.split(",");
			// 
			/*if (strs[0].equals("session_user_id")) {
				int tt = 0;
				int yy = tt;
			}*/
			if (strs[0].equals("session_user_id") && strs[4].contains("Student") && strs.length >= COMPFEEDBACKLEN_2013FALL-10) {
				titles = splitDataField(line);
				System.out.println(titles.length);
			}
			// check first data field: hex ID
			if (isHexadecimal(strs[0]) && strs.length >= COMPFEEDBACKLEN_2013FALL-10) { // -10 to counteract some irregularities
				if (!combine) combine = true;
				// add the last non-null cstr
				if (cstr != null) {
					// check "" pairs
					/*i = 0;
					for (char ch: cstr.toCharArray()) {
						if (ch == '"')
							i++;
					}
					if (i % 2 != 0) {
						j = i;
					}*/
					//compFeedbackList.add(cstr.split(","));
					compFeedbackList.add(splitDataField(cstr));
				}
				cstr = line;
			}
			else {
				if (cstr != null && combine)
				    cstr = cstr.concat(line);
			}
		}
		System.out.println("Number of students submitting survey = " + compFeedbackList.size());
		//checkMatlabQ();
		//countNumberOfStudents();
		//countNumberOf10To12HoursOfStudy();
		//countAvgQuizGradesOfLearnersInRange(0, 20);
	} 
	
	public void postCCDFAnalysis(int mode) {
		countAvgQuizGradesOfLearnersInRange(70, 100);
		countNumberOfLearnersInRange(0, 100, mode);
	}
	
	// 
	private void countNumberOf10To12HoursOfStudy() {
		int i = 0;
		int j = 0;
		for (String str: titles) {
			if (str.contains("10-12"))
				break;
			i++;
		}
		for (String [] strs: compFeedbackList) {
		    assert strs.length > i;
		    if (strs[i].equals("1")) {
		    	j++;
		    }
		}
		System.out.println("Number of students with study hours 10-12 = " + j);
	}
	
	//
	
	// small procedures to do easy calculation
	private void countNumberOfStudents() {
		int i = 0;
		studentsSubset.clear();
		nonStudentsSubset.clear();
		for (String str: titles) {
			if (str.contains("Student"))
				break;
			i++;
		}
		for (String [] strs: compFeedbackList) {
		    assert strs.length > i;
		    if (strs[i].equals("1")) {
		    	studentsSubset.add(strs[0]);
		    }
		    else {
		    	nonStudentsSubset.add(strs[0]);
		    }
		}
		//System.out.println("Number of students = " + j);
		//System.out.println("Number of non-students = " + k);
	}
	
	private void countGeoGroupLearners() {
		euroLearnersSubset.clear();
		namerLearnersSubset.clear();
		indianLearnersSubset.clear();
		chineseLearnersSubset.clear();
		//otherAreaLearnersSubset.clear();
		for (String [] strs: compFeedbackList) {
			for (Integer n_america_idx: geo_n_america_idx) {
				if (strs[n_america_idx].equals("1")) {
					namerLearnersSubset.add(strs[0]);
					break;
				}
			} 
			for (Integer europe_idx: geo_europe_idx) {
				if (strs[europe_idx].equals("1")) {
					euroLearnersSubset.add(strs[0]);
					break;
				}
			} 
			if (strs[geo_india_idx].equals("1")) {
				indianLearnersSubset.add(strs[0]);
			}
			if (strs[geo_china_idx].equals("1")) {
				chineseLearnersSubset.add(strs[0]);
			}
		}
	}
	
	private void countGenderGroupLearners() {
		maleLearnersSubset.clear();
		femaleLearnersSubset.clear();
		for (String [] strs: compFeedbackList) {
			if (strs[gender_male_idx].equals("1")) {
				maleLearnersSubset.add(strs[0]);
			}
			if (strs[gender_female_idx].equals("1")) {
				femaleLearnersSubset.add(strs[0]);
			}
		}
	}
	
	private void countMotGroupLearners() {
		boolean motprof = false;
		profLearnersSubset.clear();
		academiaLearnersSubset.clear();
		for (String [] strs: compFeedbackList) {
			motprof = false;
			for (Integer prof_idx: mot_prof_idx) {
				if (strs[prof_idx].equals("1")) {
					motprof = true;
					profLearnersSubset.add(strs[0]);
					break;
				}
			} 
			for (Integer academia_idx: mot_academia_idx) {
				if (strs[academia_idx].equals("1")) {
					if (!motprof)
					    academiaLearnersSubset.add(strs[0]);
					break;
				}
			}
		}
	}
	
	private void countAgeGroupLearners() {
		ageUnder30LearnersSubset.clear();
		ageAbove30LearnersSubset.clear();
		for (String [] strs: compFeedbackList) {
			for (Integer a20_idx: age_20_idx) {
				if (strs[a20_idx].equals("1")) {
					ageUnder30LearnersSubset.add(strs[0]);
					break;
				}
			}
			for (Integer a20_30_idx: age_20_30_idx) {
				if (strs[a20_30_idx].equals("1")) {
					ageUnder30LearnersSubset.add(strs[0]);
					break;
				}
			}
			for (Integer a30_40_idx: age_30_40_idx) {
				if (strs[a30_40_idx].equals("1")) {
					ageAbove30LearnersSubset.add(strs[0]);
					break;
				}
			}
			for (Integer a40_50_idx: age_40_50_idx) {
				if (strs[a40_50_idx].equals("1")) {
					ageAbove30LearnersSubset.add(strs[0]);
					break;
				}
			}
			for (Integer a50_60_idx: age_50_60_idx) {
				if (strs[a50_60_idx].equals("1")) {
					ageAbove30LearnersSubset.add(strs[0]);
					break;
				}
			}
			for (Integer a60_idx: age_60_idx) {
				if (strs[a60_idx].equals("1")) {
					ageAbove30LearnersSubset.add(strs[0]);
					break;
				}
			}
		}
	}
	
	private void countAvgQuizGradesOfLearnersInRange(double lbd_score, double ubd_score) {
	    int numLearners = 0;
	    double hw1_avg_scores = 0.0;
	    double hw2_avg_scores = 0.0;
	    double hw3_avg_scores = 0.0;
	    double hw4_avg_scores = 0.0;
	    double hw5_avg_scores = 0.0;
	    double hw6_avg_scores = 0.0;
	    double hw7_avg_scores = 0.0;
	    double hw8_avg_scores = 0.0;
	    double hw9_avg_scores = 0.0;
	    double hw10_avg_scores = 0.0;
	    double hw11_avg_scores = 0.0;
	    double hw12_avg_scores = 0.0;
	    double hw13_avg_scores = 0.0;
	    double hw14_avg_scores = 0.0;
	    double hw15_avg_scores = 0.0;
	    double hw16_avg_scores = 0.0;
	    double hw17_avg_scores = 0.0;
	    
	    for (String username: userFinalGradeMap.keySet()) { 
			double grades = userFinalGradeMap.get(username).normal_grade;
			if (grades >= lbd_score && grades <= ubd_score) {
				numLearners++;
				//
				hw1_avg_scores += getHWGrades(username, "Day-1-hw");
				hw2_avg_scores += getHWGrades(username, "Day-2-hw");
				hw3_avg_scores += getHWGrades(username, "Day-3-hw");
				hw4_avg_scores += getHWGrades(username, "Day-4-hw");
				hw5_avg_scores += getHWGrades(username, "Day-5-hw");
				hw6_avg_scores += getHWGrades(username, "Day-6-hw");
				hw7_avg_scores += getHWGrades(username, "Day-7-hw");
				hw8_avg_scores += getHWGrades(username, "Day-8-hw");
				hw9_avg_scores += getHWGrades(username, "Day-9-hw");
				hw10_avg_scores += getHWGrades(username, "Day-10-hw");
				hw11_avg_scores += getHWGrades(username, "Day-11-hw");
				hw12_avg_scores += getHWGrades(username, "Day-12-hw");
				hw13_avg_scores += getHWGrades(username, "Day-13-hw");
				hw14_avg_scores += getHWGrades(username, "Day-14-hw");
				hw15_avg_scores += getHWGrades(username, "Day-15-hw");
				hw16_avg_scores += getHWGrades(username, "Day-16-hw");
				hw17_avg_scores += getHWGrades(username, "Day-17-hw");
			}
	    }
	    //
	    hw1_avg_scores /= numLearners;
	    hw2_avg_scores /= numLearners;
	    hw3_avg_scores /= numLearners;
	    hw4_avg_scores /= numLearners;
	    hw5_avg_scores /= numLearners;
	    hw6_avg_scores /= numLearners;
	    hw7_avg_scores /= numLearners;
	    hw8_avg_scores /= numLearners;
	    hw9_avg_scores /= numLearners;
	    hw10_avg_scores /= numLearners;
	    hw11_avg_scores /= numLearners;
	    hw12_avg_scores /= numLearners;
	    hw13_avg_scores /= numLearners;
	    hw14_avg_scores /= numLearners;
	    hw15_avg_scores /= numLearners;
	    hw16_avg_scores /= numLearners;
	    hw17_avg_scores /= numLearners;
	    
	    System.out.println("HW1 avg grades = " + String.format("%.2f", hw1_avg_scores / 7.5 * 100));
	    System.out.println("HW2 avg grades = " + String.format("%.2f", hw2_avg_scores / 9 * 100));
	    System.out.println("HW3 avg grades = " + String.format("%.2f", hw3_avg_scores / 10 * 100));
	    System.out.println("HW4 avg grades = " + String.format("%.2f", hw4_avg_scores / 5.5 * 100));
	    System.out.println("HW5 avg grades = " + String.format("%.2f", hw5_avg_scores / 7 * 100));
	    System.out.println("HW6 avg grades = " + String.format("%.2f", hw6_avg_scores / 6 * 100));
	    System.out.println("HW7 avg grades = " + String.format("%.2f", hw7_avg_scores / 8 * 100));
	    System.out.println("HW8 avg grades = " + String.format("%.2f", hw8_avg_scores / 8 * 100));
	    System.out.println("HW9 avg grades = " + String.format("%.2f", hw9_avg_scores / 11 * 100));
	    //System.out.println("HW10 avg grades = " + String.format("%.2f", hw10_avg_scores / 9 * 100));
	    System.out.println("HW11 avg grades = " + String.format("%.2f", hw11_avg_scores / 9 * 100));
	    System.out.println("HW12 avg grades = " + String.format("%.2f", hw12_avg_scores / 8 * 100));
	    System.out.println("HW13 avg grades = " + String.format("%.2f", hw13_avg_scores / 9.5 * 100));
	    System.out.println("HW14 avg grades = " + String.format("%.2f", hw14_avg_scores / 9 * 100));
	    System.out.println("HW15 avg grades = " + String.format("%.2f", hw15_avg_scores / 8 * 100));
	    System.out.println("HW16 avg grades = " + String.format("%.2f", hw16_avg_scores / 5 * 100));
	    System.out.println("HW17 avg grades = " + String.format("%.2f", hw17_avg_scores / 7 * 100));
	}
	
	public void collectLearners() {
		countNumberOfStudents();
		countGeoGroupLearners();
		countGenderGroupLearners();
		countMotGroupLearners();
		countAgeGroupLearners();
	}
	
	public void buildSurveyIndex() {
		int i = 0;
		for (String str: titles) {
			// Which of the following descriptions best characterizes you?
			if (str.contains("Student (Graduate/ Undergraduate)")) {
				occ_student_idx = i;
			}
			else if (str.contains("Industry professional")) {
				occ_industry_idx = i;
			}
			else if (str.contains("Research Scientist")) {
				occ_research_idx = i;
			}
			else if (str.contains("Academic / Professor")) {
				occ_academia_idx = i;
			}
			else if (str.contains("Hobbyist")) {
				
			}
			/*else if (str.contains("None of the above")) {
				
			}*/
			// Why did you enroll in this class?
			else if (str.contains("For professional reasons.")) {
				mot_prof_idx.add(i);
			}
			else if (str.contains("I think taking this course will be fun and enjoyable.")) {
				//mot_personal_idx.add(i); 
			}
			else if (str.contains("Because I've heard about the teacher/school.")) {
				//mot_academia_idx.add(i);
			}
			else if (str.contains("Out of curiosity, it sounds interesting.")) {
				//mot_personal_idx.add(i);
			}
			else if (str.contains("This subject is relevant to my academic field of study.")) {
				mot_academia_idx.add(i);
			}
			else if (str.contains("It was advertised to me.")) {
				//mot_personal_idx.add(i);
			}
			else if (str.contains("This class teaches skills that will help my job/career.")) {
				//mot_prof_idx.add(i);
			}
			else if (str.contains("I want to earn some sort of credential that I can use to enhance my CV resume.")) {
				//mot_prof_idx.add(i);
			}
			// How many hours per week do you anticipate spending on this class?
			else if (str.contains("1-3")) {}
			else if (str.contains("4-6")) {}
			else if (str.contains("7-9")) {}
			else if (str.contains("10-12")) {}
			else if (str.contains("13-15")) {}
			else if (str.contains(">15")) {}
			// What is your current country of residence?
			// Europe
			else if (str.contains("Albania")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Andorra")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Azerbaijan")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Estonia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Georgia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Kazakhstan")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Latvia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Macedonia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Malta")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Moldova")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Albania")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Monaco")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Montenegro")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Russia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("San Marino")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Turkey")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("United Kingdom")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("France")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Germany")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Switzerland")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Spain")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Portugal")) {
				geo_europe_idx.add(i);
			} 
			else if (str.contains("Italy")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Greece")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Belgium")) {
				geo_europe_idx.add(i);	
			}
			else if (str.contains("Sweden")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Norway")) {
				geo_europe_idx.add(i);
			} 
			else if (str.contains("Finland")) {
				geo_europe_idx.add(i);
			} 
			else if (str.contains("Danmark")) {
				geo_europe_idx.add(i);
			} 
			else if (str.contains("Luxembourg")) {
				geo_europe_idx.add(i);
			} 
			else if (str.contains("Armenia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Austria")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Belarus")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Bosnia and Herzegovina")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Bulgaria")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Croatia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Cyprus")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Czech Republic")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Greenland")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Hungary")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Iceland")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Ireland")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Liechtenstein")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Macedonia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Netherlands")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Northern Cyprus")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Poland")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Romania")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Serbia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Slovakia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Slovenia")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Ukraine")) {
				geo_europe_idx.add(i);
			}
			else if (str.contains("Vatican City")) {
				geo_europe_idx.add(i);
			}
 			// North America
			/*
			 * */
			else if (str.contains("Saint Pierre and Miquelon")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Anguilla")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Antigua and Barbuda")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Aruba")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Bahamas")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Barbados")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Bonaire")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("British Virgin Islands")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Cayman Islands")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Clipperton Island")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Cuba")) {
				geo_n_america_idx.add(i);
			}
			//else if (str.contains("Curaao")) {}
			else if (str.contains("Grenada")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Guadeloupe")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Haiti")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Martinique")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Montserrat")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Navassa Island")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saba")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Kitts and Nevis")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Lucia")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Martin ")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Vincent and the Grenadines")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Kitts and Nevis")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("United States Virgin Islands")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Sint Eustatius")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Trinidad and Tobago")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Saint Kitts and Nevis")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Panama")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("United States")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Canada")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Mexico")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Dominica")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Dominican Republic")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Honduras")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Puerto Rico")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Bermuda")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Cuba")) {
				geo_n_america_idx.add(i);
			}
			else if (str.contains("Venezuela")) {
				geo_n_america_idx.add(i);
			}
			// India
			else if (str.contains("India")) {
				geo_india_idx = i;
			}
			// South America
			else if (str.contains("Argentina")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Brazil")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Chile")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Uruguay")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Bolivia")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Colombia")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Ecuador")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Falkland Islands")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("French Guiana")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Guyana")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Paraguay")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Peru")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("South Georgia and South Sandwich Islands")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Suriname")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Uruguay")) {
				geo_s_america_idx.add(i);
			}
			else if (str.contains("Venezuela")) {
				geo_s_america_idx.add(i);
			}
			// Africa
			else if (str.contains("Sudan")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Tunisia")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Uganda")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Algeria")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Angola")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Cameroon")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Central Africa Republic")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Chad")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Congo")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Ethiopia")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Gabon")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Liberia")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Madagascar")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Morocco")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Mozambique")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Namibia")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Nigeria")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Rwanda")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Sierra Leone")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("South Africa")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Swaziland")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Tanzania")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Togo")) {
				geo_africa_idx.add(i);
			}
			else if (str.contains("Uganda")) {
				geo_africa_idx.add(i);
			}
			// China
			else if (str.contains("China")) {
				geo_china_idx = i;
			}
			// asian countries
			else if (str.contains("Afghanistan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Bahrain")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Bangladesh")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Bhutan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Brunei")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Myanmar")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Cambodia")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("East Timor")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Indonesia")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Iran")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Iraq")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Israel")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Japan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Jordan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Kazakhstan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Kuwait")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Kyrgyzstan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Laos")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Lebanon")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Malaysia")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Maldives")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Mongolia")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Nepal")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("North Korea")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Oman")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Pakistan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Palestine")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Philippines")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Qatar")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Saudi Arabia")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Singapore")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Sri Lanka")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("South Korea")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Taiwan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Tajikistan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Thailand")) {
				geo_asia_idx.add(i);
			}
			/*else if (str.contains("Turkey")) {
				geo_asia_idx.add(i);
			}*/
			else if (str.contains("Turkmenistan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("United Arab Emirates")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Uzbekistan")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Vietnam")) {
				geo_asia_idx.add(i);
			}
			else if (str.contains("Yemen")) {
				geo_asia_idx.add(i);
			}
			// Australia
			else if (str.contains("Australia")) {
				geo_australia_idx.add(i);
			}
			else if (str.contains("New Zealand")) {
				geo_australia_idx.add(i);
			}
			// How old are you?
			else if (str.contains("0-15")) {
				age_20_idx.add(i);
			}
			else if (str.contains("16-18")) {
				age_20_idx.add(i);
			}
			else if (str.contains("19-22")) {
				age_20_idx.add(i);
			}
			else if (str.contains("23-25")) {
				age_20_30_idx.add(i);
			}
			else if (str.contains("26-29")) {
				age_20_30_idx.add(i);
			}
			else if (str.contains("30-35")) {
				age_30_40_idx.add(i);
			}
			else if (str.contains("36-40")) {
				age_30_40_idx.add(i);
			}
			else if (str.contains("41-50")) {
				age_40_50_idx.add(i);
			}
			else if (str.contains("51-60")) {
				age_50_60_idx.add(i);
			}
			else if (str.contains("60+")) {
				age_60_idx.add(i);
			}
			// What is your gender?
			else if (str.contains("Male")) {
				gender_male_idx = i;
			}
			else if (str.contains("Female")) {
				gender_female_idx = i;
			}
			// How would you rate your fluency in English?
			else if (str.contains("Poor")) {}
			else if (str.contains("Beginner")) {}
			else if (str.contains("Intermediate")) {}
			else if (str.contains("Fluent")) {}
			else if (str.contains("Native speaker")) {}
			// What is your education level and what kind of degree do you hold?
			/*else if (str.contains("None")) {
				
			}*/
			else if (str.contains("High school")) {
				edu_numHighSchool_idx = i;
			}
			else if (str.contains("Some College")) {
				edu_numCollege_idx = i;
			}
			else if (str.contains("Bachelor's degree")) {
				edu_numBS_idx = i;
			}
			else if (str.contains("Master's degree")) {
				edu_numMS_idx = i;
			}
			else if (str.contains("PhD (or equivalent).")) {
				edu_numPHD_idx = i;
			}
			// If applicable, what field of study was your highest degree in?
			else if (str.contains("Computer Science / Communication Systems / Computer Engineering")) {}
			else if (str.contains("Electrical / Electronic Engineering")) {}
			else if (str.contains("Life Sciences")) {}
			else if (str.contains("Physics")) {}
			else if (str.contains("Mathematics / Statistics")) {}
			else if (str.contains("Social Sciences")) {}
			else if (str.contains("Business / Politics / Marketing")) {}
			else if (str.contains("Humanities / Arts")) {}
			else if (str.contains("Medicine")) {}
			else if (str.contains("Other Engineering")) {}
			else if (str.contains("Others")) {}
			// Where do you usually access the internet?
			else if (str.contains("At home.")) {}
			else if (str.contains("At work.")) {}
			else if (str.contains("At school.")) {}
			else if (str.contains("In a public place.")) {}
			// Where will you be taking the class mostly?
			else if (str.contains("At home.")) {}
			else if (str.contains("At work.")) {}
			else if (str.contains("At school.")) {}
			else if (str.contains("In a public place.")) {}
			else if (str.contains("Wherever I have 10 free minutes")) {}
			i++;
		}
	}
	
	// 
	private void countNumberOfLearnersInRange(double lbd_score, double ubd_score, int mode) {
		int numLearners = 0;
		HashSet<String> users_subset = new HashSet<String>();
		String outputFileName;
		boolean motprof = false;

    	if (mode == STUDENT_MODE) {
    		users_subset.addAll(studentsSubset);
    		outputFileName = "student-only-final-grades-fall13-histogram.txt";
    	}
    	else if (mode == NONSTUDENT_MODE) {
    		users_subset.addAll(nonStudentsSubset);
    		outputFileName = "nonstudent-only-final-grades-fall13-histogram.txt";
    	}
    	else if (mode == MALE_MODE) {
    		users_subset.addAll(maleLearnersSubset);
    		outputFileName = "male-only-final-grades-fall13-histogram.txt";
    	}
    	else if (mode == FEMALE_MODE) {
    		users_subset.addAll(femaleLearnersSubset);
    		outputFileName = "female-only-final-grades-fall13-histogram.txt";
    	}
	    else if (mode == EUROPE_MODE) {
		    users_subset.addAll(euroLearnersSubset);
		    outputFileName = "europe-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == N_AMERICAN_MODE) {
		    users_subset.addAll(namerLearnersSubset);
		    outputFileName = "n_american-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == INDIA_MODE) {
	    	users_subset.addAll(indianLearnersSubset);
		    outputFileName = "indian-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == PROF_MODE) {
		    users_subset.addAll(profLearnersSubset);
		    outputFileName = "prof-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == ACADEMIA_MODE) {
		    users_subset.addAll(academiaLearnersSubset);
		    outputFileName = "academia-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == AGE_UNDER30_MODE) {
	    	users_subset.addAll(ageUnder30LearnersSubset);
		    outputFileName = "ageunder30-only-final-grades-fall13-histogram.txt";
	    }
	    else if (mode == AGE_ABOVE30_MODE) {
	    	users_subset.addAll(ageAbove30LearnersSubset);
		    outputFileName = "ageabove30-only-final-grades-fall13-histogram.txt";
	    }
    	else { // all learners
    		users_subset.addAll(userFinalGradeMap.keySet());
    	}
    	
		// count the numbers
		for (String [] strs: compFeedbackList) {
			assert strs.length > TITLE_LEN-2; // make sure everything is the right range
			if (userFinalGradeMap.get(strs[0]) != null && users_subset.contains(strs[0])) { 
				double grades = userFinalGradeMap.get(strs[0]).normal_grade;
				if (grades >= lbd_score && grades <= ubd_score) {
					numLearners++;
					motprof = false;

					// Which of the following descriptions best characterizes you?
					if (strs[occ_student_idx].equals("1")) {
						occ_student++;
					}
					if (strs[occ_industry_idx].equals("1")) {
						occ_industry++;
					}
					if (strs[occ_research_idx].equals("1")) {
						occ_research++;
					}
					if (strs[occ_academia_idx].equals("1")) {
						occ_academia++;
					}
					// Why did you enroll in this class?
					for (Integer prof_idx: mot_prof_idx) {
						if (strs[prof_idx].equals("1")) {
							motprof = true;
							mot_prof++;
							break;
						}
					} 
					for (Integer academia_idx: mot_academia_idx) {
						if (strs[academia_idx].equals("1")) {
							if (!motprof)
							    mot_academia++;
							break;
						}
					}
					for (Integer personal_idx: mot_personal_idx) {
						if (strs[personal_idx].equals("1")) {
							mot_personal++;
							break;
						}
					} 
					// What is your current country of residence?
					for (Integer n_america_idx: geo_n_america_idx) {
						if (strs[n_america_idx].equals("1")) {
							geo_n_america++;
							break;
						}
					} 
					for (Integer europe_idx: geo_europe_idx) {
						if (strs[europe_idx].equals("1")) {
							geo_europe++;
							break;
						}
					} 
					if (strs[geo_india_idx].equals("1")) {
						geo_india++;
					}
					for (Integer s_america_idx: geo_s_america_idx) {
						if (strs[s_america_idx].equals("1")) {
							geo_s_america++;
							break;
						}
					}
					for (Integer africa_idx: geo_africa_idx) {
						if (strs[africa_idx].equals("1")) {
							geo_africa++;
							break;
						}
					} 
					if (strs[geo_china_idx].equals("1")) {
						geo_china++;
					}
					for (Integer asia_idx: geo_asia_idx) {
						if (strs[asia_idx].equals("1")) {
							geo_asia++;
							break;
						}
					} 
					for (Integer australia_idx: geo_australia_idx) {
						if (strs[australia_idx].equals("1")) {
							geo_australia++;
							break;
						}
					} 
					// How old are you?
					for (Integer a20_idx: age_20_idx) {
						if (strs[a20_idx].equals("1")) {
							age_20++;
							break;
						}
					}
					for (Integer a20_30_idx: age_20_30_idx) {
						if (strs[a20_30_idx].equals("1")) {
							age_20_30++;
							break;
						}
					}
					for (Integer a30_40_idx: age_30_40_idx) {
						if (strs[a30_40_idx].equals("1")) {
							age_30_40++;
							break;
						}
					}
					for (Integer a40_50_idx: age_40_50_idx) {
						if (strs[a40_50_idx].equals("1")) {
							age_40_50++;
							break;
						}
					}
					for (Integer a50_60_idx: age_50_60_idx) {
						if (strs[a50_60_idx].equals("1")) {
							age_50_60++;
							break;
						}
					}
					for (Integer a60_idx: age_60_idx) {
						if (strs[a60_idx].equals("1")) {
							age_60++;
							break;
						}
					}
					// What is your gender?
					if (strs[gender_male_idx].equals("1")) {
						gender_male++;
					}
					if (strs[gender_female_idx].equals("1")) {
						gender_female++;
					}
					// What is your education level and what kind of degree do you hold?
					if (strs[edu_numHighSchool_idx].equals("1")) {
						edu_numHighSchool++;
					}
					if (strs[edu_numCollege_idx].equals("1")) {
						edu_numCollege++;
					}
					if (strs[edu_numBS_idx].equals("1")) {
						edu_numBS++;
					}
					if (strs[edu_numMS_idx].equals("1")) {
						edu_numMS++;
					}
					if (strs[edu_numPHD_idx].equals("1")) {
						edu_numPHD++;
					}
				}
			}
		}
		// print results
		System.out.println("Total learners: " + numLearners + " in grades range (" + lbd_score + ", " + ubd_score + ")");
		// Which of the following descriptions best characterizes you?
		System.out.println("---");
		System.out.println("  Student  = " + String.format("%.2f", (float) occ_student / numLearners * 100));
		System.out.println("  Industry = " + String.format("%.2f", (float) occ_industry / numLearners * 100));
		System.out.println("  Research = " + String.format("%.2f", (float) occ_research / numLearners * 100));
		System.out.println("  Academia = " + String.format("%.2f", (float) occ_academia / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - occ_student - occ_industry - occ_research - occ_academia) / numLearners * 100));
		System.out.println("---");
		// Why did you enroll in this class?
		System.out.println("---");
		System.out.println("  Professional  = " + String.format("%.2f", (float) mot_prof / numLearners * 100));
		System.out.println("  Academia = " + String.format("%.2f", (float) mot_academia / numLearners * 100));
		System.out.println("  Personal = " + String.format("%.2f", (float) mot_personal / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - mot_prof - mot_academia - mot_personal) / numLearners * 100));
		System.out.println("---");
		// What is your current country of residence?
		System.out.println("---");
		System.out.println("  North America  = " + String.format("%.2f", (float) geo_n_america / numLearners * 100));
		System.out.println("  Europe = " + String.format("%.2f", (float) geo_europe / numLearners * 100));
		System.out.println("  India = " + String.format("%.2f", (float) geo_india / numLearners * 100));
		System.out.println("  South America = " + String.format("%.2f", (float) geo_s_america / numLearners * 100));
		System.out.println("  Africa = " + String.format("%.2f", (float) geo_africa / numLearners * 100));
		System.out.println("  China = " + String.format("%.2f", (float) geo_china / numLearners * 100));
		System.out.println("  Asia = " + String.format("%.2f", (float) geo_asia / numLearners * 100));
		System.out.println("  Australia = " + String.format("%.2f", (float) geo_australia / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - geo_n_america - geo_europe - geo_india - geo_s_america - geo_africa - geo_china - geo_asia - geo_australia) / numLearners * 100));
		System.out.println("---");
		// How old are you?
		System.out.println("---");
		System.out.println("  < 20  = " + String.format("%.2f", (float) age_20 / numLearners * 100));
		System.out.println("  20-30 = " + String.format("%.2f", (float) age_20_30 / numLearners * 100));
		System.out.println("  30-40 = " + String.format("%.2f", (float) age_30_40 / numLearners * 100));
		System.out.println("  40-50 = " + String.format("%.2f", (float) age_40_50 / numLearners * 100));
		System.out.println("  50-60 = " + String.format("%.2f", (float) age_50_60 / numLearners * 100));
		System.out.println("  > 60 = " + String.format("%.2f", (float) age_60 / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - age_20 - age_20_30 - age_30_40 - age_40_50 - age_50_60 - age_60) / numLearners * 100));
		System.out.println("---");
		// What is your gender?
		System.out.println("---");
		System.out.println("  Male  = " + String.format("%.2f", (float) gender_male / numLearners * 100));
		System.out.println("  Female = " + String.format("%.2f", (float) gender_female / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - gender_male - gender_female) / numLearners * 100));
		System.out.println("---");
		// What is your education level and what kind of degree do you hold?
		System.out.println("---");
		System.out.println("  High school  = " + String.format("%.2f", (float) edu_numHighSchool / numLearners * 100));
		System.out.println("  College = " + String.format("%.2f", (float) edu_numCollege / numLearners * 100));
		System.out.println("  BS = " + String.format("%.2f", (float) edu_numBS / numLearners * 100));
		System.out.println("  MS = " + String.format("%.2f", (float) edu_numMS / numLearners * 100));
		System.out.println("  PHD = " + String.format("%.2f", (float) edu_numPHD / numLearners * 100));
		System.out.println("  Others = " + String.format("%.2f", (float) (numLearners - edu_numHighSchool - edu_numCollege - edu_numBS - edu_numMS - edu_numPHD) / numLearners * 100));
		System.out.println("---");
	}
	
	private String [] splitDataField(String str) {
		ArrayList<String> strs = new ArrayList<String>();
		String s = "";
		boolean quote = false;
		for (int i = 0; i < str.length(); i++) {
			char ch = str.charAt(i);
			switch (ch) {
			case '"':
				quote = !quote;
				/*if (!quote)
					strs.add(s);*/
				break;
			case ',':
				if (!quote) {
					strs.add(s);
					s = "";
				}
				break;
			default:
				s += str.charAt(i);
				break;
			}
			    
		}
		// add the last one
		strs.add(s);
		//int i = strs.size();
		String [] newStrArray = new String[strs.size()];
		newStrArray = strs.toArray(newStrArray);
		//System.out.println("Size of the data fields: " + newStrArray.length);
		return newStrArray;
	}
	
	public void run() {
		//SurveyMiner surveyminer = new SurveyMiner();
		BufferedReader br = null;
	    try {
	    	br = new BufferedReader(new FileReader(filename));
	        String line = br.readLine();
	        while (line != null) {
                parseSurveyLine(line);
	            line = br.readLine();
	        }
	        regroupFeedback();
	        br.close();
	    } catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
	    }
	}
	
	private void checkMatlabQ() {
		ArrayList<String []> matlabFeedbackList = new ArrayList<String []>();
		for (String [] strs: compFeedbackList) {
			String [] newstrs = new String[3];
			newstrs[0] = strs[MATLABQ1];
			newstrs[1] = strs[MATLABQ2];
			newstrs[2] = strs[MATLABQ3];
			/*if (str.contains("matlab")) {	
				}*/
			matlabFeedbackList.add(newstrs);
		}
		// check feedback
		int total = matlabFeedbackList.size();
		int DLLic = 0, NDLLic = 0;
		int rating0 = 0, ratingN0 = 0;
		for (String [] strs: matlabFeedbackList) {
			// download license?
			if (strs[0].equals("0"))
				NDLLic++;
			else	
				DLLic++;
			// rating of matlab
			if (strs[1].equals("0"))
				rating0++;
			else
				ratingN0++;
			// reasons
		}
		// write results
		System.out.println("Total matlab participants = " + total);
		System.out.println("Matlab survey: ");
		System.out.println("----------------------------------------");
		System.out.printf("No Download = %.2f%%, Download = %.2f%%\n", (float) NDLLic * 100 / total, (float) DLLic * 100 / total);
		System.out.printf("Rating zero = %.2f%%, Rating non-zero = %.2f%%\n", (float) rating0 * 100 / total, (float) ratingN0 * 100 / total);
	}
	
	public GettingToKnowYouMinerFall13() {
		studentsSubset = new HashSet<String>();
		nonStudentsSubset = new HashSet<String>();
		euroLearnersSubset = new HashSet<String>();
		namerLearnersSubset = new HashSet<String>();
		indianLearnersSubset = new HashSet<String>();
		//otherAreaLearnersSubset = new HashSet<String>();
		chineseLearnersSubset = new HashSet<String>();
		maleLearnersSubset = new HashSet<String>();
		femaleLearnersSubset = new HashSet<String>();
		profLearnersSubset = new HashSet<String>();
		academiaLearnersSubset = new HashSet<String>();
		ageUnder30LearnersSubset = new HashSet<String>();
		ageAbove30LearnersSubset = new HashSet<String>();
		compFeedbackLines = new ArrayList<String>();	
		compFeedbackList = new ArrayList<String []>();
		//compFeedbackMap = new HashMap<String, String []>();
	}
	
	public static void main(String[] args) {
		GettingToKnowYouMinerFall13 surveyminer = new GettingToKnowYouMinerFall13();
		surveyminer.filename = args[0];
		BufferedReader br = null;
		int STUDENT_MODE = 1;
		int NONSTUDENT_MODE = 2;
		int MALE_MODE = 3;
		int FEMALE_MODE = 4;
		int EUROPE_MODE = 5;
		int N_AMERICAN_MODE = 6;
		//int OTHER_AREA_MODE = 7;
		int INDIA_MODE = 8;
		int PROF_MODE = 9;
		int ACADEMIA_MODE = 10;
		int AGE_UNDER30_MODE = 11;
		int AGE_ABOVE30_MODE = 12;
		int ALL_MODE = 13;
		
	    try {
	    	// create filters
	    	br = new BufferedReader(new FileReader(surveyminer.filename));
	        String line = br.readLine();
	        while (line != null) {
                surveyminer.parseSurveyLine(line);
	            line = br.readLine();
	        }
	        surveyminer.regroupFeedback();
	        br.close();
	        // DB mining with filters
	        surveyminer.getDBData();
	        surveyminer.buildSurveyIndex();
	        surveyminer.collectLearners();
	        surveyminer.writeCCDF(STUDENT_MODE); 
	        surveyminer.writeCCDF(NONSTUDENT_MODE); 
	        surveyminer.writeCCDF(MALE_MODE); 
	        surveyminer.writeCCDF(FEMALE_MODE);
	        surveyminer.writeCCDF(EUROPE_MODE); 
	        surveyminer.writeCCDF(N_AMERICAN_MODE);
	        surveyminer.writeCCDF(INDIA_MODE); 
	        surveyminer.writeCCDF(PROF_MODE); 
	        surveyminer.writeCCDF(ACADEMIA_MODE); 
	        surveyminer.writeCCDF(AGE_UNDER30_MODE);
	        surveyminer.writeCCDF(AGE_ABOVE30_MODE); 
	        surveyminer.writeCCDF(AGE_ABOVE30_MODE);
	        // post ccdf analysis
	        surveyminer.postCCDFAnalysis(ALL_MODE);
	        //surveyminer.postCCDFAnalysis(MALE_MODE);
	    } catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
	    }
	}
}
