<NME> test_mutual_information_vs_correlation.py
<BEF> ADDFILE
<MSG> Updated files to reflect reorg and fixed s/2 bug in cpp

<DFF> @@ -0,0 +1,80 @@
+import numpy
+import pylab as pl
+import tabular_predDB.python_utils.inference_utils as iu
+import tabular_predDB.python_utils.data_utils as du
+import tabular_predDB.cython_code.State as State
+
+from scipy.stats import pearsonr as pearsonr
+
+import random
+
+def get_correlations(T, Q):
+	T = numpy.array(T)
+	corr = []
+
+	for query in Q:
+		r, p = scipy.stats.pearsonr(T[:,Q[0]], T[:,Q[1]])
+		corr.append(r)
+
+	return corr
+
+def gen_correlated_data( n, r, SEED=0 ):
+	numpy.random.seed(SEED)
+	T = numpy.random.multivariate_normal([0,0],[[1,r],[r,1]],n)
+
+	return T
+
+get_next_seed = lambda : random.randrange(32000)
+
+correlations = [.0, .1, .2, .3, .4 , .5, .6, .7, .8, .9, 1.0]
+N = [5, 10, 25, 50]
+n_samples = 10
+n_data_sets = 5
+pl.figure()
+burn_in = 200
+
+subplot = 0
+for n in N:
+	subplot += 1
+	nr = 0
+	
+	for r in correlations:
+		for d in range(n_data_sets): # 3 data sets
+			#
+			T = gen_correlated_data( n, r, SEED=get_next_seed())
+
+			pr, p = pearsonr(T[:,0], T[:,1])
+
+			print "num_samples: %i, R: %f, d: %i. Actual R: %f" % (n, r, d+1, pr)
+
+			M_c = du.gen_M_c_from_T(T)
+			X_Ls = []
+			X_Ds = []
+
+			for _ in range(n_samples):
+				state = State.p_State(M_c, T)
+				state.transition(n_steps=burn_in)
+				X_Ds.append(state.get_X_D())
+				X_Ls.append(state.get_X_L())
+			
+			MI, Linfoot = iu.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=200)
+
+			if d == 0:
+				data_d = numpy.transpose(Linfoot)
+			else:
+				data_d = numpy.vstack((data_d, numpy.transpose(Linfoot)))
+
+		if nr == 0:
+			data = data_d
+		else:
+			data = numpy.hstack((data, data_d))
+		
+		nr += 1
+
+
+	pl.subplot(2,2,subplot)
+	pl.boxplot(data)
+	title = "N=%i" % n
+	pl.title(title)
+
+pl.show()

{'commit': 'diff --git a/cpp_code/src/ContinuousComponentModel.cpp b/cpp_code/src/ContinuousComponentModel.cpp\nindex e4c2735d..79dd48da 100644\n--- a/cpp_code/src/ContinuousComponentModel.cpp\n+++ b/cpp_code/src/ContinuousComponentModel.cpp\n@@ -187,6 +187,9 @@ double ContinuousComponentModel::get_draw(int random_seed) const {\n   get_hyper_doubles(r, nu, s, mu);\n   get_suffstats(count, sum_x, sum_x_squared);\n   numerics::update_continuous_hypers(count, sum_x, sum_x_squared, r, nu, s, mu);\n+\n+  // s must be divided by two to work in the T-distribution (see Kevin Murphy\'s 2007 cheat sheet)\n+  s /= 2; \n   //\n   boost::mt19937  _engine(random_seed);\n   boost::uniform_01<boost::mt19937> _dist(_engine);\n@@ -214,6 +217,9 @@ double ContinuousComponentModel::get_draw_constrained(int random_seed, vector<do\n \t\t\t\t\t     constraint);\n   }\n   numerics::update_continuous_hypers(count, sum_x, sum_x_squared, r, nu, s, mu);\n+\n+  // s must be divided by two to work in the T-distribution (see Kevin Murphy\'s 2007 cheat sheet)\n+  s /= 2; \n   //\n   boost::mt19937  _engine(random_seed);\n   boost::uniform_01<boost::mt19937> _dist(_engine);\n@@ -240,6 +246,9 @@ double ContinuousComponentModel::get_predictive_cdf(double element, vector<doubl\n   }\n   numerics::update_continuous_hypers(count, sum_x, sum_x_squared, r, nu, s, mu);\n \n+  // s must be divided by two to work in the T-distribution (see Kevin Murphy\'s 2007 cheat sheet)\n+  s /= 2; \n+\n   boost::math::students_t dist(nu);\n   double coeff = sqrt((s * (r+1)) / (nu / 2. * r));\n   \n@@ -267,6 +276,9 @@ double ContinuousComponentModel::get_predictive_pdf(double element, vector<doubl\n   }\n   numerics::update_continuous_hypers(count, sum_x, sum_x_squared, r, nu, s, mu);\n \n+  // s must be divided by two to work in the T-distribution (see Kevin Murphy\'s 2007 cheat sheet)\n+  s /= 2; \n+\n   double coeff = sqrt((s * (r+1)) / (nu / 2. * r));\n \n   // non-standard T-distribution\ndiff --git a/crosscat/binary_creation/hadoop_line_processor.py b/crosscat/binary_creation/hadoop_line_processor.py\nindex db3ab34d..db4a20dd 100644\n--- a/crosscat/binary_creation/hadoop_line_processor.py\n+++ b/crosscat/binary_creation/hadoop_line_processor.py\n@@ -22,17 +22,21 @@\n import os\n import numpy\n import sys\n+import itertools\n+import random\n #\n-import crosscat.utils.data_utils as du\n-import crosscat.utils.file_utils as fu\n-import crosscat.utils.hadoop_utils as hu\n-import crosscat.utils.xnet_utils as xu\n-import crosscat.utils.general_utils as gu\n-import crosscat.utils.timing_test_utils as ttu\n-import crosscat.utils.convergence_test_utils as ctu\n-import crosscat.LocalEngine as LE\n-import crosscat.HadoopEngine as HE\n-from crosscat.settings import Hadoop as hs\n+import tabular_predDB.python_utils.data_utils as du\n+import tabular_predDB.python_utils.file_utils as fu\n+import tabular_predDB.python_utils.hadoop_utils as hu\n+import tabular_predDB.python_utils.xnet_utils as xu\n+import tabular_predDB.python_utils.general_utils as gu\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.timing_test_utils as ttu\n+import tabular_predDB.python_utils.mutual_information_test_utils as mitu\n+import tabular_predDB.python_utils.convergence_test_utils as ctu\n+import tabular_predDB.LocalEngine as LE\n+import tabular_predDB.HadoopEngine as HE\n+from tabular_predDB.settings import Hadoop as hs\n \n \n def initialize_helper(table_data, data_dict, command_dict):\n@@ -130,6 +134,66 @@ def time_analyze_helper(table_data, data_dict, command_dict):\n         )\n     return ret_dict\n \n+def mi_analyze_helper(table_data, data_dict, command_dict):\n+\n+    gen_seed = data_dict[\'SEED\']\n+    crosscat_seed = data_dict[\'CCSEED\']\n+    num_clusters = data_dict[\'num_clusters\']\n+    num_cols = data_dict[\'num_cols\']\n+    num_rows = data_dict[\'num_rows\']\n+    num_views = data_dict[\'num_views\']\n+    corr = data_dict[\'corr\']\n+    burn_in = data_dict[\'burn_in\']\n+    mean_range = float(num_clusters)*2.0\n+\n+    # 32 bit signed int\n+    random.seed(gen_seed)\n+    get_next_seed = lambda : random.randrange(2147483647)\n+\n+    # generate the stats\n+    T, M_c, M_r, X_L, X_D, view_assignment = mitu.generate_correlated_state(num_rows,\n+        num_cols, num_views, num_clusters, mean_range, corr, seed=gen_seed);\n+\n+    table_data = dict(T=T,M_c=M_c)\n+\n+    engine = LE.LocalEngine(crosscat_seed)\n+    X_L_prime, X_D_prime = engine.analyze(M_c, T, X_L, X_D, n_steps=burn_in) \n+\n+    X_L = X_L_prime\n+    X_D = X_D_prime\n+\n+    view_assignment = numpy.array(X_L[\'column_partition\'][\'assignments\'])\n+ \n+    # for each view calclate the average MI between all pairs of columns\n+    n_views = max(view_assignment)+1\n+    MI = []\n+    Linfoot = []\n+    queries = []\n+    MI = 0.0\n+    pairs = 0.0\n+    for view in range(n_views):\n+        columns_in_view = numpy.nonzero(view_assignment==view)[0]\n+        combinations = itertools.combinations(columns_in_view,2)\n+        for pair in combinations:\n+            any_pairs = True\n+            queries.append(pair)\n+            MI_i, Linfoot_i = iu.mutual_information(M_c, [X_L], [X_D], [pair], n_samples=1000)\n+            MI += MI_i[0][0]\n+            pairs += 1.0\n+\n+    \n+    if pairs > 0.0:\n+        MI /= pairs\n+\n+    ret_dict = dict(\n+        id=data_dict[\'id\'],\n+        dataset=data_dict[\'dataset\'],\n+        sample=data_dict[\'sample\'],\n+        mi=MI,\n+        )\n+\n+    return ret_dict\n+\n def convergence_analyze_helper(table_data, data_dict, command_dict):\n     gen_seed = data_dict[\'SEED\']\n     num_clusters = data_dict[\'num_clusters\']\n@@ -213,6 +277,7 @@ method_lookup = dict(\n     time_analyze=time_analyze_helper,\n     convergence_analyze=convergence_analyze_helper,\n     chunk_analyze=chunk_analyze_helper,\n+    mi_analyze=mi_analyze_helper\n     )\n \n \ndiff --git a/crosscat/tests/mi_tests/parse_mi.py b/crosscat/tests/mi_tests/parse_mi.py\nnew file mode 100644\nindex 00000000..e26cb1bc\n--- /dev/null\n+++ b/crosscat/tests/mi_tests/parse_mi.py\n@@ -0,0 +1,150 @@\n+import numpy\n+import csv\n+import os\n+import ast\n+import pickle\n+\n+import crosscat.utils.inference_utils as iu\n+import crosscat.utils.xnet_utils as xu\n+\n+import pdb\n+\n+def is_hadoop_file(filename):\n+\tname, extension = os.path.splitext(filename)\n+\tif extension is \'gz\':\n+\t\treturn True\n+\telse:\n+\t\treturn False\n+\n+def parse_line(test, mi, linfoot):\n+\tindex = test[\'id\']\n+\tnum_rows = test[\'num_rows\']\n+\tnum_cols = test[\'num_cols\']\n+\tnum_views = test[\'num_views\']\n+\tnum_clusters = test[\'num_clusters\']\n+\tcorr = test[\'corr\']\n+\t\n+\trow = [index, num_rows, num_cols, num_views, num_clusters, corr, mi, linfoot]\n+\n+\treturn row\n+\n+def hadoop_to_dict_generator(test_key_file_object):\n+\t# return read cursor to the start (or this generator cannot be called again)\n+\ttest_key_file_object.seek(0)\n+\tfor line in test_key_file_object:\n+\t\tdict_line = xu.parse_hadoop_line(line)\n+\t\tyield dict_line\n+\t\n+def parse_data_to_csv(test_key_filename, params_dict, n_tests, output_filename):\n+\t# open input file and convert to list of dicts\n+\ttest_key_file_object = open(test_key_filename, \'rb\')\n+\tinput_lines = hadoop_to_dict_generator(test_key_file_object)\n+\n+\t# open output file and convert to list of dicts\n+\toutput_file_object = open(output_filename, \'rb\')\n+\tresults = hadoop_to_dict_generator(output_file_object)\n+\t\n+\tn_datasets = params_dict[\'n_datasets\']\n+\tn_samples = params_dict[\'n_samples\']\n+\n+\theader = [\'id\', \'num_rows\', \'num_cols\', \'num_views\', \'num_clusters\', \'corr\',\'MI\',\'Linfoot\']\n+\n+\t# data_mi = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\t# data_linfoot = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\t# counts = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\n+\tdata_mi = [0.0]*n_tests\n+\tdata_linfoot = [0.0]*n_tests\n+\tcounts = [0.0]*n_tests\n+\n+\n+\tfor result in results:\n+\t\tres = result[1] # because it\'s a tuple with an id at index 0\n+\t\ttest_idx = res[\'id\']\n+\t\ttest_dataset = res[\'dataset\']\n+\t\ttest_sample = res[\'sample\']\n+\t\t\n+\t\tdata_mi[test_idx] += float(res[\'mi\']) \n+\t\tdata_linfoot[test_idx] += float(iu.mutual_information_to_linfoot(res[\'mi\']))\n+\t\tcounts[test_idx] += 1.0\n+\t\n+\tfor test_ids in range(n_tests):\n+\t\tdata_mi[test_idx] /= counts[test_idx]\n+\t\tdata_linfoot[test_idx] /= counts[test_idx]\n+\n+\t# # calculate the mean over samples\n+\t# for test in range(n_tests):\n+\t\t\n+\t# \tfor dataset in range(n_datasets):\n+\t# \t\ttry:\n+\t# \t\t\tdata_mi[test][dataset] = numpy.array(data_mi[test][dataset],dtype=float)\n+\t# \t\texcept ValueError:\n+\t# \t\t\tpdb.set_trace()\n+\n+\t# \t\ttry:\n+\t# \t\t\tdata_mi[test][dataset] = numpy.mean(data_mi[test][dataset],axis=0)\n+\t# \t\texcept TypeError:\n+\t# \t\t\tpdb.set_trace()\n+\n+\t# \t\tdata_linfoot[test][dataset] = mi_to_linfoot(data_mi[test][dataset])\n+\n+\t# \t\tdata_mi[test][dataset] = numpy.mean(data_mi[test][dataset])\n+\t# \t\tdata_linfoot[test][dataset] = numpy.mean(data_linfoot[test][dataset])\n+\n+\t# \t# now calculate the mean over datasets\n+\t# \tdata_mi[test] = numpy.mean(numpy.array(data_mi[test]))\n+\t# \tdata_linfoot[test] = numpy.mean(numpy.array(data_linfoot[test]))\n+\t\n+\tname, extension = os.path.splitext(output_filename)\n+\n+\toutfile = name + \'.csv\'\n+\n+\twith open(outfile,\'w\') as csvfile:\n+\t\tcsvwriter = csv.writer(csvfile,delimiter=\',\')\n+\t\tcsvwriter.writerow(header)\n+\t\tcurrent_idx = -1\n+\t\tfor test in input_lines:\n+\t\t\tres = test[1]\n+\t\t\ttest_idx = res[\'id\']\n+\t\t\tif test_idx != current_idx:\n+\t\t\t\tcurrent_idx = test_idx\n+\t\t\t\tline = parse_line(res, data_mi[test_idx], data_linfoot[test_idx])\n+\t\t\t\tcsvwriter.writerow(line)\n+\n+def mi_to_linfoot(mi):\n+\t#\n+\t# linfoot = numpy.zeros(mi.shape)\n+\t# if len(mi.shape) == 1:\n+\t# \tfor entry in range(mi.size):\n+\t# \t\tlinfoot[entry] = iu.mutual_information_to_linfoot(mi[entry])\n+\t# else:\n+\t# \tfor r in range(mi.shape[0]):\n+\t# \t\tfor c in range(mi.shape[1]):\n+\t# \t\t\tlinfoot[r,c] = iu.mutual_information_to_linfoot(mi[r,c])\n+\n+\n+\t# return linfoot\n+\treturn [iu.mutual_information_to_linfoot(m) for m in mi]\n+\n+if __name__ == "__main__":\n+\n+\timport argparse\n+\tparser = argparse.ArgumentParser()\n+\n+\tparser.add_argument(\'--key_filename\', type=str)\n+\tparser.add_argument(\'--params_filename\', type=str)\n+\tparser.add_argument(\'--n_tests\', type=int)\n+\tparser.add_argument(\'--output_filename\', type=str)\n+\n+\targs = parser.parse_args()\n+\n+\tkey_filename = args.key_filename\n+\toutput_filename = args.output_filename\n+\tn_tests = args.n_tests\n+\tparams_filename = args.params_filename\n+\tparams_dict = pickle.load( open( params_filename, "rb" ))\n+\n+\n+\n+\tparse_data_to_csv(key_filename, params_dict, n_tests, output_filename)\n+\ndiff --git a/crosscat/tests/mi_tests/run_mi_test.py b/crosscat/tests/mi_tests/run_mi_test.py\nnew file mode 100644\nindex 00000000..f6f2621d\n--- /dev/null\n+++ b/crosscat/tests/mi_tests/run_mi_test.py\n@@ -0,0 +1,225 @@\n+import itertools as it\n+import time\n+\n+import argparse\n+import numpy\n+import tempfile \n+import parse_mi\n+import pickle\n+\n+import os\n+\n+import crosscat.cython_code.State as State\n+import crosscat.utils.hadoop_utils as hu\n+import crosscat.utils.file_utils as fu\n+import crosscat.utils.xnet_utils as xu\n+import crosscat.LocalEngine as LE\n+import crosscat.HadoopEngine as HE\n+\n+import run_mi_test_local\n+\n+def generate_hadoop_dicts(which_kernels, impute_run_parameters, args_dict):\n+    for which_kernel in which_kernels:\n+        kernel_list = (which_kernel, )\n+        dict_to_write = dict(impute_run_parameters)\n+        dict_to_write.update(args_dict)\n+        # must write kernel_list after update\n+        dict_to_write[\'kernel_list\'] = kernel_list\n+        yield dict_to_write\n+\n+def write_hadoop_input(input_filename, impute_run_parameters, SEED):\n+    # prep settings dictionary\n+    impute_analyze_args_dict = xu.default_analyze_args_dict\n+    impute_analyze_args_dict[\'command\'] = \'impute_analyze\'\n+    with open(input_filename, \'a\') as out_fh:\n+        xu.write_hadoop_line(out_fh, key=SEED, dict_to_write=impute_run_parameters)\n+\n+# # example of how to run a simple test run on hadoop (xdata VM)\n+# python run_mi_test.py --num_datasets 5 --num_samples 10 --which_engine_binary /user/bigdata/SSCI/be_mi_tests_00.jar \\\n+# --num_rows_list 10 --num_cols_list 2 --num_clusters_list 2 --num_views_list 1 --corr_list .1 .99 -do_remote\n+# # example of a full run\n+# python run_mi_test.py --which_engine_binary /user/bigdata/SSCI/be_mi_tests_00.jar -do_remote\n+\n+# Run\n+if __name__ == \'__main__\':\n+\n+\tdefault_num_rows_list = [100, 500, 1000] \n+\tdefault_num_cols_list = [2, 4, 8, 16]\t\n+\tdefault_num_clusters_list = [10, 25, 50]\t\n+\tdefault_num_views_list = [1, 2, 4, 8, 16]\n+\tdefault_correlation_list = [.1, .5, .9]\n+\n+\t# short test\n+\t# default_num_rows_list = [10] \n+\t# default_num_cols_list = [2, 4, 8]\t\n+\t# default_num_clusters_list = [2]\t\n+\t# default_num_views_list = [1]\n+\t# default_correlation_list = [.9]\n+\t\n+\t#\n+\tparser = argparse.ArgumentParser()\n+\tparser.add_argument(\'--gen_seed\', type=int, default=0)\n+\tparser.add_argument(\'--num_datasets\', type=int, default=5)\n+\tparser.add_argument(\'--num_samples\', type=int, default=50)\n+\tparser.add_argument(\'--burn_in\', type=int, default=250)\n+\tparser.add_argument(\'--which_engine_binary\', type=str,\n+\t        default=HE.default_engine_binary)\n+\tparser.add_argument(\'--jobtracker_uri\', type=str,\n+\t        default=HE.default_jobtracker_uri)\n+\t\tparser.add_argument(\'--hdfs_uri\', type=str,\n+\t        default=HE.default_hdfs_uri)\n+\tparser.add_argument(\'--which_engine_binary\', type=str,\n+\t        default=HE.default_engine_binary)\n+\tparser.add_argument(\'-do_local\', action=\'store_true\')\n+\tparser.add_argument(\'-do_remote\', action=\'store_true\')\n+\tparser.add_argument(\'--num_rows_list\', type=int, nargs=\'*\',\n+\t        default=default_num_rows_list)\n+\tparser.add_argument(\'--num_cols_list\', type=int, nargs=\'*\',\n+\t        default=default_num_cols_list)\n+\tparser.add_argument(\'--num_clusters_list\', type=int, nargs=\'*\',\n+\t        default=default_num_clusters_list)\n+\tparser.add_argument(\'--num_views_list\', type=int, nargs=\'*\',\n+\t        default=default_num_views_list)\n+\tparser.add_argument(\'--corr_list\', type=float, nargs=\'*\',\n+\t        default=default_correlation_list)\n+\n+\t#\n+\targs = parser.parse_args()\n+\tdata_seed = args.gen_seed\n+\tdo_local = args.do_local\n+\tdo_remote = args.do_remote\n+\tburn_in = args.burn_in\n+\tjobtracker_uri = args.jobtracker_uri\n+\thdfs_uri = args.hdfs_uri\n+\tnum_samples = args.num_samples\n+\tnum_datasets = args.num_datasets\n+\tnum_rows_list = args.num_rows_list\n+\tnum_cols_list = args.num_cols_list\n+\tnum_clusters_list = args.num_clusters_list\n+\tnum_views_list = args.num_views_list\n+\tcorr_list = args.corr_list\n+\twhich_engine_binary = args.which_engine_binary\n+\t#\n+\tprint \'using burn_in: %i\' % burn_in\n+\tprint \'using num_rows_list: %s\' % num_rows_list\n+\tprint \'using num_cols_list: %s\' % num_cols_list\n+\tprint \'using num_clusters_list: %s\' % num_clusters_list\n+\tprint \'using num_views_list: %s\' % num_views_list\n+\tprint \'using corr_list: %s\' % corr_list\n+\tprint \'using engine_binary: %s\' % which_engine_binary\n+\ttime.sleep(2)\n+\n+\tdirname = \'mi_analysis\'\n+\tfu.ensure_dir(dirname)\n+\tdirectory_path = tempfile.mkdtemp(prefix=\'mi_analysis_\',\n+                                dir=dirname)\n+\n+\tprint \'output sent to %s\' % directory_path\n+\n+\toutput_path = os.path.join(directory_path, \'output\')\n+\toutput_filename = os.path.join(directory_path, \'hadoop_output\')\n+\ttable_data_filename = os.path.join(directory_path, \'table_data.pkl.gz\')\n+\n+\tassert(os.path.exists(directory_path))\n+\t\n+\tinput_filename = os.path.join(directory_path, "hadoop_input")\n+\tparams_filename = os.path.join(directory_path, "test_params.pkl")\n+\tkey_filename = os.path.join(directory_path, "test_key.pkl")\n+\t\n+\t# create a parameters dict\n+\tparams_dict = {\n+\t\t\'n_rows\' \t  : num_rows_list,\n+\t\t\'n_clusters\'  : num_clusters_list,\n+\t\t\'n_cols\' \t  : num_cols_list,\n+\t\t\'n_views\' \t  : num_views_list,\n+\t\t\'corr\' \t  \t  : corr_list,\n+\t\t\'n_datasets\'  : num_datasets,\n+\t\t\'n_samples\'   : num_samples,\n+\t\t\'n_datasets\'  : num_datasets,\n+\t\t\'burn_in\' \t  : burn_in\n+\t}\n+\n+\t# save the params file as pickle\n+\ttry:\n+\t\tpd_file = open( params_filename, "wb" )\n+\texcept IOError as err:\n+\t\tprint "Could not create %s. " % params_filename, err\n+\t\traise\n+\n+\tpickle.dump( params_dict, pd_file )\n+\tpd_file.close()\n+\n+\t# cartesian product of test parameters. \n+\ttests = list(it.product(*[num_rows_list, num_clusters_list, num_cols_list, num_views_list, corr_list]))\n+\n+\ttestlist = []\n+\t\n+\tprint "Writing tests file."\t\n+\ttest_idx = 0\n+\tfor n_rows, n_clusters, n_cols, n_views, corr in tests:\t\n+\t\tif n_rows >= n_clusters and n_cols >= n_views:\n+\t\t\tfor dataset in range(num_datasets):\n+\t\t\t\tfor sample in range(num_samples):\n+\t\t\t\t\timpute_run_parameters = dict(\n+\t\t\t\t\t\t\tid=test_idx,\n+\t\t\t\t\t\t\tdataset=dataset,\n+\t\t\t\t\t\t\tsample=sample,\n+\t\t\t\t\t\t\tnum_clusters=n_clusters,\n+\t\t\t\t\t\t\tnum_rows=n_rows,\n+\t\t\t\t\t\t\tnum_cols=n_cols,\n+\t\t\t\t\t\t\tnum_views=n_views,\n+\t\t\t\t\t\t\tcorr=corr,\n+\t\t\t\t\t\t\tburn_in=burn_in,\n+\t\t\t\t\t\t\tSEED=data_seed+dataset,\n+\t\t\t\t\t\t\tCCSEED=data_seed+dataset+sample\n+\t\t\t\t\t\t)\n+\n+\t\t\t\t\twrite_hadoop_input(input_filename, impute_run_parameters, SEED=data_seed+dataset)\n+\t\t\t\t\tif do_local:\n+\t\t\t\t\t\ttestlist.append(impute_run_parameters)\n+\t\t\ttest_idx += 1\n+\t\n+\tprint "Done."\n+\t\n+\t# table data is empty because we generate it in the mapper\n+\ttable_data=dict(T=[],M_c=[],X_L=[],X_D=[])\n+\tfu.pickle(table_data, table_data_filename)\n+\n+\t#####################\n+\tif do_local:\n+\t\toutput_filename = os.path.join(directory_path, "output_local")\n+\t\toutput_file_object = open(output_filename, \'ab\')\n+\t\twith open(input_filename,\'rb\') as infile:\n+\t\t\tfor line in infile:\n+\t\t\t\tkey, test_dict = xu.parse_hadoop_line(line)\n+\t\t\t\tret_dict = run_mi_test_local.run_mi_test_local(test_dict)\n+\t\t\t\txu.write_hadoop_line(output_file_object, key, ret_dict)\n+\t\t\t\tprint "%s\\n\\t%s" % (str(test_dict), str(ret_dict))\n+\n+\t\toutput_file_object.close()\n+\t\t# generate the csv\n+\t\tparse_mi.parse_data_to_csv(input_filename, params_dict, test_idx, output_filename)\n+\t\tprint "Done."\n+\telif do_remote:\n+\t\t# generate the massive hadoop files\n+\t\thadoop_engine = HE.HadoopEngine(output_path=output_path,\n+                                    input_filename=input_filename,\n+                                    table_data_filename=table_data_filename,\n+                                    which_engine_binary=which_engine_binary,\n+                                    hdfs_uri=hdfs_uri,\n+                                    jobtracker_uri=jobtracker_uri,\n+                                    )\n+\t\n+\t\txu.write_support_files(table_data, hadoop_engine.table_data_filename,\n+\t                              dict(command=\'mi_analyze\'), hadoop_engine.command_dict_filename)\n+\t\tt_start = time.time()\n+\t\thadoop_engine.send_hadoop_command(n_tasks=len(testlist))\n+\t\twas_successful = hadoop_engine.get_hadoop_results()\n+\t\tif was_successful:\n+\t\t\tt_end = time.time()\n+\t\t\tt_total = t_end-t_start\n+\t\t\tprint "That took %i seconds." % t_total\n+\t\t\thu.copy_hadoop_output(hadoop_engine.output_path, output_filename)\n+\t\t\tparse_mi.parse_data_to_csv(input_filename, params_dict, test_idx, output_filename)\n+\t\telse:\n+\t\t\tprint "Hadoop job was NOT successful. Check %s" % output_path\n\\ No newline at end of file\ndiff --git a/crosscat/tests/mi_tests/run_mi_test_local.py b/crosscat/tests/mi_tests/run_mi_test_local.py\nnew file mode 100644\nindex 00000000..7a93c17c\n--- /dev/null\n+++ b/crosscat/tests/mi_tests/run_mi_test_local.py\n@@ -0,0 +1,66 @@\n+import itertools\n+import random\n+import numpy\n+import crosscat.LocalEngine as LE\n+import crosscat.utils.inference_utils as iu\n+import crosscat.utils.mutual_information_test_utils as mitu\n+\n+def run_mi_test_local(data_dict):\n+\n+    gen_seed = data_dict[\'SEED\']\n+    crosscat_seed = data_dict[\'CCSEED\']\n+    num_clusters = data_dict[\'num_clusters\']\n+    num_cols = data_dict[\'num_cols\']\n+    num_rows = data_dict[\'num_rows\']\n+    num_views = data_dict[\'num_views\']\n+    corr = data_dict[\'corr\']\n+    burn_in = data_dict[\'burn_in\']\n+    mean_range = float(num_clusters)*2.0\n+\n+    # 32 bit signed int\n+    random.seed(gen_seed)\n+    get_next_seed = lambda : random.randrange(2147483647)\n+\n+    # generate the stats\n+    T, M_c, M_r, X_L, X_D, view_assignment = mitu.generate_correlated_state(num_rows,\n+        num_cols, num_views, num_clusters, mean_range, corr, seed=gen_seed);\n+\n+    table_data = dict(T=T,M_c=M_c)\n+\n+    engine = LE.LocalEngine(crosscat_seed)\n+    X_L_prime, X_D_prime = engine.analyze(M_c, T, X_L, X_D, n_steps=burn_in) \n+\n+    X_L = X_L_prime\n+    X_D = X_D_prime\n+\n+    view_assignment = numpy.array(X_L[\'column_partition\'][\'assignments\'])\n+ \n+    # for each view calclate the average MI between all pairs of columns\n+    n_views = max(view_assignment)+1\n+    MI = []\n+    Linfoot = []\n+    queries = []\n+    MI = 0.0\n+    pairs = 0.0\n+    for view in range(n_views):\n+        columns_in_view = numpy.nonzero(view_assignment==view)[0]\n+        combinations = itertools.combinations(columns_in_view,2)\n+        for pair in combinations:\n+            any_pairs = True\n+            queries.append(pair)\n+            MI_i, Linfoot_i = iu.mutual_information(M_c, [X_L], [X_D], [pair], n_samples=1000)\n+            MI += MI_i[0][0]\n+            pairs += 1.0\n+\n+    \n+    if pairs > 0.0:\n+        MI /= pairs\n+\n+    ret_dict = dict(\n+        id=data_dict[\'id\'],\n+        dataset=data_dict[\'dataset\'],\n+        sample=data_dict[\'sample\'],\n+        mi=MI,\n+        )\n+\n+    return ret_dict\n\\ No newline at end of file\ndiff --git a/crosscat/tests/mi_tests/test_mutual_information.py b/crosscat/tests/mi_tests/test_mutual_information.py\nnew file mode 100644\nindex 00000000..6acbb588\n--- /dev/null\n+++ b/crosscat/tests/mi_tests/test_mutual_information.py\n@@ -0,0 +1,213 @@\n+\n+# calculated the mutual information of various shapes of data\n+import numpy\n+import pylab as pl\n+import crosscat.utils.sample_utils as su\n+import crosscat.utils.inference_utils as iu\n+import crosscat.utils.data_utils as du\n+import crosscat.cython_code.State as State\n+\n+import random\n+import math\n+\n+def ring(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tangle = random.uniform(0,2*math.pi)\n+\t\tdistance = random.uniform(1,1.5)\n+\t\tX[i,0] = math.cos(angle)*distance\n+\t\tX[i,1] = math.sin(angle)*distance\n+\n+\treturn X\n+\n+def circle(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tangle = random.uniform(0,2*math.pi)\n+\t\tdistance = random.uniform(0,1.5)\n+\t\tX[i,0] = math.cos(angle)*distance\n+\t\tX[i,1] = math.sin(angle)*distance\n+\n+\treturn X\n+\n+def square(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tx = random.uniform(-1,1)\n+\t\ty = random.uniform(-1,1)\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def diamond(n=200):\n+\n+\tX = square(n=n)\n+\tfor i in range(n):\n+\t\tangle = math.atan(X[i,1]/X[i,0])\n+\t\tangle += math.pi/4 \n+\t\thyp = (X[i,0]**2.0+X[i,1]**2.0)**.5\n+\t\tx = math.cos(angle)*hyp\n+\t\ty = math.sin(angle)*hyp\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def four_dots(n=200):\n+\tX = numpy.zeros((n,2))\n+\tnb = n/4\n+\tmx = [ -1, 1, -1, 1]\n+\tmy = [ -1, -1, 1, 1]\n+\ts = .25\n+\t\n+\tfor i in range(n):\n+\t\tn = random.randrange(4)\n+\t\tx = random.normalvariate(mx[n], s)\n+\t\ty = random.normalvariate(my[n], s)\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\t\t\n+\treturn X\n+\n+def correlated(r,n=200):\n+\tX = numpy.random.multivariate_normal([0,0], [[1, r],[r, 1]], n)\n+\treturn X\n+\n+def sample_from_view(M_c, X_L, X_D, get_next_seed):\n+    \n+    view_col = X_L[\'column_partition\'][\'assignments\'][0]\n+    view_col2 = X_L[\'column_partition\'][\'assignments\'][1]\n+\n+    same_view = True\n+    if view_col2 != view_col:\n+    \tsame_view = False\n+\n+    view_state = X_L[\'view_state\'][view_col]\n+    view_state2 = X_L[\'view_state\'][view_col2]\n+\n+    cluster_crps = numpy.exp(su.determine_cluster_crp_logps(view_state))\n+    cluster_crps2 = numpy.exp(su.determine_cluster_crp_logps(view_state2))\n+\n+    assert( math.fabs(numpy.sum(cluster_crps) - 1) < .00000001 )\n+\n+    samples = numpy.zeros((n,2))\n+\n+    \n+    cluster_idx1 = numpy.nonzero(numpy.random.multinomial(1, cluster_crps))[0][0]\n+    cluster_model1 = su.create_cluster_model_from_X_L(M_c, X_L, view_col, cluster_idx1)\n+\n+    if same_view:\n+    \tcluster_idx2 = cluster_idx1\n+    \tcluster_model2 = cluster_model1\n+    else:\n+    \tcluster_idx2 = numpy.nonzero(numpy.random.multinomial(1, cluster_crps2))[0][0]\n+    \tcluster_model2 = su.create_cluster_model_from_X_L(M_c, X_L, view_col2, cluster_idx2)\n+\n+    component_model1 = cluster_model1[0]\n+    x = component_model1.get_draw(get_next_seed())\n+\n+    component_model2 = cluster_model2[1]\n+    y = component_model2.get_draw(get_next_seed())\n+        \n+    return x, y\n+\n+def sample_data_from_crosscat(M_c, X_Ls, X_Ds, get_next_seed, n):\n+\n+\tX = numpy.zeros((n,2))\n+\tn_samples = len(X_Ls)\n+\t\n+\tfor i in range(n):\n+\t\tcc = random.randrange(n_samples)\n+\t\tx, y = sample_from_view(M_c, X_Ls[cc], X_Ds[cc], get_next_seed)\n+\t\t\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def do_test(which_plot, max_plots, n, burn_in, cc_samples, which_test, correlation=0, do_plot=False):\n+\tif which_test is "correlated":\n+\t\tX = correlated(correlation, n=n)\n+\telif which_test is "square":\n+\t\tX = square(n=n)\n+\telif which_test is "ring":\n+\t\tX = ring(n=n)\n+\telif which_test is "circle":\n+\t\tX = circle(n=n)\n+\telif which_test is "diamond":\n+\t\tX = diamond(n=n)\n+\telif which_test is "blob":\n+\t\tX = correlated(0.0, n=n)\n+\telif which_test is "dots":\n+\t\tX = four_dots(n=n)\n+\telif which_test is "mixed":\n+\t\tX = numpy.vstack((correlated(.95, n=n/2),correlated(0, n=n/2)))\n+\n+\tget_next_seed = lambda : random.randrange(32000)\n+\n+\t# Build a state\n+\tM_c = du.gen_M_c_from_T(X.tolist())\n+\tstate = State.p_State(M_c, X.tolist())\n+\tX_Ls = []\n+\tX_Ds = []\n+\t\n+\t# collect crosscat samples\n+\tfor _ in range(cc_samples):\n+\t\tstate = State.p_State(M_c, X.tolist())\n+\t\tstate.transition(n_steps=burn_in)\n+\t\tX_Ds.append(state.get_X_D())\n+\t\tX_Ls.append(state.get_X_L())\n+\n+\tSX = sample_data_from_crosscat(M_c, X_Ls, X_Ds, get_next_seed, n)\n+\n+\tif do_plot:\n+\t\tpl.subplot(2,max_plots,which_plot)\n+\t\tpl.scatter(X[:,0],X[:,1],c=\'blue\',alpha=.5)\n+\t\tpl.title("Original data")\n+\t\tpl.subplot(2,max_plots,max_plots+which_plot)\n+\t\tpl.scatter(SX[:,0],SX[:,1],c=\'red\',alpha=.5)\n+\t\tpl.title("Sampled data")\n+\t\tpl.show\n+\n+\treturn M_c, X_Ls, X_Ds\n+\n+def MI_test(n, burn_in, cc_samples, which_test, n_MI_samples=500, correlation=0):\n+\tM_c, X_Ls, X_Ds = do_test(0, 0, n, burn_in, cc_samples, "correlated", correlation=correlation, do_plot=False)\n+\t# query column 0 and 1\n+\tMI, Linfoot = iu.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=n_MI_samples)\n+\n+\tMI = numpy.mean(MI)\n+\tLinfoot = numpy.mean(Linfoot)\n+\t\n+\tif which_test == "correlated":\n+\t\ttest_strn = "Test: correlation (%1.2f), N: %i, burn_in: %i, samples: %i, MI_samples: %i\\n\\tMI: %f, Linfoot %f" % (correlation, n, burn_in, cc_samples, n_MI_samples, MI, Linfoot)\n+\telse:\n+\t\ttest_strn = "Test: %s, N: %i, burn_in: %i, samples: %i, MI_samples: %i\\n\\tMI: %f, Linfoot %f" % (which_test, n, burn_in, cc_samples, n_MI_samples, MI, Linfoot)\n+\t\t\n+\n+\tprint test_strn\n+\treturn test_strn\n+\n+do_plot = False\n+n_mi_samples = 500\n+\n+N = [10, 100, 1000] \n+\n+burn_in = 200\n+cc_samples = 10\n+\n+print " "\n+for n in N:\n+\t\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.3)\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.6)\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.9)\n+\tstrn = MI_test(n, burn_in, cc_samples, "ring")\n+\tstrn = MI_test(n, burn_in, cc_samples, "dots")\n+\tstrn = MI_test(n, burn_in, cc_samples, "mixed")\n+\n+\ndiff --git a/crosscat/tests/mi_tests/test_mutual_information_vs_correlation.py b/crosscat/tests/mi_tests/test_mutual_information_vs_correlation.py\nnew file mode 100644\nindex 00000000..2db166a4\n--- /dev/null\n+++ b/crosscat/tests/mi_tests/test_mutual_information_vs_correlation.py\n@@ -0,0 +1,82 @@\n+# calculates mutual information of a 2 column data set with different correlations\n+import numpy\n+import pylab as pl\n+import crosscat.utils.inference_utils as iu\n+import crosscat.utils.data_utils as du\n+import crosscat.cython_code.State as State\n+\n+from scipy.stats import pearsonr as pearsonr\n+\n+import random\n+\n+def get_correlations(T, Q):\n+\tT = numpy.array(T)\n+\tcorr = []\n+\n+\tfor query in Q:\n+\t\tr, p = scipy.stats.pearsonr(T[:,Q[0]], T[:,Q[1]])\n+\t\tcorr.append(r)\n+\n+\treturn corr\n+\n+def gen_correlated_data( n, r, SEED=0 ):\n+\tnumpy.random.seed(SEED)\n+\tT = numpy.random.multivariate_normal([0,0],[[1,r],[r,1]],n)\n+\n+\treturn T\n+\n+get_next_seed = lambda : random.randrange(32000)\n+\n+correlations = [.0, .1, .2, .3, .4 , .5, .6, .7, .8, .9, 1.0]\n+# N = [5, 10, 25, 50]\n+N = [50]\n+n_samples = 10\n+n_data_sets = 3\n+pl.figure()\n+burn_in = 200\n+\n+subplot = 0\n+for n in N:\n+\tsubplot += 1\n+\tnr = 0\n+\t\n+\tfor r in correlations:\n+\t\tfor d in range(n_data_sets): # 3 data sets\n+\t\t\t#\n+\t\t\tT = gen_correlated_data( n, r, SEED=get_next_seed())\n+\n+\t\t\tpr, p = pearsonr(T[:,0], T[:,1])\n+\n+\t\t\tprint "num_samples: %i, R: %f, d: %i. Actual R: %f" % (n, r, d+1, pr)\n+\n+\t\t\tM_c = du.gen_M_c_from_T(T)\n+\t\t\tX_Ls = []\n+\t\t\tX_Ds = []\n+\n+\t\t\tfor _ in range(n_samples):\n+\t\t\t\tstate = State.p_State(M_c, T)\n+\t\t\t\tstate.transition(n_steps=burn_in)\n+\t\t\t\tX_Ds.append(state.get_X_D())\n+\t\t\t\tX_Ls.append(state.get_X_L())\n+\t\t\t\n+\t\t\tMI, Linfoot = iu.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=200)\n+\n+\t\t\tif d == 0:\n+\t\t\t\tdata_d = numpy.transpose(Linfoot)\n+\t\t\telse:\n+\t\t\t\tdata_d = numpy.vstack((data_d, numpy.transpose(Linfoot)))\n+\n+\t\tif nr == 0:\n+\t\t\tdata = data_d\n+\t\telse:\n+\t\t\tdata = numpy.hstack((data, data_d))\n+\t\t\n+\t\tnr += 1\n+\n+\n+\tpl.subplot(2,2,subplot)\n+\tpl.boxplot(data)\n+\ttitle = "N=%i" % n\n+\tpl.title(title)\n+\n+pl.show()\ndiff --git a/crosscat/utils/inferrence_utils.py b/crosscat/utils/inferrence_utils.py\nnew file mode 100644\nindex 00000000..ff3ed0c0\n--- /dev/null\n+++ b/crosscat/utils/inferrence_utils.py\n@@ -0,0 +1,218 @@\n+from scipy.misc import logsumexp\n+import numpy\n+import random\n+import math\n+\n+import crosscat.cython_code.ContinuousComponentModel as CCM\n+import crosscat.cython_code.MultinomialComponentModel as MCM\n+import crosscat.utils.sample_utils as su\n+\n+\n+def mutual_information_to_linfoot(MI):\n+    return (1.0-math.exp(-2.0*MI))**0.5\n+\n+# return the estimated mutual information for each pair of columns on Q given\n+# the set of samples in X_Ls and X_Ds. Q is a list of tuples where each tuple\n+# contains X and Y, the columns to compare. \n+# Q = [(X_1, Y_1), (X_2, Y_2), ..., (X_n, Y_n)]\n+# Returns a list of list where each sublist is a set of MI\'s and Linfoots from\n+# each crosscat posterior sample. \n+# See tests/test_mutual_information.py and \n+# tests/test_mutual_information_vs_correlation.py for useage examples\n+def mutual_information(M_c, X_Ls, X_Ds, Q, n_samples=1000):\n+    #\n+    assert(len(X_Ds) == len(X_Ls))\n+    n_postertior_samples = len(X_Ds)\n+\n+    n_rows = len(X_Ds[0][0])\n+    n_cols = len(M_c[\'column_metadata\'])\n+\n+    MI = []\n+    Linfoot = []\n+    NMI = []\n+\n+    get_next_seed = lambda: random.randrange(32767)\n+\n+    for query in Q:\n+        assert(len(query) == 2)\n+        assert(query[0] >= 0 and query[0] < n_cols)\n+        assert(query[1] >= 0 and query[1] < n_cols)\n+\n+        X = query[0]\n+        Y = query[1]\n+\n+        MI_sample = []\n+        Linfoot_sample = []\n+        \n+        for sample in range(n_postertior_samples):\n+            \n+            X_L = X_Ls[sample]\n+            X_D = X_Ds[sample]\n+\n+            MI_s = estimiate_MI_sample(X, Y, M_c, X_L, X_D, get_next_seed, n_samples=n_samples)\n+\n+            linfoot = mutual_information_to_linfoot(MI_s)\n+            \n+            MI_sample.append(MI_s)\n+\n+            Linfoot_sample.append(linfoot)\n+\n+        MI.append(MI_sample)\n+        Linfoot.append(Linfoot_sample)\n+\n+         \n+    assert(len(MI) == len(Q))\n+    assert(len(Linfoot) == len(Q))\n+\n+    return MI,  Linfoot\n+\n+# estimates the mutual information for columns X and Y.\n+def estimiate_MI_sample(X, Y, M_c, X_L, X_D, get_next_seed, n_samples=1000):\n+    \n+    get_view_index = lambda which_column: X_L[\'column_partition\'][\'assignments\'][which_column]\n+\n+    view_X = get_view_index(X)\n+    view_Y = get_view_index(Y)\n+\n+    # independent\n+    if view_X != view_Y:\n+        return 0.0\n+\n+    # get cluster logps\n+    view_state = X_L[\'view_state\'][view_X]\n+    cluster_logps = su.determine_cluster_crp_logps(view_state)\n+    cluster_crps = numpy.exp(cluster_logps) # get exp\'ed values for multinomial\n+    n_clusters = len(cluster_crps)\n+\n+    # get components models for each cluster for columns X and Y\n+    component_models_X = [0]*n_clusters\n+    component_models_Y = [0]*n_clusters\n+    for i in range(n_clusters):\n+        cluster_models = su.create_cluster_model_from_X_L(M_c, X_L, view_X, i)\n+        component_models_X[i] = cluster_models[X]\n+        component_models_Y[i] = cluster_models[Y]\n+\n+    MI = 0.0    # mutual information\n+\n+    for _ in range(n_samples):\n+        # draw a cluster \n+        cluster_idx = numpy.nonzero(numpy.random.multinomial(1, cluster_crps))[0][0]\n+\n+        # get a sample from each cluster\n+        x = component_models_X[cluster_idx].get_draw(get_next_seed())\n+        y = component_models_Y[cluster_idx].get_draw(get_next_seed())\n+\n+        # calculate marginal logs\n+        Pxy = numpy.zeros(n_clusters)   # P(x,y), Joint distribution\n+        Px = numpy.zeros(n_clusters)    # P(x)\n+        Py = numpy.zeros(n_clusters)    # P(y)\n+\n+        # get logp of x and y in each cluster. add cluster logp\'s\n+        for j in range(n_clusters):\n+\n+            X_k = get_data(T,X,j,view_X,X_D)\n+            Y_k = get_data(T,Y,j,view_Y,X_D)\n+\n+            Px[j] = component_models_X[j].calc_element_predictive_logp(x)\n+            Py[j] = component_models_Y[j].calc_element_predictive_logp(y)\n+            Pxy[j] = Px[j] + Py[j] + cluster_logps[j]   # \\sum_c P(x|c)P(y|c)P(c), Joint distribution\n+            Px[j] += cluster_logps[j]                   # \\sum_c P(x|c)P(c)\n+            Py[j] += cluster_logps[j]                   # \\sum_c P(y|c)P(c)    \n+\n+        # pdb.set_trace()\n+        \n+        # sum over clusters\n+        Px = logsumexp(Px)\n+        Py = logsumexp(Py)\n+        Pxy = logsumexp(Pxy)\n+\n+        # add to MI\n+        MI += Pxy - (Px + Py)\n+\n+    # average\n+    MI /= float(n_samples)\n+\n+    # ignore MI < 0\n+    if MI <= 0.0:\n+        MI = 0.0\n+        \n+    return MI\n+\n+# Histogram estimations are biased and shouldn\'t be used, this is just for testing purposes.\n+def estimiate_MI_sample_hist(X, Y, M_c, X_L, X_D, get_next_seed, n_samples=10000):\n+    \n+    get_view_index = lambda which_column: X_L[\'column_partition\'][\'assignments\'][which_column]\n+\n+    view_X = get_view_index(X)\n+    view_Y = get_view_index(Y)\n+\n+    # independent\n+    if view_X != view_Y:        \n+        return 0.0\n+\n+    # get cluster logps\n+    view_state = X_L[\'view_state\'][view_X]\n+    cluster_logps = su.determine_cluster_crp_logps(view_state)\n+    cluster_crps = numpy.exp(cluster_logps)\n+    n_clusters = len(cluster_crps)\n+\n+    # get components models for each cluster for columns X and Y\n+    component_models_X = [0]*n_clusters\n+    component_models_Y = [0]*n_clusters\n+    for i in range(n_clusters):\n+        cluster_models = su.create_cluster_model_from_X_L(M_c, X_L, view_X, i)\n+        component_models_X[i] = cluster_models[X]\n+        component_models_Y[i] = cluster_models[Y]\n+\n+    MI = 0.0\n+    samples = numpy.zeros((n_samples,2), dtype=float)\n+    samples_x = numpy.zeros(n_samples, dtype=float)\n+    samples_y = numpy.zeros(n_samples, dtype=float)\n+\n+    # draw the samples\n+    for i in range(n_samples):\n+        # draw a cluster \n+        cluster_idx = numpy.nonzero(numpy.random.multinomial(1, cluster_crps))[0][0]\n+\n+        x = component_models_X[cluster_idx].get_draw(get_next_seed())\n+        y = component_models_Y[cluster_idx].get_draw(get_next_seed())\n+\n+        samples[i,0] = x\n+        samples[i,1] = y\n+        samples_x[i] = x\n+        samples_y[i] = y\n+\n+    # calculate the number of bins and ranges\n+    N = float(n_samples)\n+    r,_ = corr(samples_x, samples_y)\n+    k = round(.5+.5*(1+4*((6*N*r**2.)/(1-r**2.))**.5)**.5)+1\n+    sigma_x = numpy.std(samples_x)\n+    mu_x = numpy.mean(samples_x)\n+    sigma_y = numpy.std(samples_y)\n+    mu_y = numpy.mean(samples_y)\n+    range_x = numpy.linspace(mu_x-3.*sigma_x,mu_x+3*sigma_x,k)\n+    range_y = numpy.linspace(mu_y-3.*sigma_y,mu_y+3*sigma_y,k)\n+\n+\n+    PXY, _, _ = numpy.histogram2d(samples[:,0], samples[:,1], bins=[range_x,range_y])\n+    PX,_ = numpy.histogram(samples_x,bins=range_x)\n+    PY,_ = numpy.histogram(samples_y,bins=range_y)\n+\n+    MI = 0\n+\n+    for i_x in range(PXY.shape[0]):\n+        for i_y in range(PXY.shape[1]):            \n+            Pxy = PXY[i_x,i_y]\n+            Px = PX[i_x]\n+            Py = PY[i_y]\n+            \n+            if Pxy > 0.0 and Px > 0.0 and Py > 0.0:\n+                MI += (Pxy/N)*math.log(Pxy*N/(Px*Py))\n+            \n+\n+\n+    # ignore MI < 0\n+    if MI <= 0.0:\n+        MI = 0.0\n+        \n+    return MI\ndiff --git a/crosscat/utils/mutual_information_test_utils.py b/crosscat/utils/mutual_information_test_utils.py\nnew file mode 100644\nindex 00000000..e041a685\n--- /dev/null\n+++ b/crosscat/utils/mutual_information_test_utils.py\n@@ -0,0 +1,104 @@\n+import crosscat.utils.data_utils as du\n+import crosscat.cython_code.State as State\n+import random\n+import numpy\n+\n+# Generates a num_rows by num_cols array of data with covariance matrix I^{num_cols}*corr\n+def generate_correlated_data(num_rows, num_cols, means, corr, seed=0):\n+\tassert(corr <= 1 and corr >= 0)\n+\tassert(num_cols == len(means))\n+\n+\tnumpy.random.seed(seed=seed)\n+\n+\tmu = numpy.array(means)\n+\tsigma = numpy.ones((num_cols,num_cols),dtype=float)*corr\n+\tfor i in range(num_cols):\n+\t\tsigma[i,i] = 1 \n+\tX = numpy.random.multivariate_normal(mu, sigma, num_rows)\n+\n+\treturn X\n+\n+def generate_correlated_state(num_rows, num_cols, num_views, num_clusters, mean_range, corr, seed=0):\n+\t#\n+\n+\tassert(num_clusters <= num_rows)\n+\tassert(num_views <= num_cols)\n+\tT = numpy.zeros((num_rows, num_cols))\n+\n+\trandom.seed(seed)\n+\tnumpy.random.seed(seed=seed)\n+\tget_next_seed = lambda : random.randrange(2147483647)\n+\n+\t# generate an assignment of columns to views (uniform)\n+\tcols_to_views = range(num_views)\n+\tview_counts = numpy.ones(num_views, dtype=int)\n+\tfor i in range(num_views, num_cols):\n+\t\tr = random.randrange(num_views)\n+\t\tcols_to_views.append(r)\n+\t\tview_counts[r] += 1\n+\n+\trandom.shuffle(cols_to_views)\n+\n+\tassert(len(cols_to_views) == num_cols)\n+\tassert(max(cols_to_views) == num_views-1)\n+\n+\t# for each view, generate an assignment of rows to num_clusters\n+\trow_to_clusters = []\n+\tcluster_counts = []\n+\tfor view in range(num_views):\n+\t\trow_to_cluster = range(num_clusters)\n+\t\tcluster_counts_i = numpy.ones(num_clusters,dtype=int)\n+\t\tfor i in range(num_clusters, num_rows):\n+\t\t\tr = random.randrange(num_clusters)\n+\t\t\trow_to_cluster.append(r)\n+\t\t\tcluster_counts_i[r] += 1\n+\n+\t\trandom.shuffle(row_to_cluster)\n+\n+\t\tassert(len(row_to_cluster) == num_rows)\n+\t\tassert(max(row_to_cluster) == num_clusters-1)\n+\n+\t\trow_to_clusters.append(row_to_cluster)\n+\t\tcluster_counts.append(cluster_counts_i)\n+\n+\tassert(len(row_to_clusters) == num_views)\n+\n+\t# generate the correlated data\n+\tfor view in range(num_views):\n+\t\tfor cluster in range(num_clusters):\n+\t\t\tcell_cols = view_counts[view]\n+\t\t\tcell_rows = cluster_counts[view][cluster]\n+\t\t\tmeans = numpy.random.uniform(-mean_range/2.0,mean_range/2.0,cell_cols)\n+\t\t\tX =  generate_correlated_data(cell_rows, cell_cols, means, corr, seed=get_next_seed())\n+\t\t\t# get the indices of the columns in this view\n+\t\t\tcol_indices = numpy.nonzero(numpy.array(cols_to_views)==view)[0]\n+\t\t\t# get the indices of the rows in this view and this cluster\n+\t\t\trow_indices = numpy.nonzero(numpy.array(row_to_clusters[view])==cluster)[0]\n+\t\t\t# insert the data\n+\t\t\tfor col in range(cell_cols):\n+\t\t\t\tfor row in range(cell_rows):\n+\t\t\t\t\tr = row_indices[row]\n+\t\t\t\t\tc = col_indices[col]\n+\t\t\t\t\tT[r,c] = X[row,col]\n+\n+\n+\tM_c = du.gen_M_c_from_T(T)\n+\tM_r = du.gen_M_r_from_T(T)\n+\tX_L, X_D = generate_X_L_and_X_D(T, M_c, cols_to_views, row_to_clusters, seed=get_next_seed())\n+\n+\treturn  T, M_c, M_r, X_L, X_D, cols_to_views\n+\n+def generate_X_L_and_X_D(T, M_c, cols_to_views, row_to_clusters, seed=0):\n+\tstate = State.p_State(M_c, T, SEED=seed)\n+\tX_L = state.get_X_L()\n+\n+\t# insert assigment into X_L (this is not a valid X_L because the counts and \n+\t# suffstats will be wrong)\n+\tX_L[\'column_partition\'][\'assignments\'] = cols_to_views\n+\tstate = State.p_State(M_c, T, X_L=X_L, X_D=row_to_clusters, SEED=seed)\n+\n+\tX_L = state.get_X_L()\n+\tX_D = state.get_X_D()\n+\n+\treturn X_L, X_D\n+\ndiff --git a/tabular_predDB/python_utils/inference_utils.py b/tabular_predDB/python_utils/inference_utils.py\nnew file mode 100644\nindex 00000000..a42e24b9\n--- /dev/null\n+++ b/tabular_predDB/python_utils/inference_utils.py\n@@ -0,0 +1,136 @@\n+import sys\n+import copy\n+from collections import Counter\n+\n+from scipy.misc import logsumexp\n+import numpy\n+import random\n+import math\n+\n+import tabular_predDB.cython_code.ContinuousComponentModel as CCM\n+import tabular_predDB.cython_code.MultinomialComponentModel as MCM\n+import tabular_predDB.python_utils.sample_utils as su\n+\n+import pdb\n+\n+\n+def mutual_information_to_linfoot(MI):\n+    return (1-math.exp(-2*MI))**0.5\n+\n+# return the estimated mutual information for each pair of columns on Q given\n+# the set of samples in X_Ls and X_Ds. Q is a list of tuples where each tuple\n+# contains X and Y, the columns to compare. \n+# Q = [(X_1, Y_1), (X_2, Y_2), ..., (X_n, Y_n)]\n+# Returns a list of list where each sublist is a set of MI\'s and Linfoots from\n+# each crosscat posterior sample. \n+# See tests/test_mutual_information.py and \n+# tests/test_mutual_information_vs_correlation.py for useage examples\n+def mutual_information(M_c, X_Ls, X_Ds, Q, n_samples=1000):\n+    #\n+    assert(len(X_Ds) == len(X_Ls))\n+    n_postertior_samples = len(X_Ds)\n+\n+    n_rows = len(X_Ds[0][0])\n+    n_cols = len(M_c[\'column_metadata\'])\n+\n+    MI = []\n+    Linfoot = []\n+\n+    get_next_seed = lambda: random.randrange(32767)\n+\n+    for query in Q:\n+        assert(len(query) == 2)\n+        assert(query[0] >= 0 and query[0] < n_cols)\n+        assert(query[1] >= 0 and query[1] < n_cols)\n+\n+        X = query[0]\n+        Y = query[1]\n+\n+        MI_sample = []\n+        Linfoot_sample = []\n+        for sample in range(n_postertior_samples):\n+            \n+            X_L = X_Ls[sample]\n+            X_D = X_Ds[sample]\n+\n+            MI_s = estimiate_MI_sample(X, Y, M_c, X_L, X_D, get_next_seed, n_samples=n_samples)\n+\n+            linfoot = mutual_information_to_linfoot(MI_s)\n+            if not linfoot >= 0 or not linfoot <= 1:\n+                pdb.set_trace()\n+\n+            MI_sample.append(MI_s)\n+\n+            Linfoot_sample.append(linfoot)\n+\n+        MI.append(MI_sample)\n+        Linfoot.append(Linfoot_sample)\n+\n+         \n+    assert(len(MI) == len(Q))\n+    assert(len(Linfoot) == len(Q))\n+\n+    return MI, Linfoot\n+\n+# estimates the mutual information part for columns X and Y on row.\n+def estimiate_MI_sample(X, Y, M_c, X_L, X_D, get_next_seed, n_samples=1000):\n+\n+    \n+    get_view_index = lambda which_column: X_L[\'column_partition\'][\'assignments\'][which_column]\n+    # get_cluster_index = lambda which_view, which_row : X_D[which_view][which_row]\n+\n+    view_X = get_view_index(X)\n+    view_Y = get_view_index(Y)\n+\n+    # independent\n+    if view_X != view_Y:\n+        return 0\n+\n+    # get cluster log_ps only for existing clusters\n+    view_state = X_L[\'view_state\'][view_X]\n+    cluster_logps = su.determine_cluster_crp_logps(view_state)\n+    cluster_crps = numpy.exp(cluster_logps)\n+    n_clusters = len(cluster_crps)\n+\n+    # get components models for each cluster for X and Y\n+    component_models_X = [0]*n_clusters\n+    component_models_Y = [0]*n_clusters\n+    for i in range(n_clusters):\n+        cluster_models = su.create_cluster_model_from_X_L(M_c, X_L, view_X, i)\n+        component_models_X[i] = cluster_models[X]\n+        component_models_Y[i] = cluster_models[Y]\n+\n+    MI = 0\n+    for i in range(n_samples):\n+        # draw a cluster \n+        cluster_idx = numpy.nonzero(numpy.random.multinomial(1, cluster_crps))[0][0]\n+\n+        x = component_models_X[cluster_idx].get_draw(get_next_seed())\n+        y = component_models_Y[cluster_idx].get_draw(get_next_seed())\n+\n+        # calculate marginal logs\n+        Pxy = numpy.zeros(n_clusters)\n+        Px = numpy.zeros(n_clusters)\n+        Py = numpy.zeros(n_clusters)\n+\n+        for j in range(n_clusters):\n+            Px[j] = component_models_X[j].calc_element_predictive_logp(x)\n+            Py[j] = component_models_Y[j].calc_element_predictive_logp(y)\n+            Pxy[j] = Px[j] + Py[j] + cluster_logps[j]\n+            Px[j] += cluster_logps[j]\n+            Py[j] += cluster_logps[j]\n+        \n+        Px = logsumexp(Px)\n+        Py = logsumexp(Py)\n+        Pxy = logsumexp(Pxy)\n+\n+        MI += Pxy - Px - Py\n+\n+\n+    MI /= n_samples\n+\n+    # ignore MI < 0\n+    if MI <= 0:\n+        MI = 0\n+        \n+    return MI\n\\ No newline at end of file\ndiff --git a/tabular_predDB/python_utils/mutual_information_test_utils.py b/tabular_predDB/python_utils/mutual_information_test_utils.py\nnew file mode 100644\nindex 00000000..3fab7625\n--- /dev/null\n+++ b/tabular_predDB/python_utils/mutual_information_test_utils.py\n@@ -0,0 +1,104 @@\n+import tabular_predDB.python_utils.data_utils as du\n+import tabular_predDB.cython_code.State as State\n+import random\n+import numpy\n+\n+# Generates a num_rows by num_cols array of data with covariance matrix I^{num_cols}*corr\n+def generate_correlated_data(num_rows, num_cols, means, corr, seed=0):\n+\tassert(corr <= 1 and corr >= 0)\n+\tassert(num_cols == len(means))\n+\n+\tnumpy.random.seed(seed=seed)\n+\n+\tmu = numpy.array(means)\n+\tsigma = numpy.ones((num_cols,num_cols),dtype=float)*corr\n+\tfor i in range(num_cols):\n+\t\tsigma[i,i] = 1 \n+\tX = numpy.random.multivariate_normal(mu, sigma, num_rows)\n+\n+\treturn X\n+\n+def generate_correlated_state(num_rows, num_cols, num_views, num_clusters, mean_range, corr, seed=0):\n+\t#\n+\n+\tassert(num_clusters <= num_rows)\n+\tassert(num_views <= num_cols)\n+\tT = numpy.zeros((num_rows, num_cols))\n+\n+\trandom.seed(seed)\n+\tnumpy.random.seed(seed=seed)\n+\tget_next_seed = lambda : random.randrange(2147483647)\n+\n+\t# generate an assignment of columns to views (uniform)\n+\tcols_to_views = range(num_views)\n+\tview_counts = numpy.ones(num_views, dtype=int)\n+\tfor i in range(num_views, num_cols):\n+\t\tr = random.randrange(num_views)\n+\t\tcols_to_views.append(r)\n+\t\tview_counts[r] += 1\n+\n+\trandom.shuffle(cols_to_views)\n+\n+\tassert(len(cols_to_views) == num_cols)\n+\tassert(max(cols_to_views) == num_views-1)\n+\n+\t# for each view, generate an assignment of rows to num_clusters\n+\trow_to_clusters = []\n+\tcluster_counts = []\n+\tfor view in range(num_views):\n+\t\trow_to_cluster = range(num_clusters)\n+\t\tcluster_counts_i = numpy.ones(num_clusters,dtype=int)\n+\t\tfor i in range(num_clusters, num_rows):\n+\t\t\tr = random.randrange(num_clusters)\n+\t\t\trow_to_cluster.append(r)\n+\t\t\tcluster_counts_i[r] += 1\n+\n+\t\trandom.shuffle(row_to_cluster)\n+\n+\t\tassert(len(row_to_cluster) == num_rows)\n+\t\tassert(max(row_to_cluster) == num_clusters-1)\n+\n+\t\trow_to_clusters.append(row_to_cluster)\n+\t\tcluster_counts.append(cluster_counts_i)\n+\n+\tassert(len(row_to_clusters) == num_views)\n+\n+\t# generate the correlated data\n+\tfor view in range(num_views):\n+\t\tfor cluster in range(num_clusters):\n+\t\t\tcell_cols = view_counts[view]\n+\t\t\tcell_rows = cluster_counts[view][cluster]\n+\t\t\tmeans = numpy.random.uniform(-mean_range/2.0,mean_range/2.0,cell_cols)\n+\t\t\tX =  generate_correlated_data(cell_rows, cell_cols, means, corr, seed=get_next_seed())\n+\t\t\t# get the indices of the columns in this view\n+\t\t\tcol_indices = numpy.nonzero(numpy.array(cols_to_views)==view)[0]\n+\t\t\t# get the indices of the rows in this view and this cluster\n+\t\t\trow_indices = numpy.nonzero(numpy.array(row_to_clusters[view])==cluster)[0]\n+\t\t\t# insert the data\n+\t\t\tfor col in range(cell_cols):\n+\t\t\t\tfor row in range(cell_rows):\n+\t\t\t\t\tr = row_indices[row]\n+\t\t\t\t\tc = col_indices[col]\n+\t\t\t\t\tT[r,c] = X[row,col]\n+\n+\n+\tM_c = du.gen_M_c_from_T(T)\n+\tM_r = du.gen_M_r_from_T(T)\n+\tX_L, X_D = generate_X_L_and_X_D(T, M_c, cols_to_views, row_to_clusters, seed=get_next_seed())\n+\n+\treturn  T, M_c, M_r, X_L, X_D, cols_to_views\n+\n+def generate_X_L_and_X_D(T, M_c, cols_to_views, row_to_clusters, seed=0):\n+\tstate = State.p_State(M_c, T, SEED=seed)\n+\tX_L = state.get_X_L()\n+\n+\t# insert assigment into X_L (this is not a valid X_L because the counts and \n+\t# suffstats will be wrong)\n+\tX_L[\'column_partition\'][\'assignments\'] = cols_to_views\n+\tstate = State.p_State(M_c, T, X_L=X_L, X_D=row_to_clusters, SEED=seed)\n+\n+\tX_L = state.get_X_L()\n+\tX_D = state.get_X_D()\n+\n+\treturn X_L, X_D\n+\ndiff --git a/tabular_predDB/tests/mi_tests/parse_mi.py b/tabular_predDB/tests/mi_tests/parse_mi.py\nnew file mode 100644\nindex 00000000..a4ab3b37\n--- /dev/null\n+++ b/tabular_predDB/tests/mi_tests/parse_mi.py\n@@ -0,0 +1,150 @@\n+import numpy\n+import csv\n+import os\n+import ast\n+import pickle\n+\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.xnet_utils as xu\n+\n+import pdb\n+\n+def is_hadoop_file(filename):\n+\tname, extension = os.path.splitext(filename)\n+\tif extension is \'gz\':\n+\t\treturn True\n+\telse:\n+\t\treturn False\n+\n+def parse_line(test, mi, linfoot):\n+\tindex = test[\'id\']\n+\tnum_rows = test[\'num_rows\']\n+\tnum_cols = test[\'num_cols\']\n+\tnum_views = test[\'num_views\']\n+\tnum_clusters = test[\'num_clusters\']\n+\tcorr = test[\'corr\']\n+\t\n+\trow = [index, num_rows, num_cols, num_views, num_clusters, corr, mi, linfoot]\n+\n+\treturn row\n+\n+def hadoop_to_dict_generator(test_key_file_object):\n+\t# return read cursor to the start (or this generator cannot be called again)\n+\ttest_key_file_object.seek(0)\n+\tfor line in test_key_file_object:\n+\t\tdict_line = xu.parse_hadoop_line(line)\n+\t\tyield dict_line\n+\t\n+def parse_data_to_csv(test_key_filename, params_dict, n_tests, output_filename):\n+\t# open input file and convert to list of dicts\n+\ttest_key_file_object = open(test_key_filename, \'rb\')\n+\tinput_lines = hadoop_to_dict_generator(test_key_file_object)\n+\n+\t# open output file and convert to list of dicts\n+\toutput_file_object = open(output_filename, \'rb\')\n+\tresults = hadoop_to_dict_generator(output_file_object)\n+\t\n+\tn_datasets = params_dict[\'n_datasets\']\n+\tn_samples = params_dict[\'n_samples\']\n+\n+\theader = [\'id\', \'num_rows\', \'num_cols\', \'num_views\', \'num_clusters\', \'corr\',\'MI\',\'Linfoot\']\n+\n+\t# data_mi = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\t# data_linfoot = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\t# counts = [[[0] for i in range(n_datasets)] for i in range(n_tests)]\n+\n+\tdata_mi = [0.0]*n_tests\n+\tdata_linfoot = [0.0]*n_tests\n+\tcounts = [0.0]*n_tests\n+\n+\n+\tfor result in results:\n+\t\tres = result[1] # because it\'s a tuple with an id at index 0\n+\t\ttest_idx = res[\'id\']\n+\t\ttest_dataset = res[\'dataset\']\n+\t\ttest_sample = res[\'sample\']\n+\t\t\n+\t\tdata_mi[test_idx] += float(res[\'mi\']) \n+\t\tdata_linfoot[test_idx] += float(iu.mutual_information_to_linfoot(res[\'mi\']))\n+\t\tcounts[test_idx] += 1.0\n+\t\n+\tfor test_ids in range(n_tests):\n+\t\tdata_mi[test_idx] /= counts[test_idx]\n+\t\tdata_linfoot[test_idx] /= counts[test_idx]\n+\n+\t# # calculate the mean over samples\n+\t# for test in range(n_tests):\n+\t\t\n+\t# \tfor dataset in range(n_datasets):\n+\t# \t\ttry:\n+\t# \t\t\tdata_mi[test][dataset] = numpy.array(data_mi[test][dataset],dtype=float)\n+\t# \t\texcept ValueError:\n+\t# \t\t\tpdb.set_trace()\n+\n+\t# \t\ttry:\n+\t# \t\t\tdata_mi[test][dataset] = numpy.mean(data_mi[test][dataset],axis=0)\n+\t# \t\texcept TypeError:\n+\t# \t\t\tpdb.set_trace()\n+\n+\t# \t\tdata_linfoot[test][dataset] = mi_to_linfoot(data_mi[test][dataset])\n+\n+\t# \t\tdata_mi[test][dataset] = numpy.mean(data_mi[test][dataset])\n+\t# \t\tdata_linfoot[test][dataset] = numpy.mean(data_linfoot[test][dataset])\n+\n+\t# \t# now calculate the mean over datasets\n+\t# \tdata_mi[test] = numpy.mean(numpy.array(data_mi[test]))\n+\t# \tdata_linfoot[test] = numpy.mean(numpy.array(data_linfoot[test]))\n+\t\n+\tname, extension = os.path.splitext(output_filename)\n+\n+\toutfile = name + \'.csv\'\n+\n+\twith open(outfile,\'w\') as csvfile:\n+\t\tcsvwriter = csv.writer(csvfile,delimiter=\',\')\n+\t\tcsvwriter.writerow(header)\n+\t\tcurrent_idx = -1\n+\t\tfor test in input_lines:\n+\t\t\tres = test[1]\n+\t\t\ttest_idx = res[\'id\']\n+\t\t\tif test_idx != current_idx:\n+\t\t\t\tcurrent_idx = test_idx\n+\t\t\t\tline = parse_line(res, data_mi[test_idx], data_linfoot[test_idx])\n+\t\t\t\tcsvwriter.writerow(line)\n+\n+def mi_to_linfoot(mi):\n+\t#\n+\t# linfoot = numpy.zeros(mi.shape)\n+\t# if len(mi.shape) == 1:\n+\t# \tfor entry in range(mi.size):\n+\t# \t\tlinfoot[entry] = iu.mutual_information_to_linfoot(mi[entry])\n+\t# else:\n+\t# \tfor r in range(mi.shape[0]):\n+\t# \t\tfor c in range(mi.shape[1]):\n+\t# \t\t\tlinfoot[r,c] = iu.mutual_information_to_linfoot(mi[r,c])\n+\n+\n+\t# return linfoot\n+\treturn [iu.mutual_information_to_linfoot(m) for m in mi]\n+\n+if __name__ == "__main__":\n+\n+\timport argparse\n+\tparser = argparse.ArgumentParser()\n+\n+\tparser.add_argument(\'--key_filename\', type=str)\n+\tparser.add_argument(\'--params_filename\', type=str)\n+\tparser.add_argument(\'--n_tests\', type=int)\n+\tparser.add_argument(\'--output_filename\', type=str)\n+\n+\targs = parser.parse_args()\n+\n+\tkey_filename = args.key_filename\n+\toutput_filename = args.output_filename\n+\tn_tests = args.n_tests\n+\tparams_filename = args.params_filename\n+\tparams_dict = pickle.load( open( params_filename, "rb" ))\n+\n+\n+\n+\tparse_data_to_csv(key_filename, params_dict, n_tests, output_filename)\n+\ndiff --git a/tabular_predDB/tests/mi_tests/run_mi_test.py b/tabular_predDB/tests/mi_tests/run_mi_test.py\nnew file mode 100644\nindex 00000000..060af1b1\n--- /dev/null\n+++ b/tabular_predDB/tests/mi_tests/run_mi_test.py\n@@ -0,0 +1,224 @@\n+import itertools as it\n+import time\n+\n+import argparse\n+import numpy\n+import tempfile \n+import parse_mi\n+import pickle\n+\n+import os\n+\n+import tabular_predDB.cython_code.State as State\n+import tabular_predDB.python_utils.hadoop_utils as hu\n+import tabular_predDB.python_utils.file_utils as fu\n+import tabular_predDB.python_utils.xnet_utils as xu\n+import tabular_predDB.LocalEngine as LE\n+import tabular_predDB.HadoopEngine as HE\n+\n+import run_mi_test_local\n+\n+def generate_hadoop_dicts(which_kernels, impute_run_parameters, args_dict):\n+    for which_kernel in which_kernels:\n+        kernel_list = (which_kernel, )\n+        dict_to_write = dict(impute_run_parameters)\n+        dict_to_write.update(args_dict)\n+        # must write kernel_list after update\n+        dict_to_write[\'kernel_list\'] = kernel_list\n+        yield dict_to_write\n+\n+def write_hadoop_input(input_filename, impute_run_parameters, SEED):\n+    # prep settings dictionary\n+    impute_analyze_args_dict = xu.default_analyze_args_dict\n+    impute_analyze_args_dict[\'command\'] = \'impute_analyze\'\n+    with open(input_filename, \'a\') as out_fh:\n+        xu.write_hadoop_line(out_fh, key=SEED, dict_to_write=impute_run_parameters)\n+\n+# # example of how to run a simple test run on hadoop (xdata VM)\n+# python run_mi_test.py --num_datasets 5 --num_samples 10 --which_engine_binary /user/bigdata/SSCI/be_mi_tests_00.jar \\\n+# --num_rows_list 10 --num_cols_list 2 --num_clusters_list 2 --num_views_list 1 --corr_list .1 .99 -do_remote\n+# # example of a full run\n+# python run_mi_test.py --which_engine_binary /user/bigdata/SSCI/be_mi_tests_00.jar -do_remote\n+\n+# Run\n+if __name__ == \'__main__\':\n+\n+\tdefault_num_rows_list = [100, 500, 1000] \n+\tdefault_num_cols_list = [2, 4, 8, 16]\t\n+\tdefault_num_clusters_list = [10, 25, 50]\t\n+\tdefault_num_views_list = [1, 2, 4, 8, 16]\n+\tdefault_correlation_list = [.1, .5, .9]\n+\n+\t# default_num_rows_list = [10] \n+\t# default_num_cols_list = [2, 4, 8]\t\n+\t# default_num_clusters_list = [2]\t\n+\t# default_num_views_list = [1]\n+\t# default_correlation_list = [.9]\n+\t\n+\t#\n+\tparser = argparse.ArgumentParser()\n+\tparser.add_argument(\'--gen_seed\', type=int, default=0)\n+\tparser.add_argument(\'--num_datasets\', type=int, default=5)\n+\tparser.add_argument(\'--num_samples\', type=int, default=50)\n+\tparser.add_argument(\'--burn_in\', type=int, default=250)\n+\tparser.add_argument(\'--which_engine_binary\', type=str,\n+\t        default=HE.default_engine_binary)\n+\tparser.add_argument(\'--jobtracker_uri\', type=str,\n+\t        default=HE.default_jobtracker_uri)\n+\t\tparser.add_argument(\'--hdfs_uri\', type=str,\n+\t        default=HE.default_hdfs_uri)\n+\tparser.add_argument(\'--which_engine_binary\', type=str,\n+\t        default=HE.default_engine_binary)\n+\tparser.add_argument(\'-do_local\', action=\'store_true\')\n+\tparser.add_argument(\'-do_remote\', action=\'store_true\')\n+\tparser.add_argument(\'--num_rows_list\', type=int, nargs=\'*\',\n+\t        default=default_num_rows_list)\n+\tparser.add_argument(\'--num_cols_list\', type=int, nargs=\'*\',\n+\t        default=default_num_cols_list)\n+\tparser.add_argument(\'--num_clusters_list\', type=int, nargs=\'*\',\n+\t        default=default_num_clusters_list)\n+\tparser.add_argument(\'--num_views_list\', type=int, nargs=\'*\',\n+\t        default=default_num_views_list)\n+\tparser.add_argument(\'--corr_list\', type=float, nargs=\'*\',\n+\t        default=default_correlation_list)\n+\n+\t#\n+\targs = parser.parse_args()\n+\tdata_seed = args.gen_seed\n+\tdo_local = args.do_local\n+\tdo_remote = args.do_remote\n+\tburn_in = args.burn_in\n+\tjobtracker_uri = args.jobtracker_uri\n+\thdfs_uri = args.hdfs_uri\n+\tnum_samples = args.num_samples\n+\tnum_datasets = args.num_datasets\n+\tnum_rows_list = args.num_rows_list\n+\tnum_cols_list = args.num_cols_list\n+\tnum_clusters_list = args.num_clusters_list\n+\tnum_views_list = args.num_views_list\n+\tcorr_list = args.corr_list\n+\twhich_engine_binary = args.which_engine_binary\n+\t#\n+\tprint \'using burn_in: %i\' % burn_in\n+\tprint \'using num_rows_list: %s\' % num_rows_list\n+\tprint \'using num_cols_list: %s\' % num_cols_list\n+\tprint \'using num_clusters_list: %s\' % num_clusters_list\n+\tprint \'using num_views_list: %s\' % num_views_list\n+\tprint \'using corr_list: %s\' % corr_list\n+\tprint \'using engine_binary: %s\' % which_engine_binary\n+\ttime.sleep(2)\n+\n+\tdirname = \'mi_analysis\'\n+\tfu.ensure_dir(dirname)\n+\tdirectory_path = tempfile.mkdtemp(prefix=\'mi_analysis_\',\n+                                dir=dirname)\n+\n+\tprint \'output sent to %s\' % directory_path\n+\n+\toutput_path = os.path.join(directory_path, \'output\')\n+\toutput_filename = os.path.join(directory_path, \'hadoop_output\')\n+\ttable_data_filename = os.path.join(directory_path, \'table_data.pkl.gz\')\n+\n+\tassert(os.path.exists(directory_path))\n+\t\n+\tinput_filename = os.path.join(directory_path, "hadoop_input")\n+\tparams_filename = os.path.join(directory_path, "test_params.pkl")\n+\tkey_filename = os.path.join(directory_path, "test_key.pkl")\n+\t\n+\t# create a parameters dict\n+\tparams_dict = {\n+\t\t\'n_rows\' \t  : num_rows_list,\n+\t\t\'n_clusters\'  : num_clusters_list,\n+\t\t\'n_cols\' \t  : num_cols_list,\n+\t\t\'n_views\' \t  : num_views_list,\n+\t\t\'corr\' \t  \t  : corr_list,\n+\t\t\'n_datasets\'  : num_datasets,\n+\t\t\'n_samples\'   : num_samples,\n+\t\t\'n_datasets\'  : num_datasets,\n+\t\t\'burn_in\' \t  : burn_in\n+\t}\n+\n+\t# save the params file as pickle\n+\ttry:\n+\t\tpd_file = open( params_filename, "wb" )\n+\texcept IOError as err:\n+\t\tprint "Could not create %s. " % params_filename, err\n+\t\traise\n+\n+\tpickle.dump( params_dict, pd_file )\n+\tpd_file.close()\n+\n+\t# cartesian product of test parameters. \n+\ttests = list(it.product(*[num_rows_list, num_clusters_list, num_cols_list, num_views_list, corr_list]))\n+\n+\ttestlist = []\n+\t\n+\tprint "Writing tests file."\t\n+\ttest_idx = 0\n+\tfor n_rows, n_clusters, n_cols, n_views, corr in tests:\t\n+\t\tif n_rows >= n_clusters and n_cols >= n_views:\n+\t\t\tfor dataset in range(num_datasets):\n+\t\t\t\tfor sample in range(num_samples):\n+\t\t\t\t\timpute_run_parameters = dict(\n+\t\t\t\t\t\t\tid=test_idx,\n+\t\t\t\t\t\t\tdataset=dataset,\n+\t\t\t\t\t\t\tsample=sample,\n+\t\t\t\t\t\t\tnum_clusters=n_clusters,\n+\t\t\t\t\t\t\tnum_rows=n_rows,\n+\t\t\t\t\t\t\tnum_cols=n_cols,\n+\t\t\t\t\t\t\tnum_views=n_views,\n+\t\t\t\t\t\t\tcorr=corr,\n+\t\t\t\t\t\t\tburn_in=burn_in,\n+\t\t\t\t\t\t\tSEED=data_seed+dataset,\n+\t\t\t\t\t\t\tCCSEED=data_seed+dataset+sample\n+\t\t\t\t\t\t)\n+\n+\t\t\t\t\twrite_hadoop_input(input_filename, impute_run_parameters, SEED=data_seed+dataset)\n+\t\t\t\t\tif do_local:\n+\t\t\t\t\t\ttestlist.append(impute_run_parameters)\n+\t\t\ttest_idx += 1\n+\t\n+\tprint "Done."\n+\t\n+\t# table data is empty because we generate it in the mapper\n+\ttable_data=dict(T=[],M_c=[],X_L=[],X_D=[])\n+\tfu.pickle(table_data, table_data_filename)\n+\n+\t#####################\n+\tif do_local:\n+\t\toutput_filename = os.path.join(directory_path, "output_local")\n+\t\toutput_file_object = open(output_filename, \'ab\')\n+\t\twith open(input_filename,\'rb\') as infile:\n+\t\t\tfor line in infile:\n+\t\t\t\tkey, test_dict = xu.parse_hadoop_line(line)\n+\t\t\t\tret_dict = run_mi_test_local.run_mi_test_local(test_dict)\n+\t\t\t\txu.write_hadoop_line(output_file_object, key, ret_dict)\n+\t\t\t\tprint "%s\\n\\t%s" % (str(test_dict), str(ret_dict))\n+\n+\t\toutput_file_object.close()\n+\t\t# generate the csv\n+\t\tparse_mi.parse_data_to_csv(input_filename, params_dict, test_idx, output_filename)\n+\t\tprint "Done."\n+\telif do_remote:\n+\t\t# generate the massive hadoop files\n+\t\thadoop_engine = HE.HadoopEngine(output_path=output_path,\n+                                    input_filename=input_filename,\n+                                    table_data_filename=table_data_filename,\n+                                    which_engine_binary=which_engine_binary,\n+                                    hdfs_uri=hdfs_uri,\n+                                    jobtracker_uri=jobtracker_uri,\n+                                    )\n+\t\n+\t\txu.write_support_files(table_data, hadoop_engine.table_data_filename,\n+\t                              dict(command=\'mi_analyze\'), hadoop_engine.command_dict_filename)\n+\t\tt_start = time.time()\n+\t\thadoop_engine.send_hadoop_command(n_tasks=len(testlist))\n+\t\twas_successful = hadoop_engine.get_hadoop_results()\n+\t\tif was_successful:\n+\t\t\tt_end = time.time()\n+\t\t\tt_total = t_end-t_start\n+\t\t\tprint "That took %i seconds." % t_total\n+\t\t\thu.copy_hadoop_output(hadoop_engine.output_path, output_filename)\n+\t\t\tparse_mi.parse_data_to_csv(input_filename, params_dict, test_idx, output_filename)\n+\t\telse:\n+\t\t\tprint "Hadoop job was NOT successful. Check %s" % output_path\n\\ No newline at end of file\ndiff --git a/tabular_predDB/tests/mi_tests/run_mi_test_local.py b/tabular_predDB/tests/mi_tests/run_mi_test_local.py\nnew file mode 100644\nindex 00000000..51dab355\n--- /dev/null\n+++ b/tabular_predDB/tests/mi_tests/run_mi_test_local.py\n@@ -0,0 +1,66 @@\n+import itertools\n+import random\n+import numpy\n+import tabular_predDB.LocalEngine as LE\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.mutual_information_test_utils as mitu\n+\n+def run_mi_test_local(data_dict):\n+\n+    gen_seed = data_dict[\'SEED\']\n+    crosscat_seed = data_dict[\'CCSEED\']\n+    num_clusters = data_dict[\'num_clusters\']\n+    num_cols = data_dict[\'num_cols\']\n+    num_rows = data_dict[\'num_rows\']\n+    num_views = data_dict[\'num_views\']\n+    corr = data_dict[\'corr\']\n+    burn_in = data_dict[\'burn_in\']\n+    mean_range = float(num_clusters)*2.0\n+\n+    # 32 bit signed int\n+    random.seed(gen_seed)\n+    get_next_seed = lambda : random.randrange(2147483647)\n+\n+    # generate the stats\n+    T, M_c, M_r, X_L, X_D, view_assignment = mitu.generate_correlated_state(num_rows,\n+        num_cols, num_views, num_clusters, mean_range, corr, seed=gen_seed);\n+\n+    table_data = dict(T=T,M_c=M_c)\n+\n+    engine = LE.LocalEngine(crosscat_seed)\n+    X_L_prime, X_D_prime = engine.analyze(M_c, T, X_L, X_D, n_steps=burn_in) \n+\n+    X_L = X_L_prime\n+    X_D = X_D_prime\n+\n+    view_assignment = numpy.array(X_L[\'column_partition\'][\'assignments\'])\n+ \n+    # for each view calclate the average MI between all pairs of columns\n+    n_views = max(view_assignment)+1\n+    MI = []\n+    Linfoot = []\n+    queries = []\n+    MI = 0.0\n+    pairs = 0.0\n+    for view in range(n_views):\n+        columns_in_view = numpy.nonzero(view_assignment==view)[0]\n+        combinations = itertools.combinations(columns_in_view,2)\n+        for pair in combinations:\n+            any_pairs = True\n+            queries.append(pair)\n+            MI_i, Linfoot_i = iu.mutual_information(M_c, [X_L], [X_D], [pair], n_samples=1000)\n+            MI += MI_i[0][0]\n+            pairs += 1.0\n+\n+    \n+    if pairs > 0.0:\n+        MI /= pairs\n+\n+    ret_dict = dict(\n+        id=data_dict[\'id\'],\n+        dataset=data_dict[\'dataset\'],\n+        sample=data_dict[\'sample\'],\n+        mi=MI,\n+        )\n+\n+    return ret_dict\n\\ No newline at end of file\ndiff --git a/tabular_predDB/tests/test_mutual_information.py b/tabular_predDB/tests/test_mutual_information.py\nnew file mode 100644\nindex 00000000..5230a02a\n--- /dev/null\n+++ b/tabular_predDB/tests/test_mutual_information.py\n@@ -0,0 +1,213 @@\n+import numpy\n+import pylab as pl\n+import tabular_predDB.python_utils.sample_utils as su\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.data_utils as du\n+import tabular_predDB.cython_code.State as State\n+\n+import random\n+import math\n+\n+import pdb\n+\n+def ring(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tangle = random.uniform(0,2*math.pi)\n+\t\tdistance = random.uniform(1,1.5)\n+\t\tX[i,0] = math.cos(angle)*distance\n+\t\tX[i,1] = math.sin(angle)*distance\n+\n+\treturn X\n+\n+def circle(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tangle = random.uniform(0,2*math.pi)\n+\t\tdistance = random.uniform(0,1.5)\n+\t\tX[i,0] = math.cos(angle)*distance\n+\t\tX[i,1] = math.sin(angle)*distance\n+\n+\treturn X\n+\n+def square(n=200):\n+\n+\tX = numpy.zeros((n,2))\n+\tfor i in range(n):\n+\t\tx = random.uniform(-1,1)\n+\t\ty = random.uniform(-1,1)\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def diamond(n=200):\n+\n+\tX = square(n=n)\n+\tfor i in range(n):\n+\t\tangle = math.atan(X[i,1]/X[i,0])\n+\t\tangle += math.pi/4 \n+\t\thyp = (X[i,0]**2.0+X[i,1]**2.0)**.5\n+\t\tx = math.cos(angle)*hyp\n+\t\ty = math.sin(angle)*hyp\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def four_dots(n=200):\n+\tX = numpy.zeros((n,2))\n+\tnb = n/4\n+\tmx = [ -1, 1, -1, 1]\n+\tmy = [ -1, -1, 1, 1]\n+\ts = .25\n+\t\n+\tfor i in range(n):\n+\t\tn = random.randrange(4)\n+\t\tx = random.normalvariate(mx[n], s)\n+\t\ty = random.normalvariate(my[n], s)\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\t\t\n+\treturn X\n+\n+def correlated(r,n=200):\n+\tX = numpy.random.multivariate_normal([0,0], [[1, r],[r, 1]], n)\n+\treturn X\n+\n+def sample_from_view(M_c, X_L, X_D, get_next_seed):\n+    \n+    view_col = X_L[\'column_partition\'][\'assignments\'][0]\n+    view_col2 = X_L[\'column_partition\'][\'assignments\'][1]\n+\n+    same_view = True\n+    if view_col2 != view_col:\n+    \tsame_view = False\n+\n+    view_state = X_L[\'view_state\'][view_col]\n+    view_state2 = X_L[\'view_state\'][view_col2]\n+\n+    cluster_crps = numpy.exp(su.determine_cluster_crp_logps(view_state))\n+    cluster_crps2 = numpy.exp(su.determine_cluster_crp_logps(view_state2))\n+\n+    assert( math.fabs(numpy.sum(cluster_crps) - 1) < .00000001 )\n+\n+    samples = numpy.zeros((n,2))\n+\n+    \n+    cluster_idx1 = numpy.nonzero(numpy.random.multinomial(1, cluster_crps))[0][0]\n+    cluster_model1 = su.create_cluster_model_from_X_L(M_c, X_L, view_col, cluster_idx1)\n+\n+    if same_view:\n+    \tcluster_idx2 = cluster_idx1\n+    \tcluster_model2 = cluster_model1\n+    else:\n+    \tcluster_idx2 = numpy.nonzero(numpy.random.multinomial(1, cluster_crps2))[0][0]\n+    \tcluster_model2 = su.create_cluster_model_from_X_L(M_c, X_L, view_col2, cluster_idx2)\n+\n+    component_model1 = cluster_model1[0]\n+    x = component_model1.get_draw(get_next_seed())\n+\n+    component_model2 = cluster_model2[1]\n+    y = component_model2.get_draw(get_next_seed())\n+        \n+    return x, y\n+\n+def sample_data_from_crosscat(M_c, X_Ls, X_Ds, get_next_seed, n):\n+\n+\tX = numpy.zeros((n,2))\n+\tn_samples = len(X_Ls)\n+\t\n+\tfor i in range(n):\n+\t\tcc = random.randrange(n_samples)\n+\t\tx, y = sample_from_view(M_c, X_Ls[cc], X_Ds[cc], get_next_seed)\n+\t\t\n+\t\tX[i,0] = x\n+\t\tX[i,1] = y\n+\n+\treturn X\n+\n+def do_test(which_plot, max_plots, n, burn_in, cc_samples, which_test, correlation=0, do_plot=False):\n+\tif which_test is "correlated":\n+\t\tX = correlated(correlation, n=n)\n+\telif which_test is "square":\n+\t\tX = square(n=n)\n+\telif which_test is "ring":\n+\t\tX = ring(n=n)\n+\telif which_test is "circle":\n+\t\tX = circle(n=n)\n+\telif which_test is "diamond":\n+\t\tX = diamond(n=n)\n+\telif which_test is "blob":\n+\t\tX = correlated(0.0, n=n)\n+\telif which_test is "dots":\n+\t\tX = four_dots(n=n)\n+\telif which_test is "mixed":\n+\t\tX = numpy.vstack((correlated(.95, n=n/2),correlated(0, n=n/2)))\n+\n+\tget_next_seed = lambda : random.randrange(32000)\n+\n+\t# Build a state\n+\tM_c = du.gen_M_c_from_T(X.tolist())\n+\tstate = State.p_State(M_c, X.tolist())\n+\tX_Ls = []\n+\tX_Ds = []\n+\t\n+\t# collect crosscat samples\n+\tfor _ in range(cc_samples):\n+\t\tstate = State.p_State(M_c, X.tolist())\n+\t\tstate.transition(n_steps=burn_in)\n+\t\tX_Ds.append(state.get_X_D())\n+\t\tX_Ls.append(state.get_X_L())\n+\n+\tSX = sample_data_from_crosscat(M_c, X_Ls, X_Ds, get_next_seed, n)\n+\n+\tif do_plot:\n+\t\tpl.subplot(2,max_plots,which_plot)\n+\t\tpl.scatter(X[:,0],X[:,1],c=\'blue\',alpha=.5)\n+\t\tpl.title("Original data")\n+\t\tpl.subplot(2,max_plots,max_plots+which_plot)\n+\t\tpl.scatter(SX[:,0],SX[:,1],c=\'red\',alpha=.5)\n+\t\tpl.title("Sampled data")\n+\t\tpl.show\n+\n+\treturn M_c, X_Ls, X_Ds\n+\n+def MI_test(n, burn_in, cc_samples, which_test, n_MI_samples=500, correlation=0):\n+\tM_c, X_Ls, X_Ds = do_test(0, 0, n, burn_in, cc_samples, "correlated", correlation=correlation, do_plot=False)\n+\t# query column 0 and 1\n+\tMI, Linfoot = su.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=n_MI_samples)\n+\n+\tMI = numpy.mean(MI)\n+\tLinfoot = numpy.mean(Linfoot)\n+\t\n+\tif which_test == "correlated":\n+\t\ttest_strn = "Test: correlation (%1.2f), N: %i, burn_in: %i, samples: %i, MI_samples: %i\\n\\tMI: %f, Linfoot %f" % (correlation, n, burn_in, cc_samples, n_MI_samples, MI, Linfoot)\n+\telse:\n+\t\ttest_strn = "Test: %s, N: %i, burn_in: %i, samples: %i, MI_samples: %i\\n\\tMI: %f, Linfoot %f" % (which_test, n, burn_in, cc_samples, n_MI_samples, MI, Linfoot)\n+\t\t\n+\n+\tprint test_strn\n+\treturn test_strn\n+\n+do_plot = False\n+n_mi_samples = 500\n+\n+N = [10, 100, 1000] \n+\n+burn_in = 200\n+cc_samples = 10\n+\n+print " "\n+for n in N:\n+\t\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.3)\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.6)\n+\tstrn = MI_test(n, burn_in, cc_samples, "correlated", correlation=.9)\n+\tstrn = MI_test(n, burn_in, cc_samples, "ring")\n+\tstrn = MI_test(n, burn_in, cc_samples, "dots")\n+\tstrn = MI_test(n, burn_in, cc_samples, "mixed")\n+\n+\ndiff --git a/tabular_predDB/tests/test_mutual_information_vs_correlation.py b/tabular_predDB/tests/test_mutual_information_vs_correlation.py\nnew file mode 100644\nindex 00000000..20c8716f\n--- /dev/null\n+++ b/tabular_predDB/tests/test_mutual_information_vs_correlation.py\n@@ -0,0 +1,80 @@\n+import numpy\n+import pylab as pl\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.data_utils as du\n+import tabular_predDB.cython_code.State as State\n+\n+from scipy.stats import pearsonr as pearsonr\n+\n+import random\n+\n+def get_correlations(T, Q):\n+\tT = numpy.array(T)\n+\tcorr = []\n+\n+\tfor query in Q:\n+\t\tr, p = scipy.stats.pearsonr(T[:,Q[0]], T[:,Q[1]])\n+\t\tcorr.append(r)\n+\n+\treturn corr\n+\n+def gen_correlated_data( n, r, SEED=0 ):\n+\tnumpy.random.seed(SEED)\n+\tT = numpy.random.multivariate_normal([0,0],[[1,r],[r,1]],n)\n+\n+\treturn T\n+\n+get_next_seed = lambda : random.randrange(32000)\n+\n+correlations = [.0, .1, .2, .3, .4 , .5, .6, .7, .8, .9, 1.0]\n+N = [5, 10, 25, 50]\n+n_samples = 10\n+n_data_sets = 5\n+pl.figure()\n+burn_in = 200\n+\n+subplot = 0\n+for n in N:\n+\tsubplot += 1\n+\tnr = 0\n+\t\n+\tfor r in correlations:\n+\t\tfor d in range(n_data_sets): # 3 data sets\n+\t\t\t#\n+\t\t\tT = gen_correlated_data( n, r, SEED=get_next_seed())\n+\n+\t\t\tpr, p = pearsonr(T[:,0], T[:,1])\n+\n+\t\t\tprint "num_samples: %i, R: %f, d: %i. Actual R: %f" % (n, r, d+1, pr)\n+\n+\t\t\tM_c = du.gen_M_c_from_T(T)\n+\t\t\tX_Ls = []\n+\t\t\tX_Ds = []\n+\n+\t\t\tfor _ in range(n_samples):\n+\t\t\t\tstate = State.p_State(M_c, T)\n+\t\t\t\tstate.transition(n_steps=burn_in)\n+\t\t\t\tX_Ds.append(state.get_X_D())\n+\t\t\t\tX_Ls.append(state.get_X_L())\n+\t\t\t\n+\t\t\tMI, Linfoot = iu.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=200)\n+\n+\t\t\tif d == 0:\n+\t\t\t\tdata_d = numpy.transpose(Linfoot)\n+\t\t\telse:\n+\t\t\t\tdata_d = numpy.vstack((data_d, numpy.transpose(Linfoot)))\n+\n+\t\tif nr == 0:\n+\t\t\tdata = data_d\n+\t\telse:\n+\t\t\tdata = numpy.hstack((data, data_d))\n+\t\t\n+\t\tnr += 1\n+\n+\n+\tpl.subplot(2,2,subplot)\n+\tpl.boxplot(data)\n+\ttitle = "N=%i" % n\n+\tpl.title(title)\n+\n+pl.show()\n', 'commit_subject': 'Updated files to reflect reorg and fixed s/2 bug in cpp', 'repo_name': 'probcomp/crosscat', 'committer': {'name': 'Baxter Eaves', 'email': 'ee6d82e9542d410b6fade33bc6dce15b9336c5a1@gmail.com', 'time_sec': '1380891161', 'tz_offset': '-240', 'date': {'seconds': '1380891161'}}, 'commit_tree': '6b9c1ace660dec3f448912dc94cb564755a1a974', 'commit_message': 'Updated files to reflect reorg and fixed s/2 bug in cpp\n', 'language_name': 'Matlab', 'license': 'apache-2.0', 'hunks': '@@ -0,0 +1,80 @@\n+import numpy\n+import pylab as pl\n+import tabular_predDB.python_utils.inference_utils as iu\n+import tabular_predDB.python_utils.data_utils as du\n+import tabular_predDB.cython_code.State as State\n+\n+from scipy.stats import pearsonr as pearsonr\n+\n+import random\n+\n+def get_correlations(T, Q):\n+\tT = numpy.array(T)\n+\tcorr = []\n+\n+\tfor query in Q:\n+\t\tr, p = scipy.stats.pearsonr(T[:,Q[0]], T[:,Q[1]])\n+\t\tcorr.append(r)\n+\n+\treturn corr\n+\n+def gen_correlated_data( n, r, SEED=0 ):\n+\tnumpy.random.seed(SEED)\n+\tT = numpy.random.multivariate_normal([0,0],[[1,r],[r,1]],n)\n+\n+\treturn T\n+\n+get_next_seed = lambda : random.randrange(32000)\n+\n+correlations = [.0, .1, .2, .3, .4 , .5, .6, .7, .8, .9, 1.0]\n+N = [5, 10, 25, 50]\n+n_samples = 10\n+n_data_sets = 5\n+pl.figure()\n+burn_in = 200\n+\n+subplot = 0\n+for n in N:\n+\tsubplot += 1\n+\tnr = 0\n+\t\n+\tfor r in correlations:\n+\t\tfor d in range(n_data_sets): # 3 data sets\n+\t\t\t#\n+\t\t\tT = gen_correlated_data( n, r, SEED=get_next_seed())\n+\n+\t\t\tpr, p = pearsonr(T[:,0], T[:,1])\n+\n+\t\t\tprint "num_samples: %i, R: %f, d: %i. Actual R: %f" % (n, r, d+1, pr)\n+\n+\t\t\tM_c = du.gen_M_c_from_T(T)\n+\t\t\tX_Ls = []\n+\t\t\tX_Ds = []\n+\n+\t\t\tfor _ in range(n_samples):\n+\t\t\t\tstate = State.p_State(M_c, T)\n+\t\t\t\tstate.transition(n_steps=burn_in)\n+\t\t\t\tX_Ds.append(state.get_X_D())\n+\t\t\t\tX_Ls.append(state.get_X_L())\n+\t\t\t\n+\t\t\tMI, Linfoot = iu.mutual_information(M_c, X_Ls, X_Ds, [(0,1)], n_samples=200)\n+\n+\t\t\tif d == 0:\n+\t\t\t\tdata_d = numpy.transpose(Linfoot)\n+\t\t\telse:\n+\t\t\t\tdata_d = numpy.vstack((data_d, numpy.transpose(Linfoot)))\n+\n+\t\tif nr == 0:\n+\t\t\tdata = data_d\n+\t\telse:\n+\t\t\tdata = numpy.hstack((data, data_d))\n+\t\t\n+\t\tnr += 1\n+\n+\n+\tpl.subplot(2,2,subplot)\n+\tpl.boxplot(data)\n+\ttitle = "N=%i" % n\n+\tpl.title(title)\n+\n+pl.show()\n', 'addition_count': 80, 'deletion_count': 0, 'src_file': '/dev/null', 'tgt_file': 'b/tabular_predDB/tests/test_mutual_information_vs_correlation.py', 'file_extension': '.py', 'before_file': 'ADDFILE', 'message': 'Updated files to reflect reorg and fixed s/2 bug in cpp\n', 'author_name': 'Baxter Eaves', 'lang': 'py'}