<NME> tests.py
<BEF> from datacleaner import autoclean, autoclean_cv
import pandas as pd
import numpy as np

def test_autoclean_already_clean_data():
    """Test autoclean() with already-clean data"""
    data = pd.DataFrame({'A': np.random.rand(1000),
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    cleaned_data = autoclean(data)

    # autoclean() should not change the data at all
    assert cleaned_data.equals(data)

def test_autoclean_cv_already_clean_data():
    """Test autoclean_cv() with already-clean data"""
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    training_data = data[:500].copy()
    testing_data = data[500:].copy()

    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)
    # autoclean_cv() should not change the data at all
    assert clean_training_data.equals(training_data)
    assert clean_testing_data.equals(testing_data)
    hand_cleaned_testing_data['C'].fillna(training_C_median, inplace=True)

    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)

    assert cleaned_training_data.equals(hand_cleaned_training_data)
    assert cleaned_testing_data.equals(hand_cleaned_testing_data)

def test_autoclean_no_nans_with_strings():
    """Test autoclean() with a data set that has some string-encoded categorical values and no NaNs"""
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    string_map = {0: 'oranges', 1: 'apples', 2: 'bananas'}
    data['C'] = data['C'].apply(lambda x: string_map[x])

    hand_cleaned_data = data.copy()
    hand_cleaned_data['C'] = LabelEncoder().fit_transform(hand_cleaned_data['C'].values)

    cleaned_data = autoclean(data)

    assert cleaned_data.equals(hand_cleaned_data)

def test_autoclean_cv_no_nans_with_strings():
    """Test autoclean_cv() with a data set that has some string-encoded categorical values and no NaNs"""
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    string_map = {0: 'oranges', 1: 'apples', 2: 'bananas'}
    data['C'] = data['C'].apply(lambda x: string_map[x])

    training_data = data[:500].copy()
    testing_data = data[500:].copy()

    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)

    hand_cleaned_training_data = training_data.copy()
    hand_cleaned_testing_data = testing_data.copy()

    encoder = LabelEncoder()
    hand_cleaned_training_data['C'] = encoder.fit_transform(hand_cleaned_training_data['C'].values)
    hand_cleaned_testing_data['C'] = encoder.transform(hand_cleaned_testing_data['C'].values)

    assert cleaned_training_data.equals(hand_cleaned_training_data)
    assert cleaned_testing_data.equals(hand_cleaned_testing_data)

def test_autoclean_with_nans_with_strings():
    """Test autoclean() with a data set that has some string-encoded categorical values and some NaNs"""
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    string_map = {0: 'oranges', 1: 'apples', 2: 'bananas'}
    data['C'] = data['C'].apply(lambda x: string_map[x])

    data.loc[10:20, 'A'] = np.nan
    data.loc[50:70, 'C'] = np.nan

    hand_cleaned_data = data.copy()
    hand_cleaned_data['A'].fillna(hand_cleaned_data['A'].median(), inplace=True)
    hand_cleaned_data['C'].fillna(hand_cleaned_data['C'].mode()[0], inplace=True)
    hand_cleaned_data['C'] = LabelEncoder().fit_transform(hand_cleaned_data['C'].values)

    cleaned_data = autoclean(data)

    assert cleaned_data.equals(hand_cleaned_data)

def test_autoclean_cv_with_nans_with_strings():
    """Test autoclean_cv() with a data set that has some string-encoded categorical values and some NaNs"""
    data = pd.DataFrame({'A': np.random.rand(1000),
                         'B': np.random.rand(1000),
                         'C': np.random.randint(0, 3, 1000)})

    string_map = {0: 'oranges', 1: 'apples', 2: 'bananas'}
    data['C'] = data['C'].apply(lambda x: string_map[x])

    training_data = data[:500].copy()
    testing_data = data[500:].copy()

    training_data.loc[10:20, 'A'] = np.nan
    training_data.loc[50:70, 'C'] = np.nan

    testing_data.loc[70:80, 'A'] = np.nan
    testing_data.loc[10:40, 'C'] = np.nan

    hand_cleaned_training_data = training_data.copy()
    hand_cleaned_testing_data = testing_data.copy()

    training_A_median = hand_cleaned_training_data['A'].median()
    training_C_mode = hand_cleaned_training_data['C'].mode()[0]
    hand_cleaned_training_data['A'].fillna(training_A_median, inplace=True)
    hand_cleaned_training_data['C'].fillna(training_C_mode, inplace=True)

    hand_cleaned_testing_data['A'].fillna(training_A_median, inplace=True)
    hand_cleaned_testing_data['C'].fillna(training_C_mode, inplace=True)

    encoder = LabelEncoder()
    hand_cleaned_training_data['C'] = encoder.fit_transform(hand_cleaned_training_data['C'].values)
    hand_cleaned_testing_data['C'] = encoder.transform(hand_cleaned_testing_data['C'].values)

    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)

    assert cleaned_training_data.equals(hand_cleaned_training_data)
    assert cleaned_testing_data.equals(hand_cleaned_testing_data)

def test_autoclean_real_data():
    """Test autoclean() with the adult data set"""
    adult_data = pd.read_csv('adult.csv.gz', sep='\t', compression='gzip')
    adult_data.loc[30:60, 'age'] = np.nan
    adult_data.loc[90:100, 'education'] = np.nan

    hand_cleaned_adult_data = adult_data.copy()

    hand_cleaned_adult_data['age'].fillna(hand_cleaned_adult_data['age'].median(), inplace=True)
    hand_cleaned_adult_data['education'].fillna(hand_cleaned_adult_data['education'].mode()[0], inplace=True)

    for column in ['workclass', 'education', 'marital-status',
                   'occupation', 'relationship', 'race',
                   'sex', 'native-country', 'label']:
        hand_cleaned_adult_data[column] = LabelEncoder().fit_transform(hand_cleaned_adult_data[column].values)

    cleaned_adult_data = autoclean(adult_data)

    assert cleaned_adult_data.equals(hand_cleaned_adult_data)

def test_autoclean_cv_real_data():
    """Test autoclean_cv() with the adult data set"""
    adult_data = pd.read_csv('adult.csv.gz', sep='\t', compression='gzip')

    training_adult_data = adult_data[:int(len(adult_data) / 2.)].copy()
    testing_adult_data = adult_data[int(len(adult_data) / 2.):].copy()

    training_adult_data.loc[30:60, 'age'] = np.nan
    training_adult_data.loc[90:100, 'education'] = np.nan

    testing_adult_data.loc[90:110, 'age'] = np.nan
    testing_adult_data.loc[20:40, 'education'] = np.nan

    hand_cleaned_training_adult_data = training_adult_data.copy()
    hand_cleaned_testing_adult_data = testing_adult_data.copy()

    training_age_median = hand_cleaned_training_adult_data['age'].median()
    training_education_mode = hand_cleaned_training_adult_data['education'].mode()[0]

    hand_cleaned_training_adult_data['age'].fillna(training_age_median, inplace=True)
    hand_cleaned_training_adult_data['education'].fillna(training_education_mode, inplace=True)

    hand_cleaned_testing_adult_data['age'].fillna(training_age_median, inplace=True)
    hand_cleaned_testing_adult_data['education'].fillna(training_education_mode, inplace=True)

    for column in ['workclass', 'education', 'marital-status',
                   'occupation', 'relationship', 'race',
                   'sex', 'native-country', 'label']:
        encoder = LabelEncoder()
        hand_cleaned_training_adult_data[column] = encoder.fit_transform(hand_cleaned_training_adult_data[column].values)
        hand_cleaned_testing_adult_data[column] = encoder.transform(hand_cleaned_testing_adult_data[column].values)

    cleaned_adult_training_data, cleaned_adult_testing_data = autoclean_cv(training_adult_data, testing_adult_data)

    assert cleaned_adult_training_data.equals(hand_cleaned_training_adult_data)
    assert cleaned_adult_testing_data.equals(hand_cleaned_testing_adult_data)

<MSG> NaN tests added

Related to #5

<DFF> @@ -2,6 +2,8 @@ from datacleaner import autoclean, autoclean_cv
 import pandas as pd
 import numpy as np
 
+np.random.seed(300)
+
 def test_autoclean_already_clean_data():
     """Test autoclean() with already-clean data"""
     data = pd.DataFrame({'A': np.random.rand(1000),
@@ -27,3 +29,47 @@ def test_autoclean_cv_already_clean_data():
     # autoclean_cv() should not change the data at all
     assert clean_training_data.equals(training_data)
     assert clean_testing_data.equals(testing_data)
+
+def test_autoclean_with_nans_all_numerical():
+    data = pd.DataFrame({'A': np.random.rand(1000),
+                         'B': np.random.rand(1000),
+                         'C': np.random.randint(0, 3, 1000)})
+    
+    data.loc[10:20, 'A'] = np.nan
+    data.loc[50:70, 'C'] = np.nan
+    
+    hand_cleaned_data = data.copy()
+    hand_cleaned_data['A'].fillna(hand_cleaned_data['A'].median(), inplace=True)
+    hand_cleaned_data['C'].fillna(hand_cleaned_data['C'].mode()[0], inplace=True)
+    
+    cleaned_data = autoclean(data)
+    
+    assert cleaned_data.equals(hand_cleaned_data)
+
+def test_autoclean_cv_with_nans_all_numerical():
+    data = pd.DataFrame({'A': np.random.rand(1000),
+                         'B': np.random.rand(1000),
+                         'C': np.random.randint(0, 3, 1000)})
+    
+    training_data = data[:500].copy()
+    testing_data = data[500:].copy()
+    
+    training_data.loc[10:20, 'A'] = np.nan
+    training_data.loc[50:70, 'C'] = np.nan
+    
+    testing_data.loc[70:80, 'A'] = np.nan
+    testing_data.loc[10:40, 'C'] = np.nan
+    
+    hand_cleaned_training_data = training_data.copy()
+    hand_cleaned_testing_data = testing_data.copy()
+    
+    hand_cleaned_training_data['A'].fillna(hand_cleaned_training_data['A'].median(), inplace=True)
+    hand_cleaned_training_data['C'].fillna(hand_cleaned_training_data['C'].mode()[0], inplace=True)
+    
+    hand_cleaned_testing_data['A'].fillna(hand_cleaned_testing_data['A'].median(), inplace=True)
+    hand_cleaned_testing_data['C'].fillna(hand_cleaned_testing_data['C'].mode()[0], inplace=True)
+    
+    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)
+    
+    assert cleaned_training_data.equals(hand_cleaned_training_data)
+    assert cleaned_testing_data.equals(hand_cleaned_testing_data)

{'commit': 'diff --git a/tests.py b/tests.py\nindex 3be07a1..0e18e33 100644\n--- a/tests.py\n+++ b/tests.py\n@@ -2,6 +2,8 @@ from datacleaner import autoclean, autoclean_cv\n import pandas as pd\n import numpy as np\n \n+np.random.seed(300)\n+\n def test_autoclean_already_clean_data():\n     """Test autoclean() with already-clean data"""\n     data = pd.DataFrame({\'A\': np.random.rand(1000),\n@@ -27,3 +29,47 @@ def test_autoclean_cv_already_clean_data():\n     # autoclean_cv() should not change the data at all\n     assert clean_training_data.equals(training_data)\n     assert clean_testing_data.equals(testing_data)\n+\n+def test_autoclean_with_nans_all_numerical():\n+    data = pd.DataFrame({\'A\': np.random.rand(1000),\n+                         \'B\': np.random.rand(1000),\n+                         \'C\': np.random.randint(0, 3, 1000)})\n+    \n+    data.loc[10:20, \'A\'] = np.nan\n+    data.loc[50:70, \'C\'] = np.nan\n+    \n+    hand_cleaned_data = data.copy()\n+    hand_cleaned_data[\'A\'].fillna(hand_cleaned_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_data[\'C\'].fillna(hand_cleaned_data[\'C\'].mode()[0], inplace=True)\n+    \n+    cleaned_data = autoclean(data)\n+    \n+    assert cleaned_data.equals(hand_cleaned_data)\n+\n+def test_autoclean_cv_with_nans_all_numerical():\n+    data = pd.DataFrame({\'A\': np.random.rand(1000),\n+                         \'B\': np.random.rand(1000),\n+                         \'C\': np.random.randint(0, 3, 1000)})\n+    \n+    training_data = data[:500].copy()\n+    testing_data = data[500:].copy()\n+    \n+    training_data.loc[10:20, \'A\'] = np.nan\n+    training_data.loc[50:70, \'C\'] = np.nan\n+    \n+    testing_data.loc[70:80, \'A\'] = np.nan\n+    testing_data.loc[10:40, \'C\'] = np.nan\n+    \n+    hand_cleaned_training_data = training_data.copy()\n+    hand_cleaned_testing_data = testing_data.copy()\n+    \n+    hand_cleaned_training_data[\'A\'].fillna(hand_cleaned_training_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_training_data[\'C\'].fillna(hand_cleaned_training_data[\'C\'].mode()[0], inplace=True)\n+    \n+    hand_cleaned_testing_data[\'A\'].fillna(hand_cleaned_testing_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_testing_data[\'C\'].fillna(hand_cleaned_testing_data[\'C\'].mode()[0], inplace=True)\n+    \n+    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n+    \n+    assert cleaned_training_data.equals(hand_cleaned_training_data)\n+    assert cleaned_testing_data.equals(hand_cleaned_testing_data)\n', 'commit_subject': 'NaN tests added', 'repo_name': 'rhiever/datacleaner', 'committer': {'name': 'Randy Olson', 'email': '41671b79f2d3ff9cdb980d01b2c9c186329c05c8@randalolson.com', 'time_sec': '1457225668', 'tz_offset': '-300', 'date': {'seconds': '1457225668'}}, 'commit_tree': '99b874fc5e1e540fa92639c32394bee0dab65042', 'commit_message': 'NaN tests added\n\nRelated to #5\n', 'language_name': 'Python', 'license': 'mit', 'hunks': '@@ -2,6 +2,8 @@ from datacleaner import autoclean, autoclean_cv\n import pandas as pd\n import numpy as np\n \n+np.random.seed(300)\n+\n def test_autoclean_already_clean_data():\n     """Test autoclean() with already-clean data"""\n     data = pd.DataFrame({\'A\': np.random.rand(1000),\n@@ -27,3 +29,47 @@ def test_autoclean_cv_already_clean_data():\n     # autoclean_cv() should not change the data at all\n     assert clean_training_data.equals(training_data)\n     assert clean_testing_data.equals(testing_data)\n+\n+def test_autoclean_with_nans_all_numerical():\n+    data = pd.DataFrame({\'A\': np.random.rand(1000),\n+                         \'B\': np.random.rand(1000),\n+                         \'C\': np.random.randint(0, 3, 1000)})\n+    \n+    data.loc[10:20, \'A\'] = np.nan\n+    data.loc[50:70, \'C\'] = np.nan\n+    \n+    hand_cleaned_data = data.copy()\n+    hand_cleaned_data[\'A\'].fillna(hand_cleaned_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_data[\'C\'].fillna(hand_cleaned_data[\'C\'].mode()[0], inplace=True)\n+    \n+    cleaned_data = autoclean(data)\n+    \n+    assert cleaned_data.equals(hand_cleaned_data)\n+\n+def test_autoclean_cv_with_nans_all_numerical():\n+    data = pd.DataFrame({\'A\': np.random.rand(1000),\n+                         \'B\': np.random.rand(1000),\n+                         \'C\': np.random.randint(0, 3, 1000)})\n+    \n+    training_data = data[:500].copy()\n+    testing_data = data[500:].copy()\n+    \n+    training_data.loc[10:20, \'A\'] = np.nan\n+    training_data.loc[50:70, \'C\'] = np.nan\n+    \n+    testing_data.loc[70:80, \'A\'] = np.nan\n+    testing_data.loc[10:40, \'C\'] = np.nan\n+    \n+    hand_cleaned_training_data = training_data.copy()\n+    hand_cleaned_testing_data = testing_data.copy()\n+    \n+    hand_cleaned_training_data[\'A\'].fillna(hand_cleaned_training_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_training_data[\'C\'].fillna(hand_cleaned_training_data[\'C\'].mode()[0], inplace=True)\n+    \n+    hand_cleaned_testing_data[\'A\'].fillna(hand_cleaned_testing_data[\'A\'].median(), inplace=True)\n+    hand_cleaned_testing_data[\'C\'].fillna(hand_cleaned_testing_data[\'C\'].mode()[0], inplace=True)\n+    \n+    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n+    \n+    assert cleaned_training_data.equals(hand_cleaned_training_data)\n+    assert cleaned_testing_data.equals(hand_cleaned_testing_data)\n', 'addition_count': 46, 'deletion_count': 0, 'src_file': 'a/tests.py', 'tgt_file': 'b/tests.py', 'file_extension': '.py', 'before_file': 'from datacleaner import autoclean, autoclean_cv\nimport pandas as pd\nimport numpy as np\n\ndef test_autoclean_already_clean_data():\n    """Test autoclean() with already-clean data"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    cleaned_data = autoclean(data)\n\n    # autoclean() should not change the data at all\n    assert cleaned_data.equals(data)\n\ndef test_autoclean_cv_already_clean_data():\n    """Test autoclean_cv() with already-clean data"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    training_data = data[:500].copy()\n    testing_data = data[500:].copy()\n\n    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n    # autoclean_cv() should not change the data at all\n    assert clean_training_data.equals(training_data)\n    assert clean_testing_data.equals(testing_data)\n    hand_cleaned_testing_data[\'C\'].fillna(training_C_median, inplace=True)\n\n    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n\n    assert cleaned_training_data.equals(hand_cleaned_training_data)\n    assert cleaned_testing_data.equals(hand_cleaned_testing_data)\n\ndef test_autoclean_no_nans_with_strings():\n    """Test autoclean() with a data set that has some string-encoded categorical values and no NaNs"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    string_map = {0: \'oranges\', 1: \'apples\', 2: \'bananas\'}\n    data[\'C\'] = data[\'C\'].apply(lambda x: string_map[x])\n\n    hand_cleaned_data = data.copy()\n    hand_cleaned_data[\'C\'] = LabelEncoder().fit_transform(hand_cleaned_data[\'C\'].values)\n\n    cleaned_data = autoclean(data)\n\n    assert cleaned_data.equals(hand_cleaned_data)\n\ndef test_autoclean_cv_no_nans_with_strings():\n    """Test autoclean_cv() with a data set that has some string-encoded categorical values and no NaNs"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    string_map = {0: \'oranges\', 1: \'apples\', 2: \'bananas\'}\n    data[\'C\'] = data[\'C\'].apply(lambda x: string_map[x])\n\n    training_data = data[:500].copy()\n    testing_data = data[500:].copy()\n\n    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n\n    hand_cleaned_training_data = training_data.copy()\n    hand_cleaned_testing_data = testing_data.copy()\n\n    encoder = LabelEncoder()\n    hand_cleaned_training_data[\'C\'] = encoder.fit_transform(hand_cleaned_training_data[\'C\'].values)\n    hand_cleaned_testing_data[\'C\'] = encoder.transform(hand_cleaned_testing_data[\'C\'].values)\n\n    assert cleaned_training_data.equals(hand_cleaned_training_data)\n    assert cleaned_testing_data.equals(hand_cleaned_testing_data)\n\ndef test_autoclean_with_nans_with_strings():\n    """Test autoclean() with a data set that has some string-encoded categorical values and some NaNs"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    string_map = {0: \'oranges\', 1: \'apples\', 2: \'bananas\'}\n    data[\'C\'] = data[\'C\'].apply(lambda x: string_map[x])\n\n    data.loc[10:20, \'A\'] = np.nan\n    data.loc[50:70, \'C\'] = np.nan\n\n    hand_cleaned_data = data.copy()\n    hand_cleaned_data[\'A\'].fillna(hand_cleaned_data[\'A\'].median(), inplace=True)\n    hand_cleaned_data[\'C\'].fillna(hand_cleaned_data[\'C\'].mode()[0], inplace=True)\n    hand_cleaned_data[\'C\'] = LabelEncoder().fit_transform(hand_cleaned_data[\'C\'].values)\n\n    cleaned_data = autoclean(data)\n\n    assert cleaned_data.equals(hand_cleaned_data)\n\ndef test_autoclean_cv_with_nans_with_strings():\n    """Test autoclean_cv() with a data set that has some string-encoded categorical values and some NaNs"""\n    data = pd.DataFrame({\'A\': np.random.rand(1000),\n                         \'B\': np.random.rand(1000),\n                         \'C\': np.random.randint(0, 3, 1000)})\n\n    string_map = {0: \'oranges\', 1: \'apples\', 2: \'bananas\'}\n    data[\'C\'] = data[\'C\'].apply(lambda x: string_map[x])\n\n    training_data = data[:500].copy()\n    testing_data = data[500:].copy()\n\n    training_data.loc[10:20, \'A\'] = np.nan\n    training_data.loc[50:70, \'C\'] = np.nan\n\n    testing_data.loc[70:80, \'A\'] = np.nan\n    testing_data.loc[10:40, \'C\'] = np.nan\n\n    hand_cleaned_training_data = training_data.copy()\n    hand_cleaned_testing_data = testing_data.copy()\n\n    training_A_median = hand_cleaned_training_data[\'A\'].median()\n    training_C_mode = hand_cleaned_training_data[\'C\'].mode()[0]\n    hand_cleaned_training_data[\'A\'].fillna(training_A_median, inplace=True)\n    hand_cleaned_training_data[\'C\'].fillna(training_C_mode, inplace=True)\n\n    hand_cleaned_testing_data[\'A\'].fillna(training_A_median, inplace=True)\n    hand_cleaned_testing_data[\'C\'].fillna(training_C_mode, inplace=True)\n\n    encoder = LabelEncoder()\n    hand_cleaned_training_data[\'C\'] = encoder.fit_transform(hand_cleaned_training_data[\'C\'].values)\n    hand_cleaned_testing_data[\'C\'] = encoder.transform(hand_cleaned_testing_data[\'C\'].values)\n\n    cleaned_training_data, cleaned_testing_data = autoclean_cv(training_data, testing_data)\n\n    assert cleaned_training_data.equals(hand_cleaned_training_data)\n    assert cleaned_testing_data.equals(hand_cleaned_testing_data)\n\ndef test_autoclean_real_data():\n    """Test autoclean() with the adult data set"""\n    adult_data = pd.read_csv(\'adult.csv.gz\', sep=\'\\t\', compression=\'gzip\')\n    adult_data.loc[30:60, \'age\'] = np.nan\n    adult_data.loc[90:100, \'education\'] = np.nan\n\n    hand_cleaned_adult_data = adult_data.copy()\n\n    hand_cleaned_adult_data[\'age\'].fillna(hand_cleaned_adult_data[\'age\'].median(), inplace=True)\n    hand_cleaned_adult_data[\'education\'].fillna(hand_cleaned_adult_data[\'education\'].mode()[0], inplace=True)\n\n    for column in [\'workclass\', \'education\', \'marital-status\',\n                   \'occupation\', \'relationship\', \'race\',\n                   \'sex\', \'native-country\', \'label\']:\n        hand_cleaned_adult_data[column] = LabelEncoder().fit_transform(hand_cleaned_adult_data[column].values)\n\n    cleaned_adult_data = autoclean(adult_data)\n\n    assert cleaned_adult_data.equals(hand_cleaned_adult_data)\n\ndef test_autoclean_cv_real_data():\n    """Test autoclean_cv() with the adult data set"""\n    adult_data = pd.read_csv(\'adult.csv.gz\', sep=\'\\t\', compression=\'gzip\')\n\n    training_adult_data = adult_data[:int(len(adult_data) / 2.)].copy()\n    testing_adult_data = adult_data[int(len(adult_data) / 2.):].copy()\n\n    training_adult_data.loc[30:60, \'age\'] = np.nan\n    training_adult_data.loc[90:100, \'education\'] = np.nan\n\n    testing_adult_data.loc[90:110, \'age\'] = np.nan\n    testing_adult_data.loc[20:40, \'education\'] = np.nan\n\n    hand_cleaned_training_adult_data = training_adult_data.copy()\n    hand_cleaned_testing_adult_data = testing_adult_data.copy()\n\n    training_age_median = hand_cleaned_training_adult_data[\'age\'].median()\n    training_education_mode = hand_cleaned_training_adult_data[\'education\'].mode()[0]\n\n    hand_cleaned_training_adult_data[\'age\'].fillna(training_age_median, inplace=True)\n    hand_cleaned_training_adult_data[\'education\'].fillna(training_education_mode, inplace=True)\n\n    hand_cleaned_testing_adult_data[\'age\'].fillna(training_age_median, inplace=True)\n    hand_cleaned_testing_adult_data[\'education\'].fillna(training_education_mode, inplace=True)\n\n    for column in [\'workclass\', \'education\', \'marital-status\',\n                   \'occupation\', \'relationship\', \'race\',\n                   \'sex\', \'native-country\', \'label\']:\n        encoder = LabelEncoder()\n        hand_cleaned_training_adult_data[column] = encoder.fit_transform(hand_cleaned_training_adult_data[column].values)\n        hand_cleaned_testing_adult_data[column] = encoder.transform(hand_cleaned_testing_adult_data[column].values)\n\n    cleaned_adult_training_data, cleaned_adult_testing_data = autoclean_cv(training_adult_data, testing_adult_data)\n\n    assert cleaned_adult_training_data.equals(hand_cleaned_training_adult_data)\n    assert cleaned_adult_testing_data.equals(hand_cleaned_testing_adult_data)\n', 'message': 'NaN tests added\n\nRelated to #5\n', 'author_name': 'Randy Olson', 'lang': 'py'}