public_talks/2016_02_26_columbia/do_ml_on_feature_tables (all.csv).py
def train(csv_filename):
    
    print('Loading CSV into dict ...')
    t0 = dt.datetime.utcnow()
    data, target = csv_to_dict(csv_filename)
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    
    print('Loading dict into vectorizer')
    t0 = dt.datetime.utcnow()
    vec = DictVectorizer()
    X = vec.fit_transform(data).toarray()  # change to numpy array
    Y = np.array(target)  # change to numpy array
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()

    
    '''
    -In case we need to know the features
    '''
    feature_names = vec.get_feature_names()

    '''
    -Dividing the data into train and test
    -random_state is pseudo-random number generator state used for
     random sampling
    '''
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0)
    
    # write models dir if not present
    models_dir = 'models'
    if not os.path.isdir(models_dir):
        os.mkdir(models_dir)

    '''
    -PREPOCESSING 
    -Here, scaled data has zero mean and unit varience
    -We save the scaler to later use with testing/prediction data
    '''
    print('Scaling data ...')
    t0 = dt.datetime.utcnow()
    scaler = preprocessing.StandardScaler().fit(X_train)
    joblib.dump(scaler, 'models/scaler.pickle')
    X_train_scaled = scaler.transform(X_train)
    X_test_scaled = scaler.transform(X_test)
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    

    '''
    -This is where we define the models
    -Here, I use SVM and Decision tree with pre-defined parameters
    -We can learn these parameters given our data
    '''
    print('Defining and fitting models ...')
    t0 = dt.datetime.utcnow()   
    clf0 = svm.LinearSVC(C=100.)
    clf1 = tree.DecisionTreeClassifier()

    clf0.fit(X_train_scaled, Y_train)
    clf1.fit(X_train_scaled, Y_train)

    joblib.dump(clf0, 'models/svc.pickle')
    joblib.dump(clf1, 'models/tree.pickle')

    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    

    Y_prediction_svc = clf0.predict(X_test_scaled)
    print('svc_predictions ', Y_prediction_svc)
    Y_prediction_tree = clf1.predict(X_test_scaled)
    print('tree_predictions ', Y_prediction_tree)
    expected = Y_test
    print('actual_values   ', expected)
    print()

    
    '''
    Classifiation metrics
    (Case 1): SVMs
    '''
    print()
    print('----Linear SVC_report--------------------------')
    print(classification_report(expected, Y_prediction_svc))

    '''
    Classification metrics
    (case 2): Decision tree
    '''
    print()
    print('----Tree_report--------------------------------')
    print(classification_report(expected, Y_prediction_tree))

    -In case we need to know the features
    
In case we need to know the features
In case we need to know the features
def train(csv_filename):
    print('Loading CSV into dict ...')
    t0 = dt.datetime.utcnow()
    data, target = csv_to_dict(csv_filename)
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    print('Loading dict into vectorizer')
    t0 = dt.datetime.utcnow()
    vec = DictVectorizer()
    X = vec.fit_transform(data).toarray()  
    Y = np.array(target)  
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    feature_names = vec.get_feature_names()
    X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=0)
    models_dir = 'models'
    if not os.path.isdir(models_dir):
        os.mkdir(models_dir)
    print('Scaling data ...')
    t0 = dt.datetime.utcnow()
    scaler = preprocessing.StandardScaler().fit(X_train)
    joblib.dump(scaler, 'models/scaler.pickle')
    X_train_scaled = scaler.transform(X_train)
    X_test_scaled = scaler.transform(X_test)
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    print('Defining and fitting models ...')
    t0 = dt.datetime.utcnow()   
    clf0 = svm.LinearSVC(C=100.)
    clf1 = tree.DecisionTreeClassifier()
    clf0.fit(X_train_scaled, Y_train)
    clf1.fit(X_train_scaled, Y_train)
    joblib.dump(clf0, 'models/svc.pickle')
    joblib.dump(clf1, 'models/tree.pickle')
    print('... finished in {} secs.'.format(dt.datetime.utcnow() - t0))
    print()
    Y_prediction_svc = clf0.predict(X_test_scaled)
    print('svc_predictions ', Y_prediction_svc)
    Y_prediction_tree = clf1.predict(X_test_scaled)
    print('tree_predictions ', Y_prediction_tree)
    expected = Y_test
    print('actual_values   ', expected)
    print()
    print()
    print('----Linear SVC_report--------------------------')
    print(classification_report(expected, Y_prediction_svc))
    print()
    print('----Tree_report--------------------------------')
    print(classification_report(expected, Y_prediction_tree))
def train ( csv_filename ) : print ( 'Loading CSV into dict ...' ) t0 = dt . datetime . utcnow ( ) data , target = csv_to_dict ( csv_filename ) print ( '... finished in {} secs.' . format ( dt . datetime . utcnow ( ) - t0 ) ) print ( ) print ( 'Loading dict into vectorizer' ) t0 = dt . datetime . utcnow ( ) vec = DictVectorizer ( ) X = vec . fit_transform ( data ) . toarray ( ) Y = np . array ( target ) print ( '... finished in {} secs.' . format ( dt . datetime . utcnow ( ) - t0 ) ) print ( ) feature_names = vec . get_feature_names ( ) '''
    -Dividing the data into train and test
    -random_state is pseudo-random number generator state used for
     random sampling
    ''' X_train , X_test , Y_train , Y_test = train_test_split ( X , Y , random_state = 0 ) models_dir = 'models' if not os . path . isdir ( models_dir ) : os . mkdir ( models_dir ) '''
    -PREPOCESSING 
    -Here, scaled data has zero mean and unit varience
    -We save the scaler to later use with testing/prediction data
    ''' print ( 'Scaling data ...' ) t0 = dt . datetime . utcnow ( ) scaler = preprocessing . StandardScaler ( ) . fit ( X_train ) joblib . dump ( scaler , 'models/scaler.pickle' ) X_train_scaled = scaler . transform ( X_train ) X_test_scaled = scaler . transform ( X_test ) print ( '... finished in {} secs.' . format ( dt . datetime . utcnow ( ) - t0 ) ) print ( ) '''
    -This is where we define the models
    -Here, I use SVM and Decision tree with pre-defined parameters
    -We can learn these parameters given our data
    ''' print ( 'Defining and fitting models ...' ) t0 = dt . datetime . utcnow ( ) clf0 = svm . LinearSVC ( C = 100. ) clf1 = tree . DecisionTreeClassifier ( ) clf0 . fit ( X_train_scaled , Y_train ) clf1 . fit ( X_train_scaled , Y_train ) joblib . dump ( clf0 , 'models/svc.pickle' ) joblib . dump ( clf1 , 'models/tree.pickle' ) print ( '... finished in {} secs.' . format ( dt . datetime . utcnow ( ) - t0 ) ) print ( ) Y_prediction_svc = clf0 . predict ( X_test_scaled ) print ( 'svc_predictions ' , Y_prediction_svc ) Y_prediction_tree = clf1 . predict ( X_test_scaled ) print ( 'tree_predictions ' , Y_prediction_tree ) expected = Y_test print ( 'actual_values   ' , expected ) print ( ) '''
    Classifiation metrics
    (Case 1): SVMs
    ''' print ( ) print ( '----Linear SVC_report--------------------------' ) print ( classification_report ( expected , Y_prediction_svc ) ) '''
    Classification metrics
    (case 2): Decision tree
    ''' print ( ) print ( '----Tree_report--------------------------------' ) print ( classification_report ( expected , Y_prediction_tree ) )
In case we need to know the features
In case we need to know the features
# change to numpy array # change to numpy array '''
    -In case we need to know the features
    ''' '''
    -Dividing the data into train and test
    -random_state is pseudo-random number generator state used for
     random sampling
    ''' # write models dir if not present '''
    -PREPOCESSING 
    -Here, scaled data has zero mean and unit varience
    -We save the scaler to later use with testing/prediction data
    ''' '''
    -This is where we define the models
    -Here, I use SVM and Decision tree with pre-defined parameters
    -We can learn these parameters given our data
    ''' '''
    Classifiation metrics
    (Case 1): SVMs
    ''' '''
    Classification metrics
    (case 2): Decision tree
    '''
csv_filename