evaluation/classifier.py
def train(data_dir, model_file, vect_file, is_test=False):
    """
    data_dir must contain 2 directories: positive and negative
    Each directory must contain a json file (i.e., data.json) or list of html files (ache format)
    """
    print "Loading data..."
    pos_docs = load_data(data_dir + '/positive')
    neg_docs = load_data(data_dir + '/negative')
    docs = pos_docs + neg_docs

    print "Vectorizing..."
    #vectorizer = TfidfVectorizer(tokenizer=LemmaTokenizer(), stop_words='english', ngram_range=(1,2), max_df=0.95, min_df=0.05, norm='l2', sublinear_tf=True)
    vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1,2), max_df=0.97, min_df=0.03, norm='l2', sublinear_tf=False)
    vectorizer.fit(docs) 
    pos = vectorizer.transform(pos_docs).todense()
    neg = vectorizer.transform(neg_docs).todense()
    print pos.shape, neg.shape
    x = np.concatenate((pos, neg))
    y = np.array([1]*pos.shape[0] + [-1]*neg.shape[0])
    x, y = shuffle(x, y, random_state=0)

    #clf = SVC(kernel="linear", gamma=0.01, C=0.005, probability=True) # much worse compared to default gamma and C
    clf = SVC(kernel="linear", C=1.3, probability=True)

    print "Fitting..."
    if is_test:
        scores = cross_val_score(clf, x, y, cv=5) 
        print "Cross validation score: ", scores
        print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))

    clf.fit(x, y)

    print "Saving..."
    with open(model_file, 'wb') as f:
        pickle.dump(clf, f)
    with open(vect_file, 'wb') as f:
        pickle.dump(vectorizer, f)

    data_dir must contain 2 directories: positive and negative
    Each directory must contain a json file (i.e., data.json) or list of html files (ache format)
    
data_dir must contain 2 directories: positive and negative
Each directory must contain a json file  or list of html files (ache format)
data_dir must contain 2 directories : positive and negative Each directory must contain a json file or list of html files ( ache format )
def train(data_dir, model_file, vect_file, is_test=False):
    print "Loading data..."
    pos_docs = load_data(data_dir + '/positive')
    neg_docs = load_data(data_dir + '/negative')
    docs = pos_docs + neg_docs
    print "Vectorizing..."
    vectorizer = TfidfVectorizer(stop_words='english', ngram_range=(1,2), max_df=0.97, min_df=0.03, norm='l2', sublinear_tf=False)
    vectorizer.fit(docs) 
    pos = vectorizer.transform(pos_docs).todense()
    neg = vectorizer.transform(neg_docs).todense()
    print pos.shape, neg.shape
    x = np.concatenate((pos, neg))
    y = np.array([1]*pos.shape[0] + [-1]*neg.shape[0])
    x, y = shuffle(x, y, random_state=0)
    clf = SVC(kernel="linear", C=1.3, probability=True)
    print "Fitting..."
    if is_test:
        scores = cross_val_score(clf, x, y, cv=5) 
        print "Cross validation score: ", scores
        print("Accuracy: %0.2f (+/- %0.2f)" % (scores.mean(), scores.std() * 2))
    clf.fit(x, y)
    print "Saving..."
    with open(model_file, 'wb') as f:
        pickle.dump(clf, f)
    with open(vect_file, 'wb') as f:
        pickle.dump(vectorizer, f)
def train ( data_dir , model_file , vect_file , is_test = False ) : print "Loading data..." pos_docs = load_data ( data_dir + '/positive' ) neg_docs = load_data ( data_dir + '/negative' ) docs = pos_docs + neg_docs print "Vectorizing..." vectorizer = TfidfVectorizer ( stop_words = 'english' , ngram_range = ( 1 , 2 ) , max_df = 0.97 , min_df = 0.03 , norm = 'l2' , sublinear_tf = False ) vectorizer . fit ( docs ) pos = vectorizer . transform ( pos_docs ) . todense ( ) neg = vectorizer . transform ( neg_docs ) . todense ( ) print pos . shape , neg . shape x = np . concatenate ( ( pos , neg ) ) y = np . array ( [ 1 ] * pos . shape [ 0 ] + [ - 1 ] * neg . shape [ 0 ] ) x , y = shuffle ( x , y , random_state = 0 ) clf = SVC ( kernel = "linear" , C = 1.3 , probability = True ) print "Fitting..." if is_test : scores = cross_val_score ( clf , x , y , cv = 5 ) print "Cross validation score: " , scores print ( "Accuracy: %0.2f (+/- %0.2f)" % ( scores . mean ( ) , scores . std ( ) * 2 ) ) clf . fit ( x , y ) print "Saving..." with open ( model_file , 'wb' ) as f : pickle . dump ( clf , f ) with open ( vect_file , 'wb' ) as f : pickle . dump ( vectorizer , f )
data_dir must contain 2 directories: positive and negative
Each directory must contain a json file (i.e., data.json) or list of html files (ache format)
data_dir must contain 2 directories : positive and negative Each directory must contain a json file ( i . e . data . json ) or list of html files ( ache format )
"""
    data_dir must contain 2 directories: positive and negative
    Each directory must contain a json file (i.e., data.json) or list of html files (ache format)
    """ #vectorizer = TfidfVectorizer(tokenizer=LemmaTokenizer(), stop_words='english', ngram_range=(1,2), max_df=0.95, min_df=0.05, norm='l2', sublinear_tf=True) #clf = SVC(kernel="linear", gamma=0.01, C=0.005, probability=True) # much worse compared to default gamma and C
data_dir
model_file
vect_file
is_test