The code below measures precision and recall and F-measure (source). How can I measure AUC?import collectionsimport nltk.metricsfrom nltk.classify import NaiveBayesClassifierfrom nltk.corpus import movie_reviewsdef word_feats(words):    return dict([(word, True) for word in words])negids = movie_reviews.fileids('neg')posids = movie_reviews.fileids('pos')negfeats = [(word_feats(movie_reviews.words(fileids=[f])), 'neg') for f in negids]posfeats = [(word_feats(movie_reviews.words(fileids=[f])), 'pos') for f in posids]negcutoff = len(negfeats)*3/4poscutoff = len(posfeats)*3/4trainfeats = negfeats[:negcutoff] + posfeats[:poscutoff]testfeats = negfeats[negcutoff:] + posfeats[poscutoff:]print 'train on %d instances, test on %d instances' % (len(trainfeats), len(testfeats))classifier = NaiveBayesClassifier.train(trainfeats)refsets = collections.defaultdict(set)testsets = collections.defaultdict(set)for i, (feats, label) in enumerate(testfeats):    refsets[label].add(i)    observed = classifier.classify(feats)    testsets[observed].add(i)print 'pos precision:', nltk.metrics.precision(refsets['pos'], testsets['pos'])print 'pos recall:', nltk.metrics.recall(refsets['pos'], testsets['pos'])print 'pos F-measure:', nltk.metrics.f_measure(refsets['pos'], testsets['pos'])print 'neg precision:', nltk.metrics.precision(refsets['neg'], testsets['neg'])print 'neg recall:', nltk.metrics.recall(refsets['neg'], testsets['neg'])print 'neg F-measure:', nltk.metrics.f_measure(refsets['neg'], testsets['neg'])
I need to measure Performance : AUC for this code of NLTK and skLearn
python;scikit learn;nltk
It is unclear if you are requesting AUC of ROC or Precision-Recall curve. However, instead of storing the indices of examples in sets, you can store the labels in lists and use sklearn's auc function after running precision_recall_curve or roc_curve:from sklearn.metrics import precision_recall_curvefrom sklearn.metrics import roc_curvefrom sklearn.metrics import aucdef label2int(label):    if label == 'pos':        return 1    else:        return 0y_true, y_score = [], []for i, (feats, label_true) in enumerate(testfeats):    label_predicted = classifier.classify(feats)    y_true.append(label2int(label_true))    y_score.append(label2int(label_predicted))# Precision-Recall AUCprecision, recall, _ = precision_recall_curve(y_true, y_score, pos_label=1)pr_auc = auc(recall, precision)print Precision-Recall AUC: %.2f % pr_auc# ROC AUCfpr, tpr, _ = roc_curve(y_true, y_score, pos_label=1)roc_auc = auc(fpr, tpr)print ROC AUC: %.2f % roc_aucPrecision-Recall AUC: 0.82ROC AUC: 0.73