Naive Bayes classifier for OKCupid profiles I tried to write a naive Bayes classifier to classify OkCupid profiles, and I was wondering if you could give me feedback on my code.  The classifier performs no better than chance and the coding style is probably not great either.  I had a training set of 20 good profiles and 20 bad profiles, and a test set of 16 good and 29 bad, so I don't know if I am classifying poorly or if I just don't have enough data.

#!/usr/bin/python

# Naive Bayes classifier for OkCupid data.  The features are the words in
# the profile and the classes are HIGH and LOW.  This is used to
# predict the rating I would give a profile.

# TODO: tweak the classifier so it works better.

from subprocess import call
from math import sqrt

# File containing training data in the form "&lt;rating&gt; &lt;username&gt;"
TRAINING_DATA = "train.dta"

# File containing test set in the form "&lt;rating&gt; &lt;username&gt;"
TEST_DATA = "test.dta"

# Dictionary
dictionary = "TWL06.txt"

# Hash table containing words in highly-ranked profiles.
HIGH_WORDS = {}

# Hash table containing words in low-ranked profiles.
LOW_WORDS = {}

# Number of users in each category
hi = 0
low = 0

# Add training data to the hash tables.
t = open(TRAINING_DATA, 'r')
for user in t:
    rating = int(user.split(" ")[0])
    name = user.split(" ")[1]

    # Count users
    if rating == 1:
        hi += 1
    else:
        low += 1

    # Get data for each user
    call(["curl", "-o", "tmp.dta", "http://www.okcupid.com/profile/" + name])
    d = open("tmp.dta", 'r')

    # Get word list from user
    words = {}
    for line in d:
        for word in line.split(" "):
            if "/" in word or "=" in word or "&lt;" in word or "&gt;" in word or "()" in word or "&amp;" in word or len(word) &gt; 10:
                continue
            words[word.rstrip().lower()] = None
    d.close()

    # Add words in this word list to our master word lists
    for word in words:
        if rating == 1:
            if word in HIGH_WORDS:
                HIGH_WORDS[word] += 1
            else:
                HIGH_WORDS[word] = 1
        else:
            if word in LOW_WORDS:
                LOW_WORDS[word] += 1
            else:
                LOW_WORDS[word] = 1
t.close()

print HIGH_WORDS
print LOW_WORDS

# Classify a point, assuming training has already happened.
# P(C | F1 ... Fn) proportionate to P(C) * P(F1 | C) * ... * P(Fn | C)
def classify(username):
    # P(C)
    Phi = float(hi) / float(hi + low)
    Plow = float(low) / float(hi + low)
    ratio = Phi / Plow
    # Get data
    call(["curl", "-o", "tmp.dta", "http://www.okcupid.com/profile/" + username])
    d = open("tmp.dta", 'r')
    for line in d:
        for word in line.split(" "):
            # Calculate the probability of the feature given a class
            if word in HIGH_WORDS:
                numHi = float(HIGH_WORDS[word])
            else:
                numHi = 1
            if word in LOW_WORDS:
                numLow = float(LOW_WORDS[word])
            else:
                numLow = 1

            ratio = ratio * (numHi / float(hi)) / (numLow / float(low))
#            if word in HIGH_WORDS:
##                Phi = Phi * float(HIGH_WORDS[word]) / float(hi)
##            else:
##                Phi = Phi * (1 / float(hi))
#
#            if word in LOW_WORDS:
##                Plow = Plow * float(LOW_WORDS[word]) / float(low)
##            else:
##                Plow = Plow * (1 / float(low))

    print ratio
    if ratio &gt; 1:
        return 1
    else:
        return 0

# Compute out of sample error on test set.
def getError():
    t = open(TEST_DATA, 'r')
    numUsers = 0.0
    total = 0.0
    for line in t:
        user = line.split(" ")[1]
        rating = int(line.split(" ")[0])
        prediction = classify(user)
        numUsers += 1.0
        if rating != prediction:
            total += 1.0
        print str(rating) + " " + str(prediction) + " " + user

    return total / numUsers



print getError()
#!/usr/bin/python

# Naive Bayes classifier for OkCupid data.  The features are the words in
# the profile and the classes are HIGH and LOW.  This is used to
# predict the rating I would give a profile.

# TODO: tweak the classifier so it works better.

from subprocess import call
from math import sqrt

# File containing training data in the form "&lt;rating&gt; &lt;username&gt;"
TRAINING_DATA = "train.dta"

# File containing test set in the form "&lt;rating&gt; &lt;username&gt;"
TEST_DATA = "test.dta"

# Dictionary
dictionary = "TWL06.txt"


Why is this not in ALL_CAPS like the others?

# Hash table containing words in highly-ranked profiles.
HIGH_WORDS = {}

# Hash table containing words in low-ranked profiles.
LOW_WORDS = {}


These aren't constants, so they shouldn't be global or in ALL_CAPS

# Number of users in each category
hi = 0
low = 0


Once you actually start logic, its best to do it inside a function except for really simply scripts.

# Add training data to the hash tables.
t = open(TRAINING_DATA, 'r')


Don't use single letter variable names, it results in difficult to follow code. Also, you should open a file using the with statement, to make sure it closes in all circumstances.

for user in t:
    rating = int(user.split(" ")[0])
    name = user.split(" ")[1]


Why are you passing the " " to slit instead of trusting the default. This approach is a little wasteful because you split the input twice.

    # Count users
    if rating == 1:
        hi += 1
    else:
        low += 1

    # Get data for each user
    call(["curl", "-o", "tmp.dta", "http://www.okcupid.com/profile/" + name])


Use urllib.urlopen to fetch urls. That you can avoid the temporary file.

    d = open("tmp.dta", 'r')

    # Get word list from user
    words = {}
    for line in d:
        for word in line.split(" "):
            if "/" in word or "=" in word or "&lt;" in word or "&gt;" in word or "()" in word or "&amp;" in word or len(word) &gt; 10:


Make a list of blocked characters rather then duplicating your logic so much here. And I suspect that "()" in word wasn't what you wanted.

                continue


I try to almost always avoid continue. I suggest rewriting the logic so the following line in the if block.

            words[word.rstrip().lower()] = None


You seem to be using words as a set. Use a set.

    d.close()

    # Add words in this word list to our master word lists
    for word in words:
        if rating == 1:
            if word in HIGH_WORDS:
                HIGH_WORDS[word] += 1
            else:
                HIGH_WORDS[word] = 1


Look at collections.Counter to simplify this.

        else:
            if word in LOW_WORDS:
                LOW_WORDS[word] += 1
            else:
                LOW_WORDS[word] = 1
t.close()

print HIGH_WORDS
print LOW_WORDS

# Classify a point, assuming training has already happened.
# P(C | F1 ... Fn) proportionate to P(C) * P(F1 | C) * ... * P(Fn | C)
def classify(username):
    # P(C)
    Phi = float(hi) / float(hi + low)
    Plow = float(low) / float(hi + low)


Add from __future__ import division so that all division produce floats. Then you don't have to cast to float. I also suggest calling the variable: probaility_high for greater clarity.

    ratio = Phi / Plow
    # Get data
    call(["curl", "-o", "tmp.dta", "http://www.okcupid.com/profile/" + username])
    d = open("tmp.dta", 'r')
    for line in d:
        for word in line.split(" "):


There is some duplication here. You should be able to write a function that returns the words from the url.

            # Calculate the probability of the feature given a class
            if word in HIGH_WORDS:
                numHi = float(HIGH_WORDS[word])
            else:
                numHi = 1
            if word in LOW_WORDS:
                numLow = float(LOW_WORDS[word])
            else:
                numLow = 1

            ratio = ratio * (numHi / float(hi)) / (numLow / float(low))
#            if word in HIGH_WORDS:
##                Phi = Phi * float(HIGH_WORDS[word]) / float(hi)
##            else:
##                Phi = Phi * (1 / float(hi))
#
#            if word in LOW_WORDS:
##                Plow = Plow * float(LOW_WORDS[word]) / float(low)
##            else:
##                Plow = Plow * (1 / float(low))


Don't leave dead code. Delete it.

    print ratio
    if ratio &gt; 1:
        return 1
    else:
        return 0

# Compute out of sample error on test set.
def getError():
    t = open(TEST_DATA, 'r')
    numUsers = 0.0
    total = 0.0
    for line in t:
        user = line.split(" ")[1]
        rating = int(line.split(" ")[0])
        prediction = classify(user)
        numUsers += 1.0
        if rating != prediction:
            total += 1.0


Total seems a confusing name, because its only the mismatches.

        print str(rating) + " " + str(prediction) + " " + user

    return total / numUsers



print getError()


You may want to look into numpy. It'll allow much more efficient operations on large amounts of data like this.