'Returns the information gain (IG, 0.0-1.0) for the given feature,
by measuring how much the feature contributes to each document type (class).
High information gain means low entropy. Low entropy means predictability,
i.e., a feature that is biased towards some class(es),
i.e., a feature that occurs more in one document type and less in others.'

def information_gain(self, word):

    if (not self._ig):
        H = entropy
        C = dict.fromkeys(self.classes, 0)
        for d in self.documents:
            C[d.type] += 1
        HC = H(C.values())
        F = set(self.features)
        V = dict(((f, defaultdict((lambda : defaultdict((lambda : 0))))) for f in F))
        for d in self.documents:
            if (self.weight in (IG, GR, INFOGAIN, GAINRATIO)):
                d_vector = dict.fromkeys(d.terms, True)
            else:
                d_vector = d.vector
            for (f, v) in d_vector.items():
                if isinstance(v, float):
                    v = round(v, 1)
                V[f][v][d.type] += 1
        for f in F:
            for (type, n) in C.items():
                V[f][0][type] += (n - sum((V[f][v][type] for v in V[f])))
        for f in F:
            Vf = V[f]
            n = sum((sum(Vf[v].values()) for v in Vf))
            n = (float(n) or 1)
            ig = HC
            si = 0
            for Cv in Vf.values():
                Cv = Cv.values()
                pv = (sum(Cv) / n)
                ig = (ig - (pv * H(Cv)))
                si = (si + H([pv]))
            self._ig[f] = ig
            self._gr[f] = (ig / (si or 1))
    return self._ig.get(word, 0.0)
