Script to separate shared vertices that have different UV's, normals and colors
As you noted most data is stored per-face in blender; in the UI it is called "face corner data", the API calls it "loops".

This is fairly simple: just loop over the polygons, get all vertex data you need, pack it into hex, append the vertex to a list if it is a new one (using a dummy vertex to avoid duplicating too often due to inconsistent normals) and finally index the polygon's vertices in the vertex list to get the polygon's exported indices.

#output
mesh_vertices = []
mesh_triangles = []
#used to ignore the normals for checking equality
dummy_vertices = []

for polygon in me.polygons:
    tri=[]
    for loop_index in polygon.loop_indices:
        vertex_index = me.loops[loop_index].vertex_index
        co = me.vertices[vertex_index].co
        no = me.loops[loop_index].normal

        bfb_vertex = pack('3f',co.x, co.y, co.z)
        bfb_normal = pack('3f',no.x, no.y, no.z)
        if me.vertex_colors:
            bfb_col = pack('4B',int(me.vertex_colors[0].data[loop_index].color.b*255),
                                int(me.vertex_colors[0].data[loop_index].color.g*255),
                                int(me.vertex_colors[0].data[loop_index].color.r*255),
                                int(me.vertex_colors[1].data[loop_index].color.b*255))
        bfb_uv = b''
        for uv_layer in me.uv_layers:
            bfb_uv+= pack('2f',uv_layer.data[loop_index].uv.x, 1-uv_layer.data[loop_index].uv.y)
        #we have to add new verts also if the UV is different!
        if bfb_vertex+bfb_uv not in dummy_vertices:
            dummy_vertices.append(bfb_vertex+bfb_uv)
            mesh_vertices.append(bfb_vertex+bfb_normal+bfb_col+bfb_uv)
        bfb_vert_index = dummy_vertices.index(bfb_vertex+bfb_uv)
        tri.append(bfb_vert_index)
    mesh_triangles.append(pack('3h',*tri))