Splitting many data files by quintile The program below is about finding the values below the percentage and splitting them into five different output files. This works great for small input data files. But when I am trying to run it for a big data it takes ages to compute. Any suggestions on optimizing the program so that I can get fast computation?

#seperate by col 5
import numpy as np
import os
import re
import math
import csv
import sys
import time
import multiprocessing
from scipy import stats

curdir = 'Tools/'
inputdir = 'input/'
filename = 'input'
outputdir = 'output/'

def seperate(filename):
    print "Begin seperating File: %s" % filename
    dtype = np.dtype([('id1',str,12),('id2',str,12),('c1',int),('c2',int),('c3',float)])
    data = np.loadtxt(curdir + inputdir + filename,dtype=dtype)
    col = 'c3'
    col_data = np.array(data[:][col])
    cache = {}
    def get_per(per):
            if(cache.has_key(per)):
                return cache.get(per)
            else:
                r = 100 - stats.percentileofscore(col_data,per)
                cache[per] = r
                return r
    o1 = file(curdir + outputdir+ filename + '_0_20','w')
    o2 = file(curdir + outputdir+ filename + '_20_40','w')
    o3 = file(curdir + outputdir+ filename + '_40_60','w')
    o4 = file(curdir + outputdir+ filename + '_60_80','w')
    o5 = file(curdir + outputdir+ filename + '_80_100','w')
    for line in data:
        per = get_per(line[col])
        output_format = "%s %s %d %d %.1f\n"
        output_data = (line['id1'],line['id2'],line['c1'],line['c2'],line['c3'])
        if per&gt;=0 and per &lt; 20:
            o1.write(output_format % output_data)
        elif per&gt;=20 and per&lt;40:
            o2.write(output_format % output_data)
            pass
        elif per&gt;=40 and per&lt;60:
            o3.write(output_format % output_data)
            pass
        elif per&gt;=60 and per&lt;80:
            o4.write(output_format % output_data)
            pass
        elif per&gt;=80 and per&lt;=100:
            o5.write(output_format % output_data)
            pass
    o1.close()
    o2.close()
    o3.close()
    o4.close()
    o5.close()

    print "Finish seperating File: %s" % filename


ps = []
print "Reading Files"
for parent,dirNames,fileNames in os.walk(curdir+inputdir):
    if fileNames:
        #multi_process
        print fileNames
        for fileName in fileNames:
            m = re.match(r'^(?!\.)',fileName)
            if m:
                ps.append(multiprocessing.Process(target=seperate,args=(fileName,)))

#runing by multiple processes
for p in ps:
    p.start()
for p in ps:
    p.join()
Calling percentileofscore for every distinct value is rather inefficient vs. the alternative of making just one call scoreatpercentile(col_data, [0,20,40,60,80,100]). You could directly compare the column values against the limits returned by that function.
Apart from the above you are mostly doing disk I/O. You may have made matters worse by using multiprocessing. In particular, because you lauch as many processes as you have input files, you may have way too many processes competing for RAM and disk access.