title: Python Code Speed Up
body: <p>My code should compare two vectors saved as dictionary (two pickle files) and save the result into a pickle file too. This works but very slowly. For one compare result I'm waiting about 7:2o min. Because I have a lot of videos (exactly 2033) this prog will run about 10 days. This is too long. How can I speed up my code for Python 2.7? </p>
  
  <pre><code>import math
  import csv
  import pickle
  from itertools import izip
  
  global_ddc_file = 'E:/global_ddc.p'
  io = 'E:/AV-Datensatz'
  v_source = ''
  
  def dot_product(v1, v2):
      return sum(map(lambda x: x[0] * x[1], izip(v1, v2))) # izip('ABCD', 'xy') --&gt; Ax By
  
  def cosine_measure(v1, v2):
      prod = dot_product(v1, v2)
      len1 = math.sqrt(dot_product(v1, v1))
      len2 = math.sqrt(dot_product(v2, v2))
      if (len1 * len2) &lt;&gt; 0:
          out = prod / (len1 * len2)
      else: out = 0
      return out
  
  def findSource(v):
      v_id = "/"+v[0].lstrip("&lt;http://av.tib.eu/resource/video").rstrip("&gt;")
      v_source = io + v_id
      v_file = v_source + '/vector.p'
      source = [v_id, v_source, v_file]
      return source
  
  def getVector(v, vectorCol):
      with open (v, 'rb') as f:
          try:
              vector_v = pickle.load(f)
          except: print 'file couldnt be loaded'
          tf_idf = []
          tf_idf = [vec[1][vectorCol] for vec in vector_v]
      return tf_idf
  
  def compareVectors(v1, v2, vectorCol):
      v1_source = findSource(v1)
      v2_source = findSource(v2)
      V1 = getVector(v1_source[2], vectorCol)
      V2 = getVector(v2_source[2], vectorCol)
      sim = [v1_source[0], v2_source[0], cosine_measure(V1, V2)]
      return sim
  
  #with open('videos_av_portal_cc_3.0_nur2bspStanford.csv', 'rb') as dataIn:
  with open('videos_av_portal_cc_3.0_vollstaendig.csv', 'rb') as dataIn:
  #with open('videos_av_portal_cc_3.0.csv', 'rb') as dataIn:
      try:
          reader = csv.reader(dataIn)
  
          v_source = []
          for row in reader:
              v_source.append(findSource(row))
          #print v_source
  
          for one in v_source:
              print one[1]
              compVec = []
              for another in v_source:
                  if one &lt;&gt; another: 
                      compVec.append(compareVectors(one, another, 3))
              compVec_sort = sorted(compVec, key=lambda cosim: cosim[2], reverse = True) 
  
              # save vector file for each video
              with open (one[1] + '/compare.p','wb') as f:
                  pickle.dump(compVec_sort,f)
  
      finally:
          dataIn.close()  
  </code></pre>