title: Email harvester in Python: How to improve performance?
body: <p>I have found a program called "Best Email Extractor" <a href="http://www.emailextractor.net/" rel="nofollow">http://www.emailextractor.net/</a>. The website says it is written in Python. I tried to write a similar program. The above program extracts about 300 - 1000 emails per minute. My program extracts about 30-100 emails per hour. Could someone give me tips on how to improve the performance of my program? I wrote the following:</p>
  
  <pre><code>import sqlite3 as sql
  import urllib2
  import re
  import lxml.html as lxml
  import time
  import threading
  
  
  def getUrls(start):
  
      urls = []
      try:
          dom = lxml.parse(start).getroot()
          dom.make_links_absolute()
  
          for url in dom.iterlinks():
              if not '.jpg' in url[2]:
                  if not '.JPG' in url[2]:
                      if not '.ico' in url[2]:
                          if not '.png' in url[2]:
                              if not '.jpeg' in url[2]:
                                  if not '.gif' in url[2]:
                                      if not 'youtube.com' in url[2]:
                                          urls.append(url[2])
      except:
          pass
  
      return urls
  
  def getURLContent(urlAdresse):
  
      try:
        url = urllib2.urlopen(urlAdresse)
        text = url.read()
        url.close()
        return text
      except:
          return '&lt;html&gt;&lt;/html&gt;'
  
  def harvestEmail(url):
      text = getURLContent(url)
  
      emails = re.findall('[\w\-][\w\-\.]+@[\w\-][\w\-\.]+[a-zA-Z]{1,4}', text)
  
      if emails:
          if saveEmail(emails[0]) == 1:
              print emails[0]
  
  def saveUrl(url):
  
      connection = sql.connect('url.db')
  
      url = (url, )
  
      with connection:
          cursor = connection.cursor()
          cursor.execute('SELECT COUNT(*) FROM urladressen WHERE adresse = ?', url)
          data = cursor.fetchone()
          if(data[0] == 0):
              cursor.execute('INSERT INTO urladressen VALUES(NULL, ?)', url)
              return 1
          return 0
  
  def saveEmail(email):
      connection = sql.connect('emails.db')
      email = (email, )
  
      with connection:
          cursor = connection.cursor()
          cursor.execute('SELECT COUNT(*) FROM addresse WHERE email = ?', email)
          data = cursor.fetchone()
          if(data[0] == 0):
              cursor.execute('INSERT INTO addresse VALUES(NULL, ?)', email)
              return 1
      return 0
  
  def searchrun(urls):
      for url in urls:
          if saveUrl(url) == 1:
              #time.sleep(0.6)
              harvestEmail(url)
              print url
              urls.remove(url)
              urls = urls + getUrls(url)
  
  urls1 = getUrls('http://www.google.de/#hl=de&amp;tbo=d&amp;output=search&amp;sclient=psy-ab&amp;q=DVD')
  urls2 = getUrls('http://www.google.de/#hl=de&amp;tbo=d&amp;output=search&amp;sclient=psy-ab&amp;q=Jolie')
  urls3 = getUrls('http://www.finanzen.net')
  urls4 = getUrls('http://www.google.de/#hl=de&amp;tbo=d&amp;output=search&amp;sclient=psy-ab&amp;q=Party')
  urls5 = getUrls('http://www.google.de/#hl=de&amp;tbo=d&amp;output=search&amp;sclient=psy-ab&amp;q=Games')
  urls6 = getUrls('http://www.spiegel.de')
  urls7 = getUrls('http://www.kicker.de/')
  urls8 = getUrls('http://www.chessbase.com')
  urls9 = getUrls('http://www.nba.com')
  urls10 = getUrls('http://www.nfl.com')
  
  
  try:
      threads = []
      urls = (urls1, urls2, urls3, urls4, urls5, urls6, urls7, urls8, urls9, urls10)
  
      for urlList in urls:
          thread = threading.Thread(target=searchrun, args=(urlList, )).start()
          threads.append(thread)
      print threading.activeCount()
      for thread in threads:
          thread.join()
  except RuntimeError:
      print RuntimeError
  </code></pre>