python, google app engine, and twitter: my program is not working right
<p>I made this silly little app that uses twitters api and scans through the last 'x' tweets, finds tiny URL's, find out what URL that the tiny URL is pointing to, accumulates the frequency of top level domains that have been tweeted about, and outputs a html tagcloud page based on the frequencies.</p>

<p>But it doesn't work quite right. There are some parsing issues that I'll take care of. I don't think that's what I'm most concerned with. What I'd really like to try to fix is two things:</p>

<p>Sometimes the app crashes (especially if I select a larger number of tweets to scan), usually "Downloaderror Applicationerror 2" and "Downloaderror Applicationerror 5". The other problem I have not been able to work around is the time it takes to run...it is SLOW. I tried setting short timeouts. But still it takes FOREVER to run if I scan a lot of tweets.</p>

<p>Any ideas? Thanks!</p>

<pre><code>import logging
import wsgiref.handlers
from google.appengine.ext import webapp
import urllib2
from urllib import urlencode
from urllib2 import urlopen
from BeautifulSoup import BeautifulStoneSoup
import socket
import re
from urlparse import urlparse
from google.appengine.api import urlfetch
#from google.appengine.api.urlfetch import DownloadError

#timeout = 3
#socket.setdefaulttimeout(timeout)

class Link():
def __init__(self, a, b):
    self.link = a
    self.number = b

def __str__(self):
    return "%s ; %s" % (self.link, self.number)

def getFeed(i):
    r = urlopen('http://search.twitter.com/search.atom?q=twitter&amp;since=2010-02-28&amp;rpp=100&amp;page=%i' %(i))
    return r

def processFeed(f):

    soup = BeautifulStoneSoup(f.read(),selfClosingTags=["link"])
    tweets = []
    final = {}
    k = 0
    j = 0

    for entry in soup.findAll("entry"):
        title = entry.find('title').contents[0]
        if 'http' in title:
            temp = re.search("(?P&lt;url&gt;https?://[^\s]+)", title).group("url")
            tweets.append(Link(temp,0))

#The for loop below takes care of good urls (yahoo.com), non-sense url (http://asdfaf,     http://blah.blah), pages not found (http://google.com/tuff).
#BUT...there are certain response from the host server that just totally crashes the     program.
#Downloaderror Applicationerror 5 is a timeout error and Downloaderror Applicationerror     2 is also a connection error
    for address in tweets:
    #address.link = address.link.strip()
    try:
        response = urllib2.urlopen(address.link)
        #response = urlfetch.fetch(address.link, method=urlfetch.HEAD, deadline=10)
        #url_destination = response.final_url
        url_destination = response.url
        address.link = url_destination
        j = j + 1
    except urllib2.URLError:
        pass
    except urllib2.HTTPError:
        pass
    except UnicodeDecodeError:
        pass

while k &lt; j:
    o = urlparse(tweets[k].link)
    tweets[k].link = o.netloc
    k = k + 1

for link in tweets:
    temp = link.link.split('.')
    temp[len(temp)-1] = temp[len(temp)-1][0:3]
    temp = [temp[len(temp)-2],temp[len(temp)-1]]
    link.link = '.'.join(temp)
    if link.link in final:
        final[link.link] += 1
    else:
        final[link.link] = 1
    return final

def TagCloudDivHeader(txt):
    return "&lt;div class = 'tagcloud'&gt;\n&lt;div class = 'tagcloudtitle'&gt;%s&lt;/div&gt;\n" % txt

def TagCloudDivFooter():
    return "&lt;/div&gt;\n"


def size(freq):
    return freq           

def writeTerm(term,freq):
    return " &lt;span class='term' style='font-size:"+str(size(freq))+"em'&gt;" + "&lt;a href = 'http://%s'&gt;" %term + term.encode('ISO-8859-1', 'replace') + "&lt;/a&gt;&lt;/span&gt; "+ "\n"

def genForm(prompt = ""):

    numberoftweets = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]
    res = ""
    if prompt:
        res += "&lt;div class= 'formtitle'&gt;%s&lt;/div&gt;" % (prompt)
    res += """&lt;form action="index.py" method="post"&gt;"""
    res +="""&lt;label for="Tweets"&gt;Number of Tweets to scan:&lt;/label&gt;
        &lt;select id="Tweets" name="Tweets"&gt;"""   
    for n in numberoftweets:
        res += "&lt;option value = \"%i\"&gt;%i&lt;/option&gt;" %(n*100,n*100)
    res += "&lt;/select&gt;"
    res += '&lt;input type="submit" value="Go" name="gobtn"/&gt; &lt;/form&gt;'
    res += "&lt;/br&gt;WARNING!!!! The fewer Tweets you scan, the more stable this program is!!!!"
    return res


def makeTagCloud(cloudtitle, items):
    result = ''
    result += TagCloudDivHeader(cloudtitle)
    for thing in items:
        result += writeTerm(thing, items[thing])
    result += TagCloudDivFooter()
    result += HTMLFooter()
    return result

def HTMLHeader(pageheader = ""):
    s = "&lt;!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\"&gt;"
    s += "\n&lt;html&gt;&lt;head&gt;\n &lt;title&gt;%s&lt;/title&gt;\n" % pageheader
    s = s + "&lt;link rel='stylesheet' href='/assets/mystyles.css' type='text/css' /&gt;\n"
    s = s + "&lt;/head&gt;\n"
    s = s + "&lt;body&gt;\n"
    return s

def HTMLFooter():
    return "&lt;/body&gt;\n&lt;/html&gt;"   

def generateLinks():

    result = ""
    result += HTMLHeader("Who's getting the most traffic from Twitter?")    
    result += "&lt;p&gt;" + "&lt;a href = 'results/'&gt;Proceed?&lt;/a&gt;" + "&lt;/p&gt;\n"        
    result += HTMLFooter()
    return result

class MainHandler(webapp.RequestHandler):

    def get(self):
        self.response.headers['Content-Type'] = 'text/html'
        path = self.request.path
        logging.info("path is " +  path)
        form = genForm()

        contents = generateLinks()
        self.response.out.write(HTMLHeader("Who's getting the most traffic from Twitter?"))
        self.response.out.write(form)
        self.response.out.write(HTMLFooter())

    def post(self):
        self.response.out.write("Where are links are Twitter taking you?")
        self.response.out.write(HTMLHeader("Domain cloud for Twitter Tweets"))
        tweets = int(self.request.get('Tweets'))
        tweets = int(tweets/100)
        self.response.out.write(makeTagCloud("Domains most linked to by Tweets", processFeed(getFeed(tweets))))

def main():
    application = webapp.WSGIApplication([('/.*', MainHandler)],debug=True)
    wsgiref.handlers.CGIHandler().run(application)

if __name__ == '__main__':
  main()
</code></pre>
