Python script to harvest tweets to a MongoDb works with users but not hashtags. Any ideas why not?
Question: I'm playing around the Twitter API and am in the process of developing a
script to pull all Tweets with a certain hashtag down to a local mongoDB. I
have it working fine when I'm downloading tweets from users, but when
downloading tweets from a hashtag I get:

    
    
    return loads(fp.read(),
    AttributeError: 'int' object has no attribute 'read'
    

Can anyone offer their infinite wisdom into how I could get this script to
work?

To run, save it as a .py file, cd to the folder and run:

    
    
    python twitter.py
    

Code:

    
    
    __author__ = 'Tom Cusack'
    
    import pymongo
    import oauth2 as oauth
    import urllib2, json
    import sys, argparse, time 
    
    def oauth_header(url, consumer, token):
    
        params =  {'oauth_version': '1.0',
        'oauth_nonce': oauth.generate_nonce(),
        'oauth_timestamp': int(time.time()),
        }
        req = oauth.Request(method = 'GET',url = url, parameters = params)
        req.sign_request(oauth.SignatureMethod_HMAC_SHA1(),consumer, token)
        return req.to_header()['Authorization'].encode('utf-8')
    
    def main():
        ### Twitter Settings
        numtweets = '32000'
        verbose = 'store_true'
        retweet = 'store_false'
        CONSUMER_KEY = 'M7Xu9Wte0eIZvqhb4G9HnIn3G'
        CONSUMER_SECRET = 'c8hB4Qwps2aODQUx7UsyzQuCRifEp3PKu6hPQll8wnJGIhbKgZ'
        ACCESS_TOKEN = '3213221313-APuXuNjVMbRbZpu6sVbETbgqkponGsZJVT53QmG'
        ACCESS_SECRET = 'BJHrqWC9ed3pA5oDstSMCYcUcz2pYF3DmJ7jcuDe7yxvi'
        base_url = url = 'https://api.twitter.com/1.1/search/tweets.json?include_entities=true&count=200&q=#mongodb&include_rts=%s' % (retweet)
        oauth_consumer = oauth.Consumer(key = CONSUMER_KEY, secret = CONSUMER_SECRET)
        oauth_token = oauth.Token(key = ACCESS_TOKEN, secret = ACCESS_SECRET)
    
        ### Mongodb Settings
        uri = 'mongodb://127.0.0.1:27017/SARKY'
        if uri != None:
            try: 
                conn = pymongo.MongoClient(uri)
                print 'Pulling Tweets..'
            except:
                print 'Error: Unable to connect to DB. Check uri variable.'
                return
            uri_parts = pymongo.uri_parser.parse_uri(uri)
            db = conn[uri_parts['database']]
            db['twitter-harvest'].ensure_index('id_str')
    
        ### Helper Variables for Harvest
        max_id = -1
        tweet_count = 0
        stream = 0
    
        ### Begin Harvesting
        while True:
            auth = oauth_header(url, oauth_consumer, oauth_token)
            headers = {"Authorization": auth}
            request = urllib2.Request(url, headers = headers)
            try:
                stream = urllib2.urlopen(request)
            except urllib2.HTTPError, err:
                if err.code == 404:
                    print 'Error: Unknown user. Check --user arg'
                    return
                if err.code == 401:
                    print 'Error: Unauthorized. Check Twitter credentials'
                    return
            tweet_list = json.load(stream)
    
            if len(tweet_list) == 0:
                print 'No tweets to harvest!'
                return
            if 'errors' in tweet_list:
                print 'Hit rate limit, code: %s, message: %s' % (tweets['errors']['code'], tweets['errors']['message'])
                return
            if max_id == -1:
                tweets = tweet_list
            else:
                tweets = tweet_list[1:]
                if len(tweets) == 0:
                    print 'Finished Harvest!'
                    return
    
            for tweet in tweets:
                max_id = id_str = tweet['id_str']
                try:
                    if tweet_count == numtweets:
                        print 'Finished Harvest- hit numtweets!' 
                        return 
                    if uri != None:
                        db[user].update({'id_str':id_str},tweet,upsert = True)
                    else:
                        print tweet['text']
                    tweet_count+=1
                    if verbose == True and uri != None:
                        print tweet['text']
                except Exception, err:
                    print 'Unexpected error encountered: %s' %(err)
                    return    
            url = base_url + '&max_id=' + max_id
    
    if __name__ == '__main__':
        try:
            main()
        except SystemExit as e:
            if e.code == 0:
                pass
    


Answer: You initially set `stream = 0`. When your `try...except` block catches a HTTP
response with a code that isn't 404 or 401, `stream` is still equal to `0`,
but your `except` block doesn't break out of the function.

I'd look more closely at what this response says.

