Concerting Tweets to python dictionary
Question: I want to analyse twitter data.I have downloaded some tweets and saved them in
a .txt file.

When I tried to extract useful information from the tweets data , i was not
able to make any progress because for a beginner like me it seems very
difficult to extract tweets , location etc.

while googling i found if we convert json into dictionary it would be easy to
extract the info.

Now I want to convert my JSON data to python dictionaries. I don't know how to
proceed.

Here is the code used to save tweets

    
    
    import tweepy
    import json
    import jsonpickle
    
    consumer_key = "*********"
    consumer_secret = "*******"
    
    access_token = "************"
    access_token_secret = "**********"
    
    auth = tweepy.AppAuthHandler(consumer_key, consumer_secret)
    auth.set_access_token(access_token, access_token_secret)
    
    
    # It make the Tweepy API call auto wait (sleep) when it hits the rate limit  and continue upon expiry of the window. 
    api = tweepy.API(auth, wait_on_rate_limit=True,  wait_on_rate_limit_notify=True)
    
    if (not api):
        print ("Can't Authenticate")
        sys.exit(-1)
    
    
    searchQuery = 'SomeHashtag'
    maxTweets = 10000000 # Some arbitrary large number
    tweetsPerQry = 100
    fName = 'file.txt'
    
    sinceId = None
    max_id = "Latest tweet ID"
    
    tweetCount = 0
    print("Downloading max {0} tweets".format(maxTweets))
    with open(fName, 'a') as f:
    
        while tweetCount < maxTweets:
            try:
                if (max_id <= 0):
                    if (not sinceId):
                        new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry)
    
                    else:
                        new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
                                            since_id=sinceId)
                else:
                    if (not sinceId):
                        new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
                                            max_id=str(max_id - 1))
                    else:
                        new_tweets = api.search(q=searchQuery, lang ="en", count=tweetsPerQry,
                                            max_id=str(max_id - 1),
                                            since_id=sinceId)
    
                if not new_tweets:
                    print("No more tweets found")
                    break
                for tweet in new_tweets:
                    f.write(jsonpickle.encode(tweet._json, unpicklable=False) + '\n')
    
                tweetCount += len(new_tweets)
                print("Downloaded {0} tweets".format(tweetCount))
                max_id = new_tweets[-1].id
            except tweepy.TweepError as e:
                # Just exit if any error
                print("some error : " + str(e))
                break
    
        print ("Downloaded {0} tweets, Saved to {1}".format(tweetCount, fName))
    


Answer: It seems you can just read your file line by line and unpickle it using
[`jsonpickle.decode`](https://jsonpickle.github.io/api.html#jsonpickle.decode)
method:

    
    
    tweets = []
    with open(filename) as f:
        for line in f:
            tweets.append(jsonpickle.decode(line))
    

And I think you can bypass third-party library at all:

    
    
    import json
    with open(filename, 'w') as f:
        for tweet in new_tweets:
            f.write(json.dumps(tweet) + '\n')
    
    tweets = []
    with open(filename) as f:
        for line in f:
            tweets.append(json.loads(line))
    

