__author__ = 'charles'
from twitter_connection import TwitterAgent
from IRToolKit import TF_IDF
import multiprocessing

seed_list = [
    "nytimes",
    "BBC",
    "TheEconomist",
    "CBCNews",
    "Forbes",
    "CNN",
    "washingtonpost",
    "Reuters",
    "globeandmail",
]

replace_dict = {
    "http": ("http", "url"),
    "@" : ("@", "id"),
}

def clean_data(input_file):
    for line in input_file:
        buffer = ""
        for word in line.split():
            if replace_dict["http"][0] in word:
                buffer += replace_dict["http"][1]
                buffer += " "
            elif replace_dict["@"][0] in word:
                buffer += replace_dict["@"][1]
                buffer += " "
            else:
                buffer += word
                buffer += " "

def read_tweets_id_list(file_name):
    new_list = []
    with open(file_name) as f:
        for line in f:
            new_list.append(line.strip())
    return new_list

def thread_worker(seed_id):
    agent = TwitterAgent()
    agent.request(seed_id)
    tweets = agent.get_tweets(seed_id)
    with open("inputs/"+seed_id+".tweets", "w") as f:
        for tweet in tweets["tweets"]:
            f.write(tweet+"\n")
    with open("inputs/"+seed_id+".retweets", "w") as f:
        for tweet in tweets["retweets"]:
            f.write(tweet[0]+": "+tweet[1]+"\n")
    with open("inputs/"+seed_id+".tweets") as f:
        handler = TF_IDF.DocumentHandler(f, frequency_filter=3)
        result = handler.get_term_frequency()
    print result


MINING = True
PARALLEL = True
MANY = False
if __name__ == "__main__":

    if PARALLEL:
        pool = multiprocessing.Pool()
        # pool.map(thread_worker, ["CBCCanada", "CBCWorldNews", "nytchangster"])
        pool.map(thread_worker, read_tweets_id_list("tmp/twitter_ids.txt"))
    elif MANY:
        for account in read_tweets_id_list("tmp/twitter_ids.txt"):
            seed_id = account
            if MINING:
                agent = TwitterAgent()
                agent.request(seed_id)
                tweets = agent.get_tweets(seed_id)
                with open("inputs/"+seed_id+".tweets", "w") as f:
                    for tweet in tweets["tweets"]:
                        f.write(tweet+"\n")
                with open("inputs/"+seed_id+".retweets", "w") as f:
                    for tweet in tweets["retweets"]:
                        f.write(tweet[0]+": "+tweet[1]+"\n")
            with open("inputs/"+seed_id+".tweets") as f:
                handler = TF_IDF.DocumentHandler(f, frequency_filter=3)
                result = handler.get_term_frequency()
            print result
    else:
        seed_id = "BBCFood"
        if MINING:
            agent = TwitterAgent()
            agent.request(seed_id)
            tweets = agent.get_tweets(seed_id)
            with open("inputs/"+seed_id+".tweets", "w") as f:
                for tweet in tweets["tweets"]:
                    f.write(tweet+"\n")
            with open("inputs/"+seed_id+".retweets", "w") as f:
                for tweet in tweets["retweets"]:
                    f.write(tweet[0]+": "+tweet[1]+"\n")
        with open("inputs/"+seed_id+".tweets") as f:
            handler = TF_IDF.DocumentHandler(f, frequency_filter=3)
            result = handler.get_term_frequency()
        print result

    print "done"