Remove duplicate url's python
Question: I want to remove the duplicate url's from the file having list of url's. My
bugun_url_given.txt has "<http://www.bugun.com.tr/ara/Ak%20Parti/1>" and it
fetches all the url's and they are repeating.. It saves all the unique url's
in "bugun_url_collection.tx" here is my code:

    
    
    from cookielib import CookieJar
    import urllib2
    import json
    from bs4 import BeautifulSoup
    cj = CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    try:
        text_file = open('bugun_url_given.txt', 'r')
        for line in text_file:
            print line
            soup = BeautifulSoup(opener.open(line))
            links = soup.select('div.nwslist a')
            for link in links:
                print link
                #unique_url = set(map(lambda url : url.strip("/ "), links))
                with open('bugun_url_collection.txt', 'a') as f:
                    for link in links:
                        f.write(link.get('href') + '\n')
    except ValueError:
        pass               
    


Answer: 
                for link in links:
                    f.write(link.get('href') + '\n')
    

can become

    
    
                for link in set(link.get('href') for link in links):
                    f.write(link + '\n')
    

In response to the comment (which is correct), let's rewrite this properly:

    
    
    from cookielib import CookieJar
    import urllib2
    import json
    from bs4 import BeautifulSoup
    cj = CookieJar()
    opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
    
    
    def write_links_to_file(links):
        with open('bugun_url_collection.txt', 'a') as f:
            f.writeline(link)
    
    
    def get_links_from_file(text_file):
        for line in text_file:
            print line
            soup = BeautifulSoup(opener.open(line))
            links = soup.select('div.nwslist a')
            for link in links:
                yield link.get('href')
    
    
    with open('bugun_url_given.txt', 'r') as text_file:
        links = get_links_from_file(text_file)
    
    unique_links = set(link for link in links)
    write_links_to_file(unique_links)
    

