data/candidates/dmoz/sampling/sample_urls.py
def filter_duplicate_sites():
    """
    Read urls from infile and output one url for each site
    """
    infile = sys.argv[1]
    outfile = sys.argv[2]
    out = open(outfile, "w")
    
    sites = set()
    with open(infile) as lines:
        for line in lines:
            url = line.strip()
            s = URLUtility.get_host(url)
            if s not in sites:
                out.write(s + "\n")
                sites.add(s)
    out.close()

    Read urls from infile and output one url for each site
    
Read urls from infile and output one url for each site
Read urls from infile and output one url for each site
def filter_duplicate_sites():
    infile = sys.argv[1]
    outfile = sys.argv[2]
    out = open(outfile, "w")
    sites = set()
    with open(infile) as lines:
        for line in lines:
            url = line.strip()
            s = URLUtility.get_host(url)
            if s not in sites:
                out.write(s + "\n")
                sites.add(s)
    out.close()
def filter_duplicate_sites ( ) : infile = sys . argv [ 1 ] outfile = sys . argv [ 2 ] out = open ( outfile , "w" ) sites = set ( ) with open ( infile ) as lines : for line in lines : url = line . strip ( ) s = URLUtility . get_host ( url ) if s not in sites : out . write ( s + "\n" ) sites . add ( s ) out . close ( )
Read urls from infile and output one url for each site
Read urls from infile and output one url for each site
"""
    Read urls from infile and output one url for each site
    """