Code Golf: Make a Web Crawler A Web Crawler is a program that starts with a small list of websites and gets the source code for all of them. It then finds all the links in those source codes, gets the website it links to then puts them in a list. It then starts again but with the new list of websites.The main reason for a Web Crawler is to find all (or most) of the websites in the internet. Google use this to power their search engine.Wikipedia Page about Web Crawlers

Your task is to create a Web Crawler in any language you wish. 

Rules


  
  Your first list can contain any amount of websites &lt;= 5 and any websites you wish.
  Your web crawler can ignore any websites with the HTTP GET request or any other pages other than the home page, e.g. www.somewebsite.com/?anything=1 and www.somewebsite.com/secondpage can be ignored, we are only trying to find the website/domain, e.g. www.somewebsite.com
  You can use any method you wish to store the list of websites currently being "crawled".
  Although this is code golf, the highest voted answer/solution will win, and don't forget to have fun!
Javascript, 491

Golfed

var C=function(c){w=c.length;for(i=0;i&lt;w;i++){var b=new XMLHttpRequest;b.setRequestHeader("Access-Control-Allow-Origin","*");b.setRequestHeader("X-Requested-With","XMLHttpRequest");b.open("GET",c[i],!0);b.send();if(200===b.status){a=b.responseText.match(/\&lt;[(a|img)]+\s[^\&gt;]+[(http|https)]+:\/\/([\w\.])+([.])+.+?[^.]+?./gi);u=c[i].substr(7);re=RegExp(u,"i");for(j=0;j&lt;a.length;j++)if(a[j].match(re)&amp;&amp;(b=a[j].match(/[(http|https)]+:\/\/([\w\.])+([.])/gi))&amp;&amp;!c.indexOf(b))c.push(a[j]),w++}}};


Un-Golfed

var C = function(s){
  w = s.length;
  for(i = 0; i &lt; w; i++){
    var r = new XMLHttpRequest();
    r.setRequestHeader("Access-Control-Allow-Origin", "*")
    r.setRequestHeader("X-Requested-With", "XMLHttpRequest");    
    r.open('GET', s[i], true); r.send();
    if(r.status === 200) {
      a = r.responseText.match(/\&lt;[(a|img)]+\s[^\&gt;]+[(http|https)]+:\/\/([\w\.])+([.])+.+?[^.]+?./gi);
      u = s[i].substr(7),
      re = new RegExp(u,'i');
      for(j = 0; j &lt; a.length; j++){
        if(!a[j].match(re)) continue;
        var uri = a[j].match(/[(http|https)]+:\/\/([\w\.])+([.])/gi);
        if(uri &amp;&amp; !s.indexOf(uri)) {
          s.push(a[j]); w++;
        }
      }
    }
  }
}


Method

var c = new C(['http://www.cnn.com', 'http://www.usatoday.com', 'http://www.huffingtonpost.com', 'http://www.drudgereport.com/', 'http://news.cnet.com']);