Code Golf: Make a Web Crawler
Javascript, 491

Golfed

var C=function(c){w=c.length;for(i=0;i&lt;w;i++){var b=new XMLHttpRequest;b.setRequestHeader("Access-Control-Allow-Origin","*");b.setRequestHeader("X-Requested-With","XMLHttpRequest");b.open("GET",c[i],!0);b.send();if(200===b.status){a=b.responseText.match(/\&lt;[(a|img)]+\s[^\&gt;]+[(http|https)]+:\/\/([\w\.])+([.])+.+?[^.]+?./gi);u=c[i].substr(7);re=RegExp(u,"i");for(j=0;j&lt;a.length;j++)if(a[j].match(re)&amp;&amp;(b=a[j].match(/[(http|https)]+:\/\/([\w\.])+([.])/gi))&amp;&amp;!c.indexOf(b))c.push(a[j]),w++}}};


Un-Golfed

var C = function(s){
  w = s.length;
  for(i = 0; i &lt; w; i++){
    var r = new XMLHttpRequest();
    r.setRequestHeader("Access-Control-Allow-Origin", "*")
    r.setRequestHeader("X-Requested-With", "XMLHttpRequest");    
    r.open('GET', s[i], true); r.send();
    if(r.status === 200) {
      a = r.responseText.match(/\&lt;[(a|img)]+\s[^\&gt;]+[(http|https)]+:\/\/([\w\.])+([.])+.+?[^.]+?./gi);
      u = s[i].substr(7),
      re = new RegExp(u,'i');
      for(j = 0; j &lt; a.length; j++){
        if(!a[j].match(re)) continue;
        var uri = a[j].match(/[(http|https)]+:\/\/([\w\.])+([.])/gi);
        if(uri &amp;&amp; !s.indexOf(uri)) {
          s.push(a[j]); w++;
        }
      }
    }
  }
}


Method

var c = new C(['http://www.cnn.com', 'http://www.usatoday.com', 'http://www.huffingtonpost.com', 'http://www.drudgereport.com/', 'http://news.cnet.com']);