title: Web crawler using perl
body: <p>I want to develop a web crawler which starts from a seed URL and then crawls 100 html pages it finds belonging to the same domain as the seed URL as well as keeps a record of the traversed URLs avoiding duplicates. I have written the following but the $url_count value does not seem to be incremented and the retrieved URLs contain links even from other domains. How do I solve this? Here I have inserted stackoverflow.com as my starting URL.</p>
  
  <pre><code>use strict;
  use warnings;
  
  use LWP::Simple;
  use LWP::UserAgent;
  use HTTP::Request;
  use HTTP::Response;
  
  
  ##open file to store links
  open my $file1,"&gt;&gt;", ("extracted_links.txt");
  select($file1); 
  
  ##starting URL
  my @urls = 'http://stackoverflow.com/';
  
  my $browser = LWP::UserAgent-&gt;new('IE 6');
  $browser-&gt;timeout(10);
  my %visited;
  my $url_count = 0;
  
  
  while (@urls) 
  {
       my $url = shift @urls;
       if (exists $visited{$url}) ##check if URL already exists
       {
           next;
       }
       else
       {
           $url_count++;
       }         
  
       my $request = HTTP::Request-&gt;new(GET =&gt; $url);
       my $response = $browser-&gt;request($request);
  
       if ($response-&gt;is_error()) 
       {
           printf "%s\n", $response-&gt;status_line;
       }
       else
       {
           my $contents = $response-&gt;content();
           $visited{$url} = 1;
           @lines = split(/\n/,$contents);
           foreach $line(@lines)
           {
               $line =~ m@(((http\:\/\/)|(www\.))([a-z]|[A-Z]|[0-9]|[/.]|[~]|[-_]|[()])*[^'"&gt;])@g;
               print "$1\n";  
               push @urls, $$line[2];
           }
  
           sleep 60;
  
           if ($visited{$url} == 100)
           {
              last;
           }
      }
  }
  
  close $file1;
  </code></pre>