title: How can I use Perl's LWP::UserAgent to fetch the same URL with different query strings?
body: <p>I have a running LWP::UserAgent that should be applied on following URL:</p>
  
  <pre><code>http://dms-schule.bildung.hessen.de/suchen/suche_schul_db.html?show_school=5503
  </code></pre>
  
  <p>This runs with many many similar targets see the following endings:</p>
  
  <pre><code>html?show_school=5503
  html?show_school=9002
  html?show_school=5512
  </code></pre>
  
  <p>I want to do this with use LWP::UserAgent:</p>
  
  <pre><code>for my $i (0..10000) 
  
  { $ua-&gt;get(' [here the URL should be applied] ', id =&gt; 21, extern_uid =&gt; $i); 
  # process reply }
  </code></pre>
  
  <p>In any case, using a loop like this for that kind of job is a way to do it. I guess the LWP's API does not aim to replace the functionality of the core Perl, and I can use Perl loops to query multiple URLs.</p>
  
  <p>The code that does not run because the loop has to be applied:</p>
  
  <pre><code>#use strict;
  
  use DBI;
  use LWP::UserAgent;
  use HTTP::Request::Common;
  use HTML::TreeBuilder::XPath;
  
  # first get a list of all schools
  my ($url = '[here the url should be applied] =',id);
  
  for my $id (0..10000) {
    $ua-&gt;get(' [here the url should be applied ] ', id =&gt; 21, extern_uid =&gt; $i);
    # process reply
  }  
  
  #my $request = POST $url,
  #                 [
  #         Schulsuche=&gt; "Ergebnisse anzeigen",
  #         order =&gt; "schule_ort",
  #         schulname =&gt; undef, 
  #         schulort =&gt; undef, 
  #         typid =&gt; "11",
  #         verbinder =&gt; "AND"
  #                 ];
  
  my $ua = LWP::UserAgent-&gt;new;
  print "getting all schools - this could take some time\n";
  my $response = $ua-&gt;request($request);
  
  # extract the ids
  my @ids = $response-&gt;content =~ /getSchoolDetail\((\d+)/gs;
  print "found " . scalar @ids . " schools\n";
  
  # for this demo we only do the first 5
  my @ids_to_do = @ids[0..4];
  
  # use your own user and password
  my $dbh = DBI-&gt;connect("DBI:mysql:database=schulen", "user", "pass", { AutoCommit =&gt; 0 }) or die $!;
  
  my $sth = $dbh-&gt;prepare(&lt;&lt;sqlend);
     insert into schulen ( name , plz , ort, strasse , tel, fax , mail, quelle , original_id )
                 values  ( ?, ?, ?, ?, ?, ?, ?, ?, ? )
  sqlend
  
  # now loop over ids
  for my $id (@ids_to_do) {
  
    # get detail information for id
    my $res = $ua-&gt;get("[url]=&gt; &amp;gid=$id");
  
    # parse the response
    my $tree = HTML::TreeBuilder::XPath-&gt;new;
    $tree-&gt;parse($res-&gt;content);
  
    my $xpath = q|//div[@id='MCinhview']//div[@class='contentitem']//table|;
    my ($adress_table, $tel_table) = $tree-&gt;findnodes($xpath);
  
    my ($adr) = $adress_table-&gt;find("td");
    my ($name, $city, $street) = map { s/^\s*//; s/\s*$//; $_ } ($adr-&gt;content_list)[2,4,6];
  
    my($plz, $ort) = $city =~ /^(\d+)\s*(.*)/;
    my ($tel, $fax, $mail) = map { s/^\s*//; s/\s*$//; $_ } map { ($_-&gt;content_list)[1] } $tel_table-&gt;find("td");
  
    $sth-&gt;execute($name, $plz, $ort, $street, $tel, $fax, $mail, "SA", $id);
    $dbh-&gt;commit;
  
    $tree-&gt;delete;
  
    print "$name done\n";
  }
  </code></pre>
  
  <hr>
  
  <p><strong>update on sunday october 25 th:</strong> I have applied the advice from OmnipotentEntity. </p>
  
  <pre><code>#!/usr/bin/perl -W
  
  use strict;
  use warnings;         # give out some warnings if something does not run well
  use diagnostics;      # tell me when something is wrong 
  use DBI;
  use LWP::UserAgent;
  use HTTP::Request::Common;
  use HTML::TreeBuilder::XPath;
  
  # first get a list of all schools
  
  my $ua = LWP::UserAgent-&gt;new;
  
  $ua-&gt;agent("Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.8.1.7) Gecko/20070914 Firefox/2.0.0.7"); 
  
  #pretending to be firefox on linux.
  
  for my $i (0..10000) {
    my $request = HTTP::Request-&gt;new(GET =&gt; sprintf(" here to put the URL into =%d", $i));
    $request-&gt;header('Accept' =&gt; 'text/html');
    my $response = $ua-&gt;request($request);
    if ($response-&gt;is_success) {
      $pagecontent = $response -&gt; content;
    }
  # now we can do whatever with the $pagecontent
  
  }
  my $request = POST $url,
  [
            order =&gt; "schule_ort",
            schulname =&gt; undef, 
            Basisdaten =&gt; undef,        
            Profil  =&gt; undef, 
            Schulort =&gt; undef, 
            typid =&gt; "11",
            Fax  =&gt; 
            Homepage  =&gt; undef, 
            verbinder =&gt; "AND"
  
  ];
  
  print "getting all schools - this could take some time\n";
  my $response = $ua-&gt;request($request);
  
  # extract the ids
  my @ids = $response-&gt;content =~ /getSchoolDetail\((\d+)/gs;
  print "found " . scalar @ids . " schools\n";
  
  # for this demo we only do the first 5
  my @ids_to_do = @ids[0..4];
  
  # use your own user and password
  my $dbh = DBI-&gt;connect("DBI:mysql:database=schulen", "user", "pass", { AutoCommit =&gt; 0 }) or die $!;
  
  my $sth = $dbh-&gt;prepare(&lt;&lt;sqlend);
     insert into schulen ( name , plz , ort, strasse , tel, fax , mail, quelle , original_id )
                 values  ( ?, ?, ?, ?, ?, ?, ?, ?, ? )
  sqlend
  
  # now loop over ids
  for my $id (@ids_to_do) {
  
    # get detail information for id
    my $res = $ua-&gt;get(" here to put the URL into =&gt; &amp;gid=$id");
  
    # parse the response
    my $tree = HTML::TreeBuilder::XPath-&gt;new;
    $tree-&gt;parse($res-&gt;content);
  
    my $xpath = q|//div[@id='MCinhview']//div[@class='floatbox']//table|;
    my ($adress_table, $tel_table) = $tree-&gt;findnodes($xpath);
  
    my ($adr) = $adress_table-&gt;find("td");
    my ($name, $city, $street) = map { s/^\s*//; s/\s*$//; $_ } ($adr-&gt;content_list)[2,4,6];
  
    my($plz, $ort) = $city =~ /^(\d+)\s*(.*)/;
    my ($tel, $fax, $mail) = map { s/^\s*//; s/\s*$//; $_ } map { ($_-&gt;content_list)[1] } $tel_table-&gt;find("td");
  
    $sth-&gt;execute($name, $plz, $ort, $street, $tel, $fax, $mail, "SA", $id);
    $dbh-&gt;commit;
  
    $tree-&gt;delete;
  
    print "$name done\n";
  }
  </code></pre>
  
  <p>I want to loop over the results and therefore I tried to apply the corresponding URLs but I got a bunch of errors:</p>
  
  <pre>
  suse-linux:/usr/perl # perl perl_mecha_example_two.pl
  Global symbol "$pagecontent" requires explicit package name at perl_mecha_example_two.pl line 24.
  Global symbol "$url" requires explicit package name at perl_mecha_example_two.pl line 29.
  Execution of perl_mecha_example_two.pl aborted due to compilation errors (#1)
      (F) You've said "use strict" or "use strict vars", which indicates 
      that all variables must either be lexically scoped (using "my" or "state"), 
      declared beforehand using "our", or explicitly qualified to say 
      which package the global variable is in (using "::").
  
  Uncaught exception from user code:
  Global symbol "$pagecontent" requires explicit package name at perl_mecha_example_two.pl line 24.
  Global symbol "$url" requires explicit package name at perl_mecha_example_two.pl line 29.
  Execution of perl_mecha_example_two.pl aborted due to compilation errors.
  at perl_mecha_example_two.pl line 86
  </pre>
  
  <p>Now the debugging part. What do I change? How to apply the URLs in the right way?</p>
  
  <p>When I use strict I'm not allowed to use a variable before I declare it. The usual fix is to prepend <code>my</code>, e.g. <code>my $url</code> and <code>my $pagecontent</code> on the first appearance of it.</p>