title: How to add CSV-format to LWP::Simple
body: <p>Regarding this Target-Url a germany schoolary-database <a href="http://192.68.214.70/km/asps/schulsuche.asp?q=a&amp;a=100&amp;s=1750" rel="nofollow noreferrer">http://192.68.214.70/km/asps/schulsuche.asp?q=a&amp;a=100&amp;s=1750</a></p>
  
  <p>this target url has got a grid- (or a table) - I have a code that runs nicely - but it cannot handle the grid... </p>
  
  <p>it spits out the data like the following ... </p>
  
  <pre><code>lfd. Nr. Schul- nummer Schulname Straße PLZ Ort Telefon Fax Schulart Webseite
  
  1 0401 Mädchenrealschule Marienburg, Abenberg, der Diözese Eichstätt Marienburg 1 91183  Abenberg   09178/509210  Realschulen  mrs-marienburg.homepage.t-online.de 
  2 6581 Volksschule Abenberg (Grundschule) Güssübelstr. 2 91183  Abenberg   09178/215 09178/905060 Volksschulen  home.t-online.de/home/vs-abenberg 
  3 6913 Mittelschule Abenberg  Güssübelstr. 2 91183  Abenberg   09178/215 09178/905060 Volksschulen  home.t-online.de/home/vs-abenberg 
  4 0402 Johann-Turmair-Realschule Staatliche Realschule Abensberg Stadionstraße 46 93326  Abensberg   09443/9143-0,12,13 09443/914330 Realschulen  www.rs-abensberg.de 
  </code></pre>
  
  <p>That is bad - I need to have a separator - how can I  get some separators into the results - either 
  commas or tabs.... ?</p>
  
  <p>Here the code: </p>
  
  <pre><code>  #!/usr/bin/perl
      use strict;
      use warnings;
      use HTML::TableExtract;
      use LWP::Simple;
      use Cwd;
      use POSIX qw(strftime);
      my $te = HTML::TableExtract-&gt;new;
      my $total_records = 0;
      my $suchbegriffe = "e";
      my $treffer = 50;
      my $range = 0;
      my $url_to_process = "http://192.68.214.70/km/asps/schulsuche.asp?q=";
      my $processdir = "processing";
      my $counter = 50;
      my $displaydate = "";
      my $percent = 0;
  
      &amp;workDir();
      chdir $processdir;
      &amp;processURL();
      print "\nPress &lt;enter&gt; to continue\n";
      &lt;&gt;;
      $displaydate = strftime('%Y%m%d%H%M%S', localtime);
      open OUTFILE, "&gt;webdata_for_$suchbegriffe\_$displaydate.txt";
      &amp;processData();
      close OUTFILE;
      print "Finished processing $total_records records...\n";
      print "Processed data saved to $ENV{HOME}/$processdir/webdata_for_$suchbegriffe\_$displaydate.txt\n";
      unlink 'processing.html';
      die "\n";
  
      sub processURL() {
      print "\nProcessing $url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range\n";
      getstore("$url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range", 'tempfile.html') or die 'Unable to get page';
  
         while( &lt;tempfile.html&gt; ) {
            open( FH, "$_" ) or die;
            while( &lt;FH&gt; ) {
               if( $_ =~ /^.*?(Treffer &lt;b&gt;)(d+)( - )(d+)(&lt;/b&gt; w+ w+ &lt;b&gt;)(d+).*/ ) {
                  $total_records = $6;
                  print "Total records to process is $total_records\n";
                  }
               }
               close FH;
         }
         unlink 'tempfile.html';
      }
  
      sub processData() {
         while ( $range &lt;= $total_records) {
            getstore("$url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range", 'processing.html') or die 'Unable to get page';
            $te-&gt;parse_file('processing.html');
            my ($table) = $te-&gt;tables;
            for my $row ( $table-&gt;rows ) {
               cleanup(@$row);
               print OUTFILE "@$row\n";
            }
            $| = 1; 
            print "Processed records $range to $counter";
            print "\r";
            $counter = $counter + 50;
            $range = $range + 50;
            $te = HTML::TableExtract-&gt;new;
         }
      }
  
      sub cleanup() {
         for ( @_ ) {
            s/s+/ /g;
         }
      }
  
      sub workDir() {
      # Use home directory to process data
      chdir or die "$!";
      if ( ! -d $processdir ) {
         mkdir ("$ENV{HOME}/$processdir", 0755) or die "Cannot make directory $processdir: $!";
         }
      }  
  </code></pre>
  
  <p>btw - see one example of the separators... :</p>
  
  <pre><code>my @cols = qw(
      rownum
      number
      name
      phone
      type
      website
  );
  
  my @fields = qw(
      rownum
      number
      name
      street
      postal
      town
      phone
      fax
      type
      website
  );
  </code></pre>
  
  <p>note: this steems from a user  that gave me some great hints on this thread. See 
  <a href="https://stackoverflow.com/questions/5050314/htmltableextract-how-to-run-the-right-argument-see-live-example">HTML::TableExtract: how to run the right argument [see live example]</a></p>
  
  <p>Well - i want to migrate the ideas into the above mentioned spider&amp;parser. Is this possible..Can you give me some hints how to add some lines of code - so that the results are spit out in csv-formate...!?</p>
  
  <p>any and all help will be greatly appreciated..</p>
  
  <p>zero</p>
  
  <p><strong>Update:</strong> according the ideas of muu is too short...: </p>
  
  <pre><code>  #!/usr/bin/perl
      use strict;
      use warnings;
      use HTML::TableExtract;
      use LWP::Simple;
      use Text::CSV
      use Cwd;
      use POSIX qw(strftime);
      my $te = HTML::TableExtract-&gt;new;
      my $total_records = 0;
      my $suchbegriffe = "e";
      my $treffer = 50;
      my $range = 0;
      my $url_to_process = "http://192.68.214.70/km/asps/schulsuche.asp?q=";
      my $processdir = "processing";
      my $counter = 50;
      my $displaydate = "";
      my $percent = 0;
  
      &amp;workDir();
      chdir $processdir;
      &amp;processURL();
      print "\nPress &lt;enter&gt; to continue\n";
      &lt;&gt;;
      $displaydate = strftime('%Y%m%d%H%M%S', localtime);
      open OUTFILE, "&gt;webdata_for_$suchbegriffe\_$displaydate.txt";
      &amp;processData();
      close OUTFILE;
      print "Finished processing $total_records records...\n";
      print "Processed data saved to $ENV{HOME}/$processdir/webdata_for_$suchbegriffe\_$displaydate.txt\n";
      unlink 'processing.html';
      die "\n";
  
      sub processURL() {
      print "\nProcessing $url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range\n";
      getstore("$url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range", 'tempfile.html') or die 'Unable to get page';
  
         while( &lt;tempfile.html&gt; ) {
            open( FH, "$_" ) or die;
            while( &lt;FH&gt; ) {
               if( $_ =~ /^.*?(Treffer &lt;b&gt;)(d+)( - )(d+)(&lt;/b&gt; w+ w+ &lt;b&gt;)(d+).*/ ) {
                  $total_records = $6;
                  print "Total records to process is $total_records\n";
                  }
               }
               close FH;
         }
         unlink 'tempfile.html';
      }
  
      sub processData() {
         while ( $range &lt;= $total_records) {
            getstore("$url_to_process$suchbegriffe&amp;a=$treffer&amp;s=$range", 'processing.html') or die 'Unable to get page';
            $te-&gt;parse_file('processing.html');
            my ($table) = $te-&gt;tables;
            for my $row ( $table-&gt;rows ) {
               cleanup(@$row);
  
              $csv-&gt;combine(@$row);
        print OUTFILE $csv-&gt;string();
  
            }
            $| = 1; 
            print "Processed records $range to $counter";
            print "\r";
            $counter = $counter + 50;
            $range = $range + 50;
            $te = HTML::TableExtract-&gt;new;
         }
      }
  
  
  sub cleanup {
     for ( @_ ) {
        s/\s+/ /g;
     }
  }
      sub workDir() {
      # Use home directory to process data
      chdir or die "$!";
      if ( ! -d $processdir ) {
         mkdir ("$ENV{HOME}/$processdir", 0755) or die "Cannot make directory $processdir: $!";
         }
      }  
  </code></pre>
  
  <p><strong>Update:</strong> I did some trials: </p>
  
  <blockquote>
    <p>martin@suse-linux:~/perl> perl
    perl_bayern_newstack.pl Unmatched ( in
    regex; marked by &lt;-- HERE in
    m/^.*?(Treffer )(d+)( - )(d+)( &lt;--
    HERE </p>
  </blockquote>
  
  <p>what is wrong here ...? </p>