title: Error String index out of range when crawling
body: <p>I keep getting an error with my program after it craws the first 2 URL's "Exception in thread "AWT-EventQueue-0" java.lang.StringIndexOutOfBoundsException: String index out of range: 0". The first couple of URL's craw as I want them to and I get the text from them using a method in another class. The other class could be the problem I don't know. Please have a look at my code and see whats happening. </p>
  
  <pre><code>package WebCrawler;
  
  import java.util.Scanner;
  import java.util.ArrayList;
  
  import static TextAnalyser.Textanalyser.analyse;
  
  public class Crawler {
  
      public static void main(String[] args) {
          //   java.util.Scanner input = new java.util.Scanner(System.in);
          //  System.out.print("Enter a URL: ");
          //  String url = input.nextLine();
          crawler("http://www.port.ac.uk/"); // Traverse the Web from the a starting url 
      }
  
      public static void crawler(String startingURL) {
          ArrayList&lt;String&gt; listOfPendingURLs = new ArrayList&lt;String&gt;();
          ArrayList&lt;String&gt; listOfTraversedURLs = new ArrayList&lt;String&gt;();
  
          listOfPendingURLs.add(startingURL);
          while (!listOfPendingURLs.isEmpty() &amp;&amp; listOfTraversedURLs.size() &lt;= 100) {
              String urlString = listOfPendingURLs.remove(0);
  
              if (!listOfTraversedURLs.contains(urlString)) {
                  listOfTraversedURLs.add(urlString);
                  String text = urlString;
                  text = ReadTextfromURL.gettext(text);
                  text = analyse(text);
                  System.out.println("text : " + text);
                  System.out.println("Craw " + urlString);
  
                  for (String s: getSubURLs(urlString)) {
                      if (!listOfTraversedURLs.contains(s)) {
                          listOfPendingURLs.add(s);
                      }
                  }
              }
          }
      }
  
      public static ArrayList&lt;String&gt; getSubURLs(String urlString) {
          ArrayList &lt;String&gt; list = new ArrayList&lt;String&gt;();
  
          try {
              java.net.URL url = new java.net.URL(urlString);
              Scanner input = new Scanner(url.openStream());
              int current = 0;
              while (input.hasNext()) {
                  String line = input.nextLine();
                  current = line.indexOf("http:", current);
                  while (current &gt; 0) {
                      int endIndex = line.indexOf("\"", current);
                      if (endIndex &gt; 0) { // Ensure that a correct URL is found 
                          list.add(line.substring(current, endIndex));
                          current = line.indexOf("http:", endIndex);
                      } else {
                          current = -1;
                      }
                  }
              }
          } catch (Exception ex) {
              System.out.println("Error: " + ex.getMessage());
          }
  
          return list;
      }
  }
  </code></pre>