CR Stack Exchange crawler I am writing a program which automatically crawls codes from this site!

Would you please review my code?

The required .jars: jsoup, org.apache.commons.io.

Main.java:

public class Main { 
    public static void main(String[] args) throws Exception { 
        final String site1 = "http://codereview.stackexchange.com/questions/"; 
        String site2[] = {"69", "109", "131"}; 
        String site3[] = {"is-this-implementation-of-shamos-hoey-algorithm-ok",
                "are-there-any-ways-to-improve-my-http-request-path-parser",
                "law-of-demeter-and-data-models"}; 
        HtmlGetter htmlGetter = new HtmlGetter();

        for(int i = 0 ; i &lt; site2.length ; ++i) {
            htmlGetter.setUrl(site1, site2[i], site3[i]);
            htmlGetter.download(); 
            htmlGetter.parse(i+1);
        } 
    } 
} 


HtmlGetter.java:

import java.io.*;
import java.net.URL; 
import java.util.Scanner; 

import org.apache.commons.lang3.StringEscapeUtils;  
import org.apache.commons.io.IOUtils; 
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

public class HtmlGetter { 
    private URL url;
    private String fileName;

    public void setUrl(String site1, String site2, String site3) throws Exception {
        this.url = new URL(site1 + site2 + "/" + site3);
        this.fileName = "[" + site2 + "]" + site3 + ".html";
    }

    public void download() throws Exception { 
        final InputStream in = url.openStream(); 
        final OutputStream out = new FileOutputStream(fileName); 
        IOUtils.copy(in, out); 
        in.close(); 
        out.close();
        divideFile();
    }

    private void divideFile() throws Exception {
        Scanner scanner = new Scanner(new File(fileName)); 
        String oneLine; 
        final BufferedWriter out1 = new BufferedWriter(new FileWriter("question.html"));
        final BufferedWriter out2 = new BufferedWriter(new FileWriter("answer.html"));

        while(scanner.hasNextLine()) { 
            oneLine = scanner.nextLine(); 
            if(oneLine.matches(".*&lt;div class=\"post-text\" itemprop=\"description\"&gt;.*")) { 
                while(scanner.hasNextLine()&amp;&amp;(!oneLine.matches(".*&lt;div class=\"post-menu\"&gt;.*"))) { 
                    out1.write(oneLine);
                    out1.newLine();
                    oneLine = scanner.nextLine(); 
                } 
            }
            if(oneLine.matches("&lt;td class=\"answercell\"&gt;")) {
                while(scanner.hasNextLine()&amp;&amp;(!oneLine.matches(".*&lt;div class=\"post-menu\"&gt;.*"))) {
                    out2.write(oneLine);
                    out2.newLine();
                    oneLine = scanner.nextLine();
                }
                break;
            }
        }
        out1.close();
        out2.close();
     }

    public void parse(int articleNumber) throws Exception {
        parse2("question.html", articleNumber);
        parse2("answer.html", articleNumber);
    }

    public void parse2(String strFileName, int articleNumber) throws Exception {
        File input = new File(strFileName);
        Document doc = Jsoup.parse(input, "UTF-8");
        Elements codes = doc.select("code");
        BufferedWriter out;
        int subNumber = 0;
        String qa = "x";

        for(Element code: codes) 
        {
            if(strFileName.equals("question.html")) { qa = "q"; }
            else if(strFileName.equals("answer.html")) { qa = "a"; }
            else { System.out.println("Error!"); }
            out = new BufferedWriter(new FileWriter(qa + articleNumber + "-" + (++subNumber) + ".txt"));
            out.write(code.text().replaceAll("[\r\n]", "\r\n"));
            out.close();
        } 
    }
}
While ZeroOne's answer has the good idea to use an enum, the enum itself is not implemented very well. Java enums are a powerful system, and using a switch statement inside the getFileName() method is very unnecessary. Furthermore, the naming conventions are a bit off. Here's a better implementation of the same enum:

private enum FileType {
    QUESTION("question.html"),
    ANSWER("answer.html");

    private final String fileName;

    FileType(String fileName) {
        this.fileName = fileName;
    }

    public String getFileName() {
        return fileName;
    }
}


This uses the unique capabilities of Java enums to store individual parameters. This also changes the names of the enum and its values. Java enum names should be upper camel-case (same as class names), while enum values should be all caps.