/**
 * boilerpipe
 *
 * Copyright (c) 2009, 2014 Christian Kohlschütter
 *
 * The author licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.kohlschutter.boilerpipe.sax;

import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.net.URLConnection;
import java.nio.charset.Charset;
import java.nio.charset.UnsupportedCharsetException;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;

/**
 * A very simple HTTP/HTML fetcher, really just for demo purposes.
 */
public class HTMLFetcher {
  private HTMLFetcher() {
  }

  private static final Pattern PAT_CHARSET = Pattern.compile("charset=([^; ]+)$");

  /**
   * Fetches the document at the given URL, using {@link URLConnection}.
   * 
   * @param url
   * @return
   * @throws IOException
   */
  public static HTMLDocument fetch(final URL url) throws IOException {
    final URLConnection conn = url.openConnection();
    final String ct = conn.getContentType();

    if (ct == null || !(ct.equals("text/html") || ct.startsWith("text/html;"))) {
      throw new IOException("Unsupported content type: " + ct);
    }

    Charset cs = Charset.forName("Cp1252");
    if (ct != null) {
      Matcher m = PAT_CHARSET.matcher(ct);
      if (m.find()) {
        final String charset = m.group(1);
        try {
          cs = Charset.forName(charset);
        } catch (UnsupportedCharsetException e) {
          // keep default
        }
      }
    }

    InputStream in = conn.getInputStream();

    final String encoding = conn.getContentEncoding();
    if (encoding != null) {
      if ("gzip".equalsIgnoreCase(encoding)) {
        in = new GZIPInputStream(in);
      } else {
        System.err.println("WARN: unsupported Content-Encoding: " + encoding);
      }
    }

    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    byte[] buf = new byte[4096];
    int r;
    while ((r = in.read(buf)) != -1) {
      bos.write(buf, 0, r);
    }
    in.close();

    final byte[] data = bos.toByteArray();

    return new HTMLDocument(data, cs);
  }
}
