package org.apache.nutch.indexer.more;

import java.text.ParseException;
import java.util.Collection;
import java.util.Date;
import java.util.HashSet;

import org.apache.avro.util.Utf8;
import org.apache.commons.lang.time.DateUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.indexer.IndexingException;
import org.apache.nutch.indexer.IndexingFilter;
import org.apache.nutch.indexer.NutchDocument;
import org.apache.nutch.metadata.HttpHeaders;
import org.apache.nutch.net.protocols.HttpDateFormat;
import org.apache.nutch.storage.WebPage;
import org.apache.nutch.storage.WebPage.Field;
import org.apache.nutch.util.MimeUtil;
import org.apache.oro.text.regex.MalformedPatternException;
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.PatternMatcher;
import org.apache.oro.text.regex.Perl5Compiler;
import org.apache.oro.text.regex.Perl5Matcher;
import org.apache.oro.text.regex.Perl5Pattern;
import org.apache.solr.common.util.DateUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Add (or reset) a few metaData properties as respective fields (if they are
 * available), so that they can be accurately used within the search index.
 * 
 * 'lastModifed' is indexed to support query by date, 'contentLength' obtains
 * content length from the HTTP header, 'type' field is indexed to support query
 * by type and finally the 'title' field is an attempt to reset the title if a
 * content-disposition hint exists. The logic is that such a presence is
 * indicative that the content provider wants the filename therein to be used as
 * the title.
 * 
 * Still need to make content-length searchable!
 * 
 * @author John Xing
 */

public class MoreIndexingFilter implements IndexingFilter {
  public static final Logger LOG = LoggerFactory
      .getLogger(MoreIndexingFilter.class);

  /** Get the MimeTypes resolver instance. */
  private MimeUtil MIME;

  private static Collection<WebPage.Field> FIELDS = new HashSet<WebPage.Field>();

  static {
    FIELDS.add(WebPage.Field.HEADERS);
    FIELDS.add(WebPage.Field.CONTENT_TYPE);
    FIELDS.add(WebPage.Field.MODIFIED_TIME);
  }

  @Override
  public NutchDocument filter(NutchDocument doc, String url, WebPage page)
      throws IndexingException {
    addTime(doc, page, url);
    addLength(doc, page, url);
    addType(doc, page, url);
    resetTitle(doc, page, url);
    return doc;
  }

  // Add time related meta info. Add last-modified if present. Index date as
  // last-modified, or, if that's not present, use fetch time.
  private NutchDocument addTime(NutchDocument doc, WebPage page, String url) {
    long time = -1;
    CharSequence lastModified = page.getHeaders().get(
        new Utf8(HttpHeaders.LAST_MODIFIED));
    // String lastModified = data.getMeta(Metadata.LAST_MODIFIED);
    if (lastModified != null) { // try parse last-modified
      time = getTime(lastModified.toString(), url); // use as time
      String formlastModified = DateUtil.getThreadLocalDateFormat().format(
          new Date(time));
      // store as string
      doc.add("lastModified", formlastModified);
    }

    if (time == -1) { // if no last-modified
      time = page.getModifiedTime(); // use Modified time
    }

    String dateString = DateUtil.getThreadLocalDateFormat().format(
        new Date(time));

    // un-stored, indexed and un-tokenized
    doc.add("date", dateString);

    return doc;
  }

  private long getTime(String date, String url) {
    long time = -1;
    try {
      time = HttpDateFormat.toLong(date);
    } catch (ParseException e) {
      // try to parse it as date in alternative format
      try {
        Date parsedDate = DateUtils.parseDate(date,
            new String[] { "EEE MMM dd HH:mm:ss yyyy",
                "EEE MMM dd HH:mm:ss yyyy zzz", "EEE MMM dd HH:mm:ss zzz yyyy",
                "EEE, dd MMM yyyy HH:mm:ss zzz",
                "EEE,dd MMM yyyy HH:mm:ss zzz", "EEE, dd MMM yyyy HH:mm:sszzz",
                "EEE, dd MMM yyyy HH:mm:ss", "EEE, dd-MMM-yy HH:mm:ss zzz",
                "yyyy/MM/dd HH:mm:ss.SSS zzz", "yyyy/MM/dd HH:mm:ss.SSS",
                "yyyy/MM/dd HH:mm:ss zzz", "yyyy/MM/dd", "yyyy.MM.dd HH:mm:ss",
                "yyyy-MM-dd HH:mm", "MMM dd yyyy HH:mm:ss. zzz",
                "MMM dd yyyy HH:mm:ss zzz", "dd.MM.yyyy HH:mm:ss zzz",
                "dd MM yyyy HH:mm:ss zzz", "dd.MM.yyyy; HH:mm:ss",
                "dd.MM.yyyy HH:mm:ss", "dd.MM.yyyy zzz",
                "yyyy-MM-dd'T'HH:mm:ss'Z'" });
        time = parsedDate.getTime();
        // if (LOG.isWarnEnabled()) {
        // LOG.warn(url + ": parsed date: " + date +" to:"+time);
        // }
      } catch (Exception e2) {
        if (LOG.isWarnEnabled()) {
          LOG.warn(url + ": can't parse erroneous date: " + date);
        }
      }
    }
    return time;
  }

  // Add Content-Length
  private NutchDocument addLength(NutchDocument doc, WebPage page, String url) {
    CharSequence contentLength = page.getHeaders().get(
        new Utf8(HttpHeaders.CONTENT_LENGTH));
    if (contentLength != null) {
      // NUTCH-1010 ContentLength not trimmed
      String trimmed = contentLength.toString().trim();
      if (!trimmed.isEmpty())
        doc.add("contentLength", trimmed);
    }

    return doc;
  }

  /**
   * <p>
   * Add Content-Type and its primaryType and subType add contentType,
   * primaryType and subType to field "type" as un-stored, indexed and
   * un-tokenized, so that search results can be confined by contentType or its
   * primaryType or its subType.
   * </p>
   * <p>
   * For example, if contentType is application/vnd.ms-powerpoint, search can be
   * done with one of the following qualifiers
   * type:application/vnd.ms-powerpoint type:application type:vnd.ms-powerpoint
   * all case insensitive. The query filter is implemented in
   * {@link TypeQueryFilter}.
   * </p>
   * 
   * @param doc
   * @param data
   * @param url
   * @return
   */
  private NutchDocument addType(NutchDocument doc, WebPage page, String url) {
    String mimeType = null;
    CharSequence contentType = page.getContentType();
    if (contentType == null)
      contentType = page.getHeaders().get(new Utf8(HttpHeaders.CONTENT_TYPE));
    if (contentType == null) {
      // Note by Jerome Charron on 20050415:
      // Content Type not solved by a previous plugin
      // Or unable to solve it... Trying to find it
      // Should be better to use the doc content too
      // (using MimeTypes.getMimeType(byte[], String), but I don't know
      // which field it is?
      // if (MAGIC) {
      // contentType = MIME.getMimeType(url, content);
      // } else {
      // contentType = MIME.getMimeType(url);
      // }
      mimeType = MIME.getMimeType(url);
    } else {
      mimeType = MIME.forName(MimeUtil.cleanMimeType(contentType.toString()));
    }

    // Checks if we solved the content-type.
    if (mimeType == null) {
      return doc;
    }

    doc.add("type", mimeType);

    // Check if we need to split the content type in sub parts
    if (conf.getBoolean("moreIndexingFilter.indexMimeTypeParts", true)) {
      String[] parts = getParts(mimeType);

      for (String part : parts) {
        doc.add("type", part);
      }
    }

    // leave this for future improvement
    // MimeTypeParameterList parameterList = mimeType.getParameters()

    return doc;
  }

  /**
   * Utility method for splitting mime type into type and subtype.
   * 
   * @param mimeType
   * @return
   */
  static String[] getParts(String mimeType) {
    return mimeType.split("/");
  }

  // Reset title if we see non-standard HTTP header "Content-Disposition".
  // It's a good indication that content provider wants filename therein
  // be used as the title of this url.

  // Patterns used to extract filename from possible non-standard
  // HTTP header "Content-Disposition". Typically it looks like:
  // Content-Disposition: inline; filename="foo.ppt"
  private PatternMatcher matcher = new Perl5Matcher();

  private Configuration conf;
  static Perl5Pattern patterns[] = { null, null };
  static {
    Perl5Compiler compiler = new Perl5Compiler();
    try {
      // order here is important
      patterns[0] = (Perl5Pattern) compiler
          .compile("\\bfilename=['\"](.+)['\"]");
      patterns[1] = (Perl5Pattern) compiler.compile("\\bfilename=(\\S+)\\b");
    } catch (MalformedPatternException e) {
      // just ignore
    }
  }

  private NutchDocument resetTitle(NutchDocument doc, WebPage page, String url) {
    CharSequence contentDisposition = page.getHeaders().get(
        new Utf8(HttpHeaders.CONTENT_DISPOSITION));
    if (contentDisposition == null)
      return doc;

    MatchResult result;
    for (int i = 0; i < patterns.length; i++) {
      if (matcher.contains(contentDisposition.toString(), patterns[i])) {
        result = matcher.getMatch();
        doc.removeField("title");
        doc.add("title", result.group(1));
        break;
      }
    }

    return doc;
  }

  public void addIndexBackendOptions(Configuration conf) {
  }

  public void setConf(Configuration conf) {
    this.conf = conf;
    MIME = new MimeUtil(conf);
  }

  public Configuration getConf() {
    return this.conf;
  }

  @Override
  public Collection<Field> getFields() {
    return FIELDS;
  }

}
