package edu.stanford.nlp.process;

import java.io.File;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.io.Reader;
import java.io.Serializable;
import java.io.StringReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.swing.text.MutableAttributeSet;
import javax.swing.text.html.HTML;
import javax.swing.text.html.HTMLEditorKit;
import javax.swing.text.html.parser.ParserDelegator;

import edu.stanford.nlp.io.IOUtils;
import edu.stanford.nlp.ling.Word;
import edu.stanford.nlp.objectbank.ObjectBank;
import edu.stanford.nlp.trees.international.pennchinese.ChineseUtils;
import edu.stanford.nlp.util.Generics;
import edu.stanford.nlp.util.Pair;
import edu.stanford.nlp.util.StringUtils;
import edu.stanford.nlp.util.logging.Redwood;

import static edu.stanford.nlp.trees.international.pennchinese.ChineseUtils.WHITEPLUS;

/**
 * Convert a Chinese Document into a List of sentence Strings.
 *
 * @author Pi-Chuan Chang
 */
public class ChineseDocumentToSentenceProcessor implements Serializable  {

  /** A logger for this class */
  private static Redwood.RedwoodChannels log = Redwood.channels(ChineseDocumentToSentenceProcessor.class);

  // todo: This class is a mess. We should try to get it out of core

  private static final long serialVersionUID = 4054964767812217460L;

  private static final Set<Character> fullStopsSet = Generics.newHashSet(Arrays.asList(new Character[]{'。', '！', '？', '!', '?'}));
  // not \uff0e . (too often separates English first/last name, etc.)

  private static final Set<Character> rightMarkSet = Generics.newHashSet(Arrays.asList(new Character[]{'”', '’', '》', '』', '〉', '」', '＞', '＇', '）', '\'', '"', ')', ']', '>'}));

  // private final String normalizationTableFile;

  private static final String encoding = "UTF-8";
  private final List<Pair<String,String>> normalizationTable;


  public ChineseDocumentToSentenceProcessor() {
    this(null);
  }

  private static final Pattern PAIR_PATTERN = Pattern.compile("([^\\s]+)\\s+([^\\s]+)");

  /** @param normalizationTableFile A file listing character pairs for
   *     normalization.  Currently the normalization table must be in UTF-8.
   *     If this parameter is {@code null}, the default normalization
   *     of the zero-argument constructor is used.
   */
  public ChineseDocumentToSentenceProcessor(String normalizationTableFile) {
    // this.normalizationTableFile = normalizationTableFile;
    if (normalizationTableFile != null) {
      normalizationTable = new ArrayList<>();
      for (String line : ObjectBank.getLineIterator(new File(normalizationTableFile), encoding)) {
        Matcher pairMatcher = PAIR_PATTERN.matcher(line);
        if (pairMatcher.find()) {
          normalizationTable.add(new Pair<>(pairMatcher.group(1), pairMatcher.group(2)));
        } else {
          log.info("Didn't match: "+line);
        }
      }
    } else {
      normalizationTable = null;
    }
  }
  /*
  public ChineseDocumentToSentenceProcessor(String normalizationTableFile, String encoding) {
    log.info("WARNING: ChineseDocumentToSentenceProcessor ignores normalizationTableFile argument!");
    log.info("WARNING: ChineseDocumentToSentenceProcessor ignores encoding argument!");
    // encoding is never read locally
    this.encoding = encoding;
  }
  */


  /** This should now become disused, and other people should call
   *  ChineseUtils directly!  CDM June 2006.
   */
  public String normalization(String in) {
    //log.info("BEFOR NORM: "+in);
    String norm = ChineseUtils.normalize(in);
    String out = normalize(norm);
    //log.info("AFTER NORM: "+out);
    return out;
  }

  private static final Pattern WHITEPLUS_PATTERN = Pattern.compile(WHITEPLUS);
  private static final Pattern START_WHITEPLUS_PATTERN = Pattern.compile('^' + WHITEPLUS);
  private static final Pattern END_WHITEPLUS_PATTERN = Pattern.compile(WHITEPLUS + '$');

  private String normalize(String inputString) {
    if (normalizationTable == null) {
      return inputString;
    }

    Pattern replacePattern = WHITEPLUS_PATTERN;
    Matcher replaceMatcher = replacePattern.matcher(inputString);
    inputString = replaceMatcher.replaceAll(" ");

    for (Pair<String,String> p : normalizationTable) {
      replacePattern = Pattern.compile(p.first(), Pattern.LITERAL);
      replaceMatcher = replacePattern.matcher(inputString);
      String escape = p.second();
      if (escape.equals("$")) {escape="\\$";}
      inputString = replaceMatcher.replaceAll(escape);
    }
    return inputString;
  }


  /** usage: java ChineseDocumentToSentenceProcessor [-segmentIBM]
   *  -file filename [-encoding encoding]
   *  <p>
   *  The -segmentIBM option is for IBM GALE-specific splitting of an
   *  XML element into sentences.
   */
  public static void main(String[] args) throws Exception {
    //String encoding = "GB18030";
    Properties props = StringUtils.argsToProperties(args);
    // log.info("Here are the properties:");
    // props.list(System.err);
    boolean alwaysAddS = props.containsKey("alwaysAddS");
    ChineseDocumentToSentenceProcessor cp;
    if (! props.containsKey("file")) {
      log.info("usage: java ChineseDocumentToSentenceProcessor [-segmentIBM] -file filename [-encoding encoding]");
      return;
    }
    cp = new ChineseDocumentToSentenceProcessor();
    if (props.containsKey("encoding")) {
      log.info("WARNING: for now the default encoding is "+cp.encoding+". It's not changeable for now");
    }
    String input = IOUtils.slurpFileNoExceptions(props.getProperty("file"),
                                                     cp.encoding);
    // String input = StringUtils.slurpGBURLNoExceptions(new URL(props.getProperty("file")));

    if (props.containsKey("segmentIBM")) {
      Tokenizer<Word> tok = WhitespaceTokenizer.
        newWordWhitespaceTokenizer(new StringReader(input), true);
      String parseInside = props.getProperty("parseInside");
      if (parseInside == null) parseInside = "";

      Pattern p1, p2, p3, p4;
      PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.out, cp.encoding), true);
      StringBuilder buff = new StringBuilder();
      StringBuilder sgmlbuff = new StringBuilder();
      String lastSgml = "";

      p1 = Pattern.compile("<.*>");
      p2 = Pattern.compile("\uFEFF?<[\\p{Alpha}]+");
      p3 = Pattern.compile("[A-Za-z0-9=\"]+>");
      p4 = Pattern.compile("<(?:" + parseInside + ")[ >]");

      boolean inSGML = false;
      int splitItems = 0;
      int numAdded = 0;
      while (tok.hasNext()) {
        String s = tok.next().word();
        // pw.println("The token is |" + s + "|");
        if (p2.matcher(s).matches()) {
          inSGML = true;
          sgmlbuff.append(s).append(" ");
        } else if (p1.matcher(s).matches() || inSGML && p3.matcher(s).matches() || "\n".equals(s)) {
          inSGML = false;
          if (buff.toString().trim().length() > 0) {
            // pw.println("Dumping sentences");
            // pw.println("Buff is " + buff);
            boolean processIt = false;
            if (parseInside.equals("")) {
              processIt = true;
            } else if (p4.matcher(lastSgml).find()) {
              processIt = true;
            }
            if (processIt) {
              List<String> sents = ChineseDocumentToSentenceProcessor.fromPlainText(buff.toString(), true);
              // pw.println("Sents is " + sents);
              // pw.println();
              if (alwaysAddS || sents.size() > 1) {
                int i = 1;
                for (String str : sents) {
                  pw.print("<s id=\"" + i + "\">");
                  pw.print(str);
                  pw.println("</s>");
                  i++;
                }
                if (sents.size() > 1) {
                  splitItems++;
                  numAdded += sents.size() - 1;
                }
              } else if (sents.size() == 1) {
                pw.print(sents.get(0));
              }
            } else {
              pw.print(buff);
            }
            buff = new StringBuilder();
          }
          sgmlbuff.append(s);
          // pw.println("sgmlbuff is " + sgmlbuff);
          pw.print(sgmlbuff);
          lastSgml = sgmlbuff.toString();
          sgmlbuff = new StringBuilder();
        } else {
          if (inSGML) {
            sgmlbuff.append(s).append(" ");
          } else {
            buff.append(s).append(" ");
          }
          // pw.println("Buff is now |" + buff + "|");
        }
      } // end while (tok.hasNext()) {
      // empty remaining buffers
      pw.flush();
      pw.close();
      log.info("Split " + splitItems + " segments, adding " +
                         numAdded + " sentences.");
    } else {
      List<String> sent = cp.fromHTML(input);
      PrintWriter pw = new PrintWriter(new OutputStreamWriter(System.err, cp.encoding), true);

      for (String a : sent) {
        pw.println(a);
      }
    }
  }


  /**
   * Strip off HTML tags before processing.
   * Only the simplest tag stripping is implemented.
   *
   * @param inputString Chinese document text which contains HTML tags
   * @return a List of sentence strings
   */
  public static List<String> fromHTML(String inputString) throws IOException {
    //HTMLParser parser = new HTMLParser();
    //return fromPlainText(parser.parse(inputString));
    List<String> ans = new ArrayList<>();
    MyHTMLParser parser = new MyHTMLParser();
    List<String> sents = parser.parse(inputString);
    for (String s : sents) {
      ans.addAll(fromPlainText(s));
    }
    return ans;
  }


  /**
   * @param contentString Chinese document text
   * @return a List of sentence strings
   * @throws IOException
   */
  public static List<String> fromPlainText(String contentString) throws IOException {
    return fromPlainText(contentString, false);
  }

  public static List<String> fromPlainText(String contentString, boolean segmented) throws IOException {
    if (segmented) {
      contentString = ChineseUtils.normalize(contentString,
                                             ChineseUtils.LEAVE,
                                             ChineseUtils.ASCII);
    } else {
      contentString = ChineseUtils.normalize(contentString,
                                             ChineseUtils.FULLWIDTH,
                                             ChineseUtils.ASCII);
    }

    String sentenceString = "";

    char[] content = contentString.toCharArray();
    boolean sentenceEnd = false;
    List<String> sentenceList = new ArrayList<>();

    int lastCh = -1;
    for (Character c : content) {
      // EncodingPrintWriter.out.println("Char is |" + c + "|", "UTF-8");
      String newChar = c.toString();

      if ( ! sentenceEnd) {
        if (segmented && fullStopsSet.contains(c) &&
            (lastCh == -1 || Character.isSpaceChar(lastCh))) {
          // require it to be a standalone punctuation mark -- cf. URLs
          sentenceString += newChar;
          sentenceEnd = true;
        } else if ( ! segmented && fullStopsSet.contains(c)) {
          // EncodingPrintWriter.out.println("  End of sent char", "UTF-8");
          sentenceString += newChar;
          sentenceEnd = true;
        } else {
          sentenceString += newChar;
        }
      } else { // sentenceEnd == true
        if (rightMarkSet.contains(c)) {
          sentenceString += newChar;
          // EncodingPrintWriter.out.println("  Right mark char", "UTF-8");
        } else if (newChar.matches("\\s")) {
          sentenceString += newChar;
        } else if (fullStopsSet.contains(c)) {
          // EncodingPrintWriter.out.println("  End of sent char (2+)", "UTF-8");
          sentenceString += newChar;
        } else { // otherwise
          if (sentenceString.length() > 0) {
            sentenceEnd = false;
          }
          sentenceString = removeWhitespace(sentenceString, segmented);
          if (sentenceString.length() > 0) {
            //log.info("<<< "+sentenceString+" >>>");
            sentenceList.add(sentenceString);
          }
          sentenceString = "";
          sentenceString += newChar;
        }
      }
      lastCh = c.charValue();
    } // end for (Character c : content)

    sentenceString = removeWhitespace(sentenceString, segmented);
    if (sentenceString.length() > 0) {
      //log.info("<<< "+sentenceString+" >>>");
      sentenceList.add(sentenceString);
    }
    return sentenceList;
  }

  /** In non-segmented mode, all whitespace is removed,
   *  in segmented mode only leading and trailing whitespace goes away.
   *
   */
  private static String removeWhitespace(String str, boolean segmented) {
    if (str.length() > 0) {
      //System.out.println("Add: "+sentenceString);
      Pattern replacePattern = START_WHITEPLUS_PATTERN;
      Matcher replaceMatcher = replacePattern.matcher(str);
      str = replaceMatcher.replaceAll("");
      replacePattern = END_WHITEPLUS_PATTERN;
      replaceMatcher = replacePattern.matcher(str);
      str = replaceMatcher.replaceAll("");

      if ( ! segmented) {
        replacePattern = WHITEPLUS_PATTERN;
        replaceMatcher = replacePattern.matcher(str);
        str = replaceMatcher.replaceAll("");
      }
    }
    return str;
  }



  static class MyHTMLParser extends HTMLEditorKit.ParserCallback {

    protected StringBuilder textBuffer;
    protected List<String> sentences;
    protected String title;
    protected boolean isTitle;
    protected boolean isBody;
    protected boolean isScript;
    protected boolean isBreak;

    public MyHTMLParser() {
      super();
      title = "";
      isTitle = false;
      isBody = false;
      isScript = false;
      isBreak = false;
    }


    @Override
    public void handleText(char[] data, int pos) {
      if (data.length == 0) return;

      if (isTitle) {
        title = new String(data);
      } else if (isBody && !isScript) {
        //textBuffer.append(data).append(" ");
      }
      //if (isBreak) {
      if (true) {
        textBuffer.append(data);
        String text = textBuffer.toString();
        text = text.replaceAll("\u00a0","");
        text = text.trim();
        if (text.length()==0) return;


        sentences.add(text);
        textBuffer = new StringBuilder(500);
      }
    }

    /**
     * Sets a flag if the start tag is the "TITLE" element start tag.
     */
    @Override
    public void handleStartTag(HTML.Tag tag, MutableAttributeSet attrSet, int pos) {
      if (tag == HTML.Tag.TITLE) {
        isTitle = true;
      } else if (tag == HTML.Tag.BODY) {
        isBody = true;
      } else if (tag == HTML.Tag.SCRIPT) {
        isScript = true;
      }

      isBreak = tag.breaksFlow();
    }

    /**
     * Sets a flag if the end tag is the "TITLE" element end tag
     */
    @Override
    public void handleEndTag(HTML.Tag tag, int pos) {
      if (tag == HTML.Tag.TITLE) {
        isTitle = false;
      } else if (tag == HTML.Tag.BODY) {
        isBody = false;
      } else if (tag == HTML.Tag.SCRIPT) {
        isScript = false;
      }
    }

    public List<String> parse(URL url) throws IOException {
      return (parse(IOUtils.slurpURL(url)));
    }

    public List<String> parse(Reader r) throws IOException {
      return parse(IOUtils.slurpReader(r));
    }

    /**
     * The parse method that actually does the work.
     * Now it first gets rid of singleton tags before running.
     * @throws IOException
     */
    public List<String> parse(String text) throws IOException {
      text = text.replaceAll("/>", ">");
      text = text.replaceAll("<\\?","<");
      StringReader r = new StringReader(text);
      textBuffer = new StringBuilder(200);
      sentences = new ArrayList<>();
      new ParserDelegator().parse(r, this, true);
      return sentences;
    }

    public String title() {
      return title;
    }

    /*
    public static void main(String[] args) throws IOException {
      MyHTMLParser parser = new MyHTMLParser();
      String input = StringUtils.slurpGBURLNoExceptions(new URL(args[0]));
      List<String> result = parser.parse(input);
      PrintWriter orig = new PrintWriter("file.orig");
      PrintWriter parsed = new PrintWriter("file.parsed");
      log.info("output to file.orig");
      orig.println(input);
      for (String s : result) {
        log.info("output to file.parsed");
        parsed.println(s);
        parsed.println("-----------------------------------------");
      }
      orig.close();
      parsed.close();
    }
    */

  }


} // end class ChineseDocumentToSentenceProcessor
