package org.allenai.scienceparse;

import com.gs.collections.api.tuple.Pair;
import com.gs.collections.impl.tuple.Tuples;

import lombok.val;
import lombok.extern.slf4j.Slf4j;

import java.io.DataInputStream;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import org.allenai.ml.linalg.DenseVector;
import org.allenai.ml.linalg.Vector;
import org.allenai.ml.sequences.StateSpace;
import org.allenai.ml.sequences.crf.CRFFeatureEncoder;
import org.allenai.ml.sequences.crf.CRFModel;
import org.allenai.ml.sequences.crf.CRFWeightsEncoder;
import org.allenai.ml.util.IOUtils;
import org.allenai.ml.util.Indexer;
import org.nustaq.serialization.FSTObjectInput;
import org.nustaq.serialization.FSTObjectOutput;

@Slf4j
public class ExtractReferences {

  public static final String authUnit = "\\p{Lu}[\\p{L}'`\\-]+";
  public static final String authOneName = authUnit + "(?: " + authUnit + ")?"; //space and repetition for things like De Mori
  public static final String authLastCommaInitial = authOneName + ", (?:\\p{Lu}\\.-? ?)+";
  public static final String authConnect = "(?:; |, |, and |; and | and )";

  public static final String authInitialsLast = "(?:\\p{Lu}\\.?(?:-| )?)+ " + authOneName;
  public static final String authInitialsLastList = authInitialsLast + "(?:" + authConnect + authInitialsLast + ")*";
  public static final String authPlain = authOneName + "(?:\\p{Lu}\\. )?" + authOneName;
  public static final String authPlainList = authPlain + "(?:(?:, and|,) (?:" + authPlain + "))*";

  //pattern for matching single author name, format as in Jones, C. M.
  public static final String authGeneral = "\\p{Lu}[\\p{L}\\.'`\\- ]+";
  public static final String authGeneralList = authGeneral + "(?:(?:; |, |, and |; and | and )" + authGeneral + ")*";

  // patterns for mentions
  private static final String numberOrRangeRe = "(?:[1-9][0-9]*(?: *- *[1-9][0-9]*)?)";
  private static final String separatorRe = "(?: *[,;|] *)";
  private static final String citeCharactersRe = numberOrRangeRe + "(?:" + separatorRe + numberOrRangeRe + ")*";
  public static final Pattern mentions = Pattern.compile(
      "(?:" + citeCharactersRe + ")|" +
      "(?:\\[" + citeCharactersRe + "\\])|" +
      "(?:\\(" + citeCharactersRe + "\\))"
  );

  private ArrayList<BibStractor> extractors = null;

  public static Pattern pBracket = Pattern.compile("\\[([0-9]+)\\](.*)");
  public static Pattern pDot = Pattern.compile("([0-9]+)\\.(.*)");
  
  CheckReferences cr;
  final CRFBibRecordParser bibCRFRecordParser;

  public static final String DATA_VERSION = "0.3-BIB";  // Faster serialization
  
  public ExtractReferences(String gazFile) throws IOException {
    this(new FileInputStream(gazFile));
  }

  public ExtractReferences(String gazFile, String bibModelFile) throws IOException {
    this(new FileInputStream(gazFile), new DataInputStream(new FileInputStream(bibModelFile)));
  }
  
  public ExtractReferences(final InputStream is) throws IOException {
    this(is, null);
  }

  public ExtractReferences(final InputStream is, final DataInputStream bibCRFModel) throws IOException {
    this(is, bibCRFModel, null);
  }

  public ExtractReferences(
      final InputStream is,
      final DataInputStream bibCRFModel,
      final InputStream gazCacheInputStream
  ) throws IOException {
    if(gazCacheInputStream != null) {
      try(final FSTObjectInput in = new FSTObjectInput(gazCacheInputStream)) {
        cr = (CheckReferences)in.readObject();
      } catch(final Exception e) {
        log.warn("Could not load gazetteer from cache. Loading it slowly instead.", e);
      }
    }

    if(cr == null)
      cr = new CheckReferences(is);

    extractors = new ArrayList<>();
    
    if(bibCRFModel != null) {
      final CRFModel<String, String, String> bibCRF = loadModel(bibCRFModel);
      bibCRFRecordParser = new CRFBibRecordParser(bibCRF);
      extractors.addAll(Arrays.asList(
          new NumberCloseParen(new Class [] {
              NumberCloseParenBibRecordParser.class,
              CRFBibRecordParser.class}),
          new BracketNumber(new Class [] {
              BracketNumberInitialsQuotedBibRecordParser.class,
              CRFBibRecordParser.class}),
          new NamedYear(new Class [] {
              NamedYearBibRecordParser.class,
              CRFBibRecordParser.class}),
          new NamedYear(new Class [] {
              NamedYearInParensBibRecordParser.class,
              CRFBibRecordParser.class}),
          new NumberDot(new Class [] {
              NumberDotYearParensBibRecordParser.class,
              CRFBibRecordParser.class}),
          new NumberDot(new Class [] {
              NumberDotAuthorNoTitleBibRecordParser.class,
              CRFBibRecordParser.class}),
          new NumberDot(new Class [] {
              NumberDotYearNoParensBibRecordParser.class,
              CRFBibRecordParser.class}),
          new BracketNumber(new Class [] {
              BracketNumberInitialsYearParensCOMMAS.class,
              CRFBibRecordParser.class}),
          new BracketNumber(new Class [] {
              BracketNumberBibRecordParser.class,
              CRFBibRecordParser.class}),
          new BracketName(new Class [] {
              BracketNameBibRecordParser.class,
              CRFBibRecordParser.class})
      ));
      extractors.addAll(Arrays.asList(
        new BracketNumber(new Class [] {CRFBibRecordParser.class}),
        new NumberDot(new Class [] {CRFBibRecordParser.class}),
        new NumberDotNaturalLineBreaks(new Class [] {CRFBibRecordParser.class}),
        new NamedYear(new Class [] {CRFBibRecordParser.class}),
        new BracketName(new Class [] {CRFBibRecordParser.class})));
    } else {
      bibCRFRecordParser = null;
      extractors.addAll(Arrays.asList(
          new BracketNumber(new Class [] {BracketNumberInitialsQuotedBibRecordParser.class}),
          new NamedYear(new Class [] {NamedYearBibRecordParser.class}),
          new NamedYear(new Class [] {NamedYearInParensBibRecordParser.class}),
          new NumberDot(new Class [] {NumberDotYearParensBibRecordParser.class}),
          new NumberDot(new Class [] {NumberDotAuthorNoTitleBibRecordParser.class}),
          new NumberDot(new Class [] {NumberDotYearNoParensBibRecordParser.class}),
          new BracketNumber(new Class [] {BracketNumberInitialsYearParensCOMMAS.class}),
          new BracketNumber(new Class [] {BracketNumberBibRecordParser.class}),
          new BracketName(new Class [] {BracketNameBibRecordParser.class})));
    }
  }

  public static ExtractReferences createAndWriteGazCache(
      final InputStream is,
      final DataInputStream bibCRFModel,
      final OutputStream gazCacheFileOutputStream
  ) throws IOException {
    val result = new ExtractReferences(is, bibCRFModel);

    final FSTObjectOutput out = new FSTObjectOutput(gazCacheFileOutputStream);
    out.writeObject(result.cr);

    return result;
  }

  public static CRFModel<String, String, String> loadModel(
    DataInputStream dis
  ) throws IOException {
    IOUtils.ensureVersionMatch(dis, ExtractReferences.DATA_VERSION);
    val stateSpace = StateSpace.load(dis);
    Indexer<String> nodeFeatures = Indexer.load(dis);
    Indexer<String> edgeFeatures = Indexer.load(dis);
    Vector weights = DenseVector.of(IOUtils.loadDoubles(dis));

    ParserLMFeatures plf = null;
    GazetteerFeatures gf = null;
    try(FSTObjectInput in = new FSTObjectInput(dis)) {
      try {
        plf = (ParserLMFeatures)in.readObject();
      } catch (final Exception e) {
        // do nothing
        // but now plf is NULL. Is that OK?
      }

      try {
        gf = (GazetteerFeatures)in.readObject();
      } catch (final Exception e) {
        log.info("Failed to load kermit gazetteer with this error:", e);
      }
      if (gf != null)
        log.info("kermit gazetteer successfully loaded.");
      else
        log.info("could not load kermit gazetter");
    }
      
    val predExtractor = new ReferencesPredicateExtractor(plf);
    predExtractor.setGf(gf);
    val featureEncoder =
        new CRFFeatureEncoder<String, String, String>(predExtractor, stateSpace, nodeFeatures, edgeFeatures);
    val weightsEncoder =
        new CRFWeightsEncoder<String>(stateSpace, nodeFeatures.size(), edgeFeatures.size());

    return new CRFModel<String, String, String>(featureEncoder, weightsEncoder, weights);
  }
  
  public static Pattern authStrToPat(String s) {
    if(s == null || s.length() == 0)
      s = "";
    return Pattern.compile(s, Pattern.CASE_INSENSITIVE);
  }

  //returns pattern-ready form of author
  private static String cleanAuthString(String s) {
    return s.replaceAll("\\p{P}", ".");//allow anything for any punctuation
  }

  private static final Pattern yearPattern = Pattern.compile("[1-2][0-9][0-9][0-9]");
  public static int extractRefYear(String sYear) {
    final Matcher mYear = RegexWithTimeout.matcher(yearPattern, sYear);
    int a = 0;
    while (mYear.find()) {
      try {
        a = Integer.parseInt(mYear.group().trim());
      } catch (final NumberFormatException e) {
        // do nothing
      }
      if (a > BibRecord.MINYEAR && a < BibRecord.MAXYEAR)
        return a;
    }
    return a;
  }

  private static final Pattern firstNamesPattern =
      Pattern.compile(
          "\\p{javaUpperCase}{1,3}|" +
          "(\\p{javaUpperCase}\\.){1,3}|" +
          "(\\p{javaUpperCase}\\s+){0,2}(\\p{javaUpperCase})|" +
          "(\\p{javaUpperCase}\\.\\s+){0,2}(\\p{javaUpperCase}\\.)");
  /**
   * Takes in a string mentioning several authors, returns normalized list of authors
   *
   * @param authString
   * @return
   */
  public static List<String> authorStringToList(String authString) {
    authString = authString.trim();

    // remove punctuation at the end, all punctuation but .
    authString = authString.replaceAll("[\\p{Punct}&&[^.]]*$", "");

    // remove punctuation at the beginning
    authString = authString.replaceAll("^\\p{Punct}*", "");

    // remove "et al" at the end
    authString = authString.replaceAll("[\\s\\p{Punct}]*[eE][tT]\\s+[aA][lL].?$", "");

    // remove "etc" at the end
    authString = authString.replaceAll("[\\s\\p{Punct}]*[eE][tT][cC].?$", "");

    // find out the top-level separator of names
    final String mainSplitString;
    if(authString.contains(";"))
      mainSplitString = ";";
    else
      mainSplitString = ",";

    // replace "and" with the top level separator
    authString = authString.replaceAll("\\b[aA][nN][dD]\\b|&", mainSplitString);

    // split into names
    final String[] names = authString.split(mainSplitString);

    // clean up the names
    for(int i = 0; i < names.length; ++i) {
      names[i] = names[i].trim().
        // clean up names that start with punctuation
        replaceAll("^\\p{Punct}\\s+", "");
    }

    // Some look like this: "Divakaran, A., Forlines, C., Lanning, T., Shipman, S., Wittenburg, K."
    // If we split by comma, we need to make sure to glue these back together.
    if(mainSplitString.equals(",")) {
      for(int i = 1; i < names.length; ++i) {
        if(firstNamesPattern.matcher(names[i]).matches()) {
          names[i - 1] = names[i] + " " + names[i - 1];
          names[i] = "";  // We'll clean up empty strings later.
          i += 1;
        }
      }
    }

    // see if we have to reorder first and last names
    int invertAroundCommaCount = 0;
    int invertAroundSpaceCount = 0;
    int doNothingCount = 0;
    for(final String name : names) {
      if(name.isEmpty())
        continue;
      if(name.contains(",")) {
        invertAroundCommaCount += 1;
      } else {
        final String[] individualNames = name.split("\\s+");
        // If the last individual name looks like not-a-last-name, we assume we have to invert.
        if(firstNamesPattern.matcher(individualNames[individualNames.length - 1]).matches())
          invertAroundSpaceCount += 1;
        else
          doNothingCount += 1;
      }
    }

    // invert, if we have to
    if(invertAroundCommaCount > invertAroundSpaceCount && invertAroundCommaCount > doNothingCount) {
      // invert around comma
      for(int i = 0; i < names.length; ++i) {
        final String[] parts = names[i].split("\\s*,\\s*");
        if(parts.length == 2)
          names[i] = parts[1] + " " + parts[0];
      }
    } else if(invertAroundSpaceCount > invertAroundCommaCount && invertAroundSpaceCount > doNothingCount) {
      // invert around space, i.e., make Johnson M into M Johnson
      final StringBuilder b = new StringBuilder(128);
      for(int i = 0; i < names.length; ++i) {
        final String[] parts = names[i].split("\\s+");
        b.append(parts[parts.length - 1]);
        b.append(' ');
        for(int j = 0; j < parts.length - 1; ++j) {
          b.append(parts[j]);
          b.append(' ');
        }
        names[i] = b.toString().trim();
        b.setLength(0);
      }
    }

    // strip out empty strings
    return Arrays.asList(names).stream().filter(s -> !s.isEmpty()).collect(Collectors.toList());
  }

  private static <T> List<T> removeNulls(List<T> in) {
    return in.stream().filter(a -> (a != null)).collect(Collectors.toList());
  }

  private static String getAuthorLastName(String authName) {
    int idx = authName.lastIndexOf(" ");
    return authName.substring(idx + 1);
  }

  private static int refStart(List<String> paper) {
    for (int i = paper.size() / 3; i < paper.size(); i++) { //heuristic, assume refs start at least 1/3 into doc
      String s = paper.get(i);
      if (s.endsWith("References") || s.endsWith("Citations") || s.endsWith("Bibliography") || s.endsWith("Bibliographie") ||
        s.endsWith("REFERENCES") || s.endsWith("CITATIONS") || s.endsWith("BIBLIOGRAPHY") || s.endsWith("BIBLIOGRAPHIE"))
        return i;
      else if (s.contains("References<lb>") || s.contains("Citations<lb>") || s.contains("Bibliography<lb>") || s.contains("Bibliographie<lb>") ||
        s.contains("REFERENCES<lb>") || s.contains("CITATIONS<lb>") || s.contains("BIBLIOGRAPHY<lb>") || s.contains("BIBLIOGRAPHIE<lb>")) {
        return i - 1;
      }
    }
    return -1;
  }

  public static int getIdxOf(List<BibRecord> bib, String citeStr) {
    //note: slow
    for (int i = 0; i < bib.size(); i++) {
      if (bib.get(i).citeRegEx.matcher(citeStr).matches())
        return i;
    }
    return -1;
  }

  public static Pair<Integer, Integer> shortCiteSearch(int yearPos, int year, String s, List<BibRecord> bib) {
	  int start = -1;
	  int idx = -1;
	  int ct = 0;
	  for(BibRecord br : bib) {
		 Matcher m = RegexWithTimeout.matcher(br.shortCiteRegEx, s);
		 if(m.find()) {
			 //TODO: handle multiple matches
			 if(m.start() > yearPos)
				 continue;
			 start = m.start();
			 idx = ct;
			 break;
		 }
		 ct++;
	  }
	  return Tuples.pair(start, idx);
  }
  
  //note, also replaces <lb> with spaces in lines with found references
  public static List<CitationRecord> findCitations(List<String> paper, List<BibRecord> bib, BibStractor bs) {
    ArrayList<CitationRecord> out = new ArrayList<>();
    Pattern p = Pattern.compile(bs.getCiteRegex());
    Pattern pRange = Pattern.compile("([1-9][0-9]*)\\p{Pd}([1-9][0-9]*)");
    
    int stop = refStart(paper); //stop at start of refs
    if (stop < 0)
      stop = paper.size(); //start of refs not found (should never happen for non-null bibliography...)

    for (int i = 0; i < stop; i++) {
      String s = paper.get(i).replaceAll("-<lb>", "").replaceAll("<lb>", " ");
      paper.set(i, s);
      Matcher m = RegexWithTimeout.matcher(p, s);
      while (m.find()) {
        String citationMeat = m.group(1);
        if(citationMeat == null)
          citationMeat = m.group(2);
        String[] citations = citationMeat.split(bs.getCiteDelimiter());
        for (final String citation : citations) {
          Matcher mRange = RegexWithTimeout.matcher(pRange, citation);
          if(mRange.matches()) { //special case for ranges
              int st = Integer.parseInt(mRange.group(1));
              int end = Integer.parseInt(mRange.group(2));
              for(int j=st;j<=end;j++) {
                if(Thread.interrupted())
                  throw new Parser.ParsingTimeout();
                int idx = getIdxOf(bib, j + "");
                if (idx >= 0) {
                  out.add(new CitationRecord(idx, paper.get(i), m.start(), m.end()));
                }
              }
          }
          else {
            int idx = getIdxOf(bib, citation.trim());
            if (idx >= 0) {
              int start = m.start();
              // Some citeRegexes depend on context, specifically, they only work at the end of a
              // sentence. That means they have to include the preceding period in the citeRegex,
              // which makes m.start() point to the period. To avoid this, we go to the next
              // character.
              if(paper.get(i).charAt(m.start()) == '.')
                start += 1;
              out.add(new CitationRecord(idx, paper.get(i), start, m.end()));
            }
          }
        }
      }
      //short-cites are assumed to be e.g.: Etzioni et al. (2005)
      if(bs.getShortCiteRegex() != null) {
    	  Pattern p2 = Pattern.compile(bs.getShortCiteRegex());
    	  Matcher m2 = RegexWithTimeout.matcher(p2, s);
    	  while(m2.find()) {
    		  Pair<Integer, Integer> shct =
            shortCiteSearch(m2.start(), Integer.parseInt(m2.group(1).substring(0, 4)), s, bib);
    		  int start = shct.getOne();
    		  int idx = shct.getTwo();
    		  if(start > 0) {
    			  out.add(new CitationRecord(idx, paper.get(i), start, m2.end()+1));
    		  }
    	  }
      }
    }
    return out;
  }

  public int numFound(List<BibRecord> brs) {
    int i = 0;
    for (BibRecord br : brs) {
      if (cr.hasPaper(br.title, br.author, br.year, br.venue))
        i++;
    }
    return i;
  }

  public int longestIdx(List<BibRecord>[] results) {
    int maxLen = -1;
    int idx = -1;
    for (int i = 0; i < results.length; i++) {
      int f = 10000 * numFound(results[i]) + results[i].size(); //order by num found, then by size
      if (f > maxLen) {
        idx = i;
        maxLen = f;
      }
    }
//    log.info("chose " + idx + " with " + maxLen);
    return idx;
  }

  //heuristic
  public boolean refEnd(String s) {
    if (s.endsWith("Appendix") || s.endsWith("APPENDIX"))
        return true;
    else
      return false;
  }
  
  private static List<BibRecord> clean(final List<BibRecord> brs) {
    val result = new ArrayList<BibRecord>(brs.size());

    // remove punctuation at the beginning and end of the title
    for(final BibRecord b : brs) {
      final String newTitle =
          b.title.trim().replaceAll("^\\p{P}", "").replaceAll("[\\p{P}&&[^)]]$", "");
      if(
          !newTitle.isEmpty() &&      // delete empty titles
          newTitle.length() < 512 &&  // delete absurdly long bib entries
          (b.venue == null || b.venue.length() < 512) &&
          (b.author == null || b.author.stream().allMatch(a -> a.length() < 512))
      ) {
        result.add(b.withTitle(newTitle));
      }
    }

    return result;
  }
  
  /**
   * Returns the list of BibRecords, plus the extractor that produced them (in order to enable
   * citation parsing)
   */
  public Pair<List<BibRecord>, BibStractor> findReferences(List<String> paper) {
    int start = refStart(paper) + 1;
    List<BibRecord>[] results = new ArrayList[extractors.size()];
    for (int i = 0; i < results.length; i++)
      results[i] = new ArrayList<>();
    StringBuilder sb = new StringBuilder();
    for (int i = start; i < paper.size(); i++) {
      if(refEnd(paper.get(i)))
        break;
      sb.append("<bb>");
      sb.append(paper.get(i));
    }
    String text = sb.toString();
    for (int i = 0; i < results.length; i++) {
      results[i] = extractors.get(i).parse(text);
      results[i] = clean(results[i]);
    }
    int idx = longestIdx(results);
    //log.info("references: " + results[idx].toString());
    return Tuples.pair(results[idx], extractors.get(idx));
  }

  public interface BibRecordParser {
    BibRecord parseRecord(String line);
  }

  public abstract class BibStractor {
    final BibRecordParser [] recParser;

    BibStractor(Class [] c) {
      BibRecordParser b = null;
      recParser = new BibRecordParser[c.length];
      for(int i=0; i<c.length; i++) {
        if(c[i] == CRFBibRecordParser.class) { //special case
          b = bibCRFRecordParser;
        } else {
          try {
            b = (BibRecordParser) c[i].newInstance();
          } catch (final Exception e) {
            log.warn("Exception while creating BibStractor", e);
          }
        }
        recParser[i] = b;
      }
    }

    BibStractor() {
      recParser = null;
    }
    
    public abstract List<BibRecord> parse(String source);

    public abstract String getCiteRegex();
    
    public String getShortCiteRegex() { return null; } //may return null for bibstractors without short cites

    public abstract String getCiteDelimiter();
  }

  private static class DefaultBibRecordParser implements BibRecordParser {
    public BibRecord parseRecord(String line) {
      return new BibRecord(line, null, null, null, null, 0);
    }
  }
  
  private static class BracketNumberInitialsQuotedBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "\\[([0-9]+)\\] (.*)(?:,|\\.|:) [\\p{Pi}\"\']+(.*),[\\p{Pf}\"\']+ (?:(?:I|i)n )?(.*)\\.?";
    private static final Pattern pattern = Pattern.compile(regEx);

    public BracketNumberInitialsQuotedBibRecordParser() {
    }

    //example:
    //	"[1] E. Chang and A. Zakhor, “Scalable video data placement on parallel disk "
    //+ "arrays," in IS&T/SPIE Int. Symp. Electronic Imaging: Science and Technology, "
    //+ "Volume 2185: Image and Video Databases II, San Jose, CA, Feb. 1994, pp. 208–221."
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(4)));
      } else {
        return null;
      }
    }
  }

  private static class BracketNumberBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "\\[([0-9]+)\\] (" + authInitialsLastList + ")\\. ([^\\.]+)\\. (?:(?:I|i)n )?(.*) ([1-2][0-9]{3})\\.( .*)?";
    private static final Pattern pattern = Pattern.compile(regEx);

    public BracketNumberBibRecordParser() {
    }

    //example:
    //TODO
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(5)));
      } else {
        return null;
      }
    }
  }

  // Example:
  // 6) Farrell GC, Larter CZ. Nonalcoholic fatty liver disease: from steatosis to cirrhosis.
  //    Hepatology 2006;43(Suppl. 1):S99-S112.
  private static class NumberCloseParenBibRecordParser implements BibRecordParser {
    private static final String uppercaseNameUnit = authUnit;
    private static final String anycaseNameUnit = "[\\p{L}'`\\-]+";
    private static final String multipleNameUnits = "(?:" + anycaseNameUnit + " +)*" + uppercaseNameUnit;
    private static final String initialsLastAuthor = multipleNameUnits + " +\\p{Lu}+";
    private static final String multipleInitialsLastAuthors =
        initialsLastAuthor + "(?:" + authConnect + initialsLastAuthor + ")*";

    private static final String regEx = (
        "([0-9]+)\\)" +                       // "6)"
        " +" +
        "(" + multipleInitialsLastAuthors + ")(?:, et al)?\\." + // "Farrell GC, Larter CZ."
        " +" +
        "([^\\.]+[\\.\\?])" +                 // "Nonalcoholic fatty liver disease: from steatosis to cirrhosis."
        " +" +
        "(.*) +([1-2][0-9]{3});" +            // "Hepatology 2006;"
        ".*");                                // "43(Suppl. 1):S99-S112."
    private static final Pattern pattern = Pattern.compile(regEx);

    public NumberCloseParenBibRecordParser() {
    }

    public BibRecord parseRecord(final String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        String title = m.group(3);
        if(title.endsWith("."))
          title = title.substring(0, title.length() - 1);
        return new BibRecord(
            title,
            authorStringToList(m.group(2)),
            m.group(4),
            Pattern.compile(m.group(1)), null,
            extractRefYear(m.group(5)));
      } else {
        return null;
      }
    }
  }

  static class NumberDotAuthorNoTitleBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "([0-9]+)\\. +(" + authLastCommaInitial + "(?:; " + authLastCommaInitial + ")*)" + " ([^0-9]*) ([1-2][0-9]{3})(?:\\.|,[0-9, ]*)(?:.*)";
    private static final Pattern pattern = Pattern.compile(regEx);

    public NumberDotAuthorNoTitleBibRecordParser() {
    }

    //example:
    //1. Jones, C. M.; Henry, E. R.; Hu, Y.; Chan C. K; Luck S. D.; Bhuyan, A.; Roder, H.; Hofrichter, J.;
    //Eaton, W. A. Proc Natl Acad Sci USA 1993, 90, 11860.
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        return new BibRecord(
          "",
          authorStringToList(m.group(2)),
          m.group(3),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(4)));
      } else {
        return null;
      }
    }
  }

  private static class NumberDotYearParensBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "([0-9]+)\\. ([^:]+): ([^\\.]+)\\. (?:(?:I|i)n: )?(.*) \\([^)]*([0-9]{4})\\)\\.?(?: .*)?"; //last part is for header break
    private static final Pattern pattern = Pattern.compile(regEx);

    public NumberDotYearParensBibRecordParser() {
    }

    //example:
    //TODO
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(5)));
      } else {
        return null;
      }
    }
  }

  private static class NumberDotYearNoParensBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "([0-9]+)\\. (" + authInitialsLastList + "). ([^\\.]+)\\. (?:(?:I|i)n: )?(.*) ([1-2][0-9]{3}).( .*)?"; //last part for header break
    private static final Pattern pattern = Pattern.compile(regEx);

    public NumberDotYearNoParensBibRecordParser() {
    }

    //example:
    //TODO
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(5)));
      } else {
        return null;
      }
    }
  }

  private static class NamedYearBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "(" + authGeneralList + ") +([1-2][0-9]{3}[a-z]?)\\. ([^\\.]+)\\. (?:(?:I|i)n )?(.*)\\.?";
    private static final Pattern pattern = Pattern.compile(regEx);

    public NamedYearBibRecordParser() {
    }
    
    //example:
    //STONEBREAKER, M. 1986. A Case for Shared Nothing. Database Engineering 9, 1, 4–9.
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        final List<String> authors = authorStringToList(m.group(1));
        final int year = Integer.parseInt(m.group(2).substring(0, 4));
        final String nameStr = getCiteAuthorFromAuthors(authors);
        final String citeStr = nameStr + ",? " + m.group(2);
        return new BibRecord(m.group(3), authors, m.group(4), authStrToPat(citeStr), authStrToPat(nameStr), year);
      } else {
        return null;
      }
    }

  }

  private static class NamedYearInParensBibRecordParser implements BibRecordParser {
    private static final String regEx =
      "(" + authGeneralList + ") +\\(([1-2][0-9]{3}[a-z]?)\\)\\. ([^\\.]+)\\. (?:(?:I|i)n )?(.*)\\.?";
    private static final Pattern pattern = Pattern.compile(regEx);

    public NamedYearInParensBibRecordParser() {
    }

    //example:
    //STONEBREAKER, M. 1986. A Case for Shared Nothing. Database Engineering 9, 1, 4–9.
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern, line.trim());
      if (m.matches()) {
        List<String> authors = authorStringToList(m.group(1));
        int year = Integer.parseInt(m.group(2).substring(0, 4));
        String nameStr = getCiteAuthorFromAuthors(authors);
        String citeStr = nameStr + ",? " + m.group(2);
        return new BibRecord(m.group(3), authors, m.group(4), authStrToPat(citeStr), authStrToPat(nameStr), year);
      } else {
        return null;
      }
    }
  }

  private static class BracketNumberInitialsYearParensCOMMAS implements BibRecordParser {
    private static final String regEx1 =
      "\\[([0-9]+)\\] (" + authInitialsLastList + ")(?:,|\\.) ([^\\.,]+)(?:,|\\.) (?:(?:I|i)n )?(.*)\\(.*([1-2][0-9]{3}).*\\).*";
    private static final Pattern pattern1 = Pattern.compile(regEx1);
    private static final String regEx2 =
      "\\[([0-9]+)\\] (" + authInitialsLastList + ")(?:,|\\.) ([^\\.,]+)(?:,|\\.) (?:(?:I|i)n )?(.*), ((?:20|19)[0-9]{2})(?:\\.|,).*";
    private static final Pattern pattern2 = Pattern.compile(regEx2);

    public BracketNumberInitialsYearParensCOMMAS() {
    }

    //example:
    // [1] S. Abiteboul, H. Kaplan, and T. Milo, Compact labeling schemes for ancestor queries. Proc. 12th Ann. ACM-SIAM Symp.
    // on Discrete Algorithms (SODA 2001), 547-556.
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern1, line.trim());
      Matcher m2 = RegexWithTimeout.matcher(pattern2, line.trim());
      if (m.matches()) {
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          Pattern.compile(m.group(1)), null,
          extractRefYear(m.group(5)));
      } else if (m2.matches()) {
        return new BibRecord(
          m2.group(3),
          authorStringToList(m2.group(2)),
          m2.group(4),
          Pattern.compile(m2.group(1)), null,
          extractRefYear(m2.group(5)));
      } else {
        return null;
      }
    }
  }


  
  private static class BracketNameBibRecordParser implements BibRecordParser {
    private static final String regEx1 =
      "\\[([^\\]]+)\\] (" + authInitialsLastList + ")(?:,|\\.) ([^\\.,]+)(?:,|\\.) (?:(?:I|i)n )?(.*), ([1-2][0-9]{3})\\.";
    private static final Pattern pattern1 = Pattern.compile(regEx1);
    private static final String regEx2 =
      "\\[([^\\]]+)\\] (" + authGeneralList + ")(?:,|\\.) ([^\\.,]+)(?:,|\\.) (?:(?:I|i)n )?(.*),? ([1-2][0-9]{3})\\..*";
    private static final Pattern pattern2 = Pattern.compile(regEx2);

    public BracketNameBibRecordParser() {
    }

    //example:
    // [Magnini et al., 2002] B. Magnini, M. Negri, R. Prevete, and
    // H. Tanev. Is it the right answer? exploiting web redundancy
    //   for answer validation. In ACL, 2002.
    public BibRecord parseRecord(String line) {
      Matcher m = RegexWithTimeout.matcher(pattern1, line.trim());
      Matcher m2 = RegexWithTimeout.matcher(pattern2, line.trim());
      if (m.matches()) {
        if (m.group(1).matches("[0-9]+")) //don't override BracketNumber
          return null;
        return new BibRecord(
          m.group(3),
          authorStringToList(m.group(2)),
          m.group(4),
          authStrToPat(cleanAuthString(m.group(1))), null,
          extractRefYear(m.group(5)));
      } else if (m2.matches()) {
        if (m2.group(1).matches("[0-9]+")) //don't override BracketNumber
          return null;
        return new BibRecord(
          m2.group(3),
          authorStringToList(m2.group(2)),
          m2.group(4),
          authStrToPat(cleanAuthString(m2.group(1))), null,
          extractRefYear(m2.group(5)));
      } else {
        return null;
      }
    }
  }

  //in regex form
  public static String getCiteAuthorFromAuthors(List<String> authors) {
    if(authors == null || authors.size()==0)
      return null;
    if (authors.size() > 2) {
      return cleanAuthString(getAuthorLastName(authors.get(0))) + " et al\\.";
    } else if (authors.size() == 1) {
      return cleanAuthString(getAuthorLastName(authors.get(0)));
    } else if (authors.size() == 2) {
      return cleanAuthString(getAuthorLastName(authors.get(0))) + " and " + cleanAuthString(getAuthorLastName(authors.get(1)));
    }
    return null;
  }

  public class NamedYear extends BibStractor {
    private final static String citeRegex =
      "(?:\\[|\\()([^\\[\\(\\]\\)]+ [1-2][0-9]{3}[a-z]?)+(?:\\]|\\))";
    private final static String shortCiteRegex = "(?:\\[|\\()([1-2][0-9]{3}[a-z]?)(?:\\]|\\))";
    private final static String citeDelimiter = "; ?";

    NamedYear(Class [] c) {
      super(c);
    }

    public String getCiteRegex() {
      return citeRegex;
    }

    @Override
    public String getShortCiteRegex() {
    	return shortCiteRegex;
    }
    
    public String getCiteDelimiter() {
      return citeDelimiter;
    }

    public List<BibRecord> parse(String line) {
      if (line.startsWith("<bb>"))
        line = line.substring(4);
      String[] citesa = line.split("<bb>");
      List<String> cites = Arrays.asList(citesa);
      List<BibRecord> out = new ArrayList<BibRecord>();
      boolean first = true;
      for (String s : cites) {
        s = s.replaceAll("-<lb>", "").replaceAll("<lb>", " ").trim();
        if(first) { //don't overwrite number-bracket or number-dot
          if(s.length() > 0)
              if(RegexWithTimeout.matcher(pBracket, s).matches() ||
            RegexWithTimeout.matcher(pDot, s).matches()) {
                return removeNulls(out);
              }
              else {
                first = false;
              }
        }
        for(int i=0; i<recParser.length;i++) {
          BibRecord br = this.recParser[i].parseRecord(s);
          if(br!=null) {
            out.add(br);
            break;
          }
        }
      }
      out = removeNulls(out);
      return out;
    }
  }

  private class NumberDotNaturalLineBreaks extends NumberDot {

    NumberDotNaturalLineBreaks(Class [] c) {
      super(c);
    }
    
    @Override
    public List<BibRecord> parse(String line) {
      return parseWithGivenBreaks(line, true);
    }
    
  }
  
  private class NumberDot extends BracketNumber {
    NumberDot(Class [] c) {
      super(c);
    }
    
    protected List<BibRecord> parseWithGivenBreaks(String line, boolean bigBreaks) {
      if(!bigBreaks)
        line = line.replaceAll("<bb>", "<lb>");
      else
        line = line.replaceAll("<lb>", " ");
      int i = 0;
      String preTag = "<bb>";
      if(!bigBreaks)
        preTag = "<lb>";
      String tag = preTag + (++i) + ". ";
      List<String> cites = new ArrayList<String>();
      int st = line.indexOf(tag);
      while (st >= 0) {
        tag = preTag + (++i) + ". ";
        int end = line.indexOf(tag, st);
        if (end > 0) {
          cites.add(line.substring(st, end));
        } else {
          cites.add(line.substring(st));
        }
        st = end;
      }
      List<BibRecord> out = new ArrayList<BibRecord>();
      for (String s : cites) {
        s = s.replaceAll("-<lb>", "").replaceAll("<lb>", " ");
        s = s.replaceAll("-<bb>", "").replaceAll("<bb>", " ");
        for(int j=0; j<recParser.length;j++) {
          BibRecord br = this.recParser[j].parseRecord(s);
          if(br!=null) {
            out.add(br);
            break;
          }
        }
      }
      out = removeNulls(out);
      return out;
    }      
    
    public List<BibRecord> parse(String line) {
      return parseWithGivenBreaks(line,  false);
    }
  }

  private class BracketNumber extends BibStractor {
    protected final String citeRegex =
        "(?:" +
          "[\\[\\(]" +                   // open bracket/paren
          "(" + citeCharactersRe + ")" + // the meat
          "[\\]\\)]" +                   // close bracket/paren
        ")|(?:" +                        // or
          "\\.⍐" +                       // period, followed by superscript
          "(" + citeCharactersRe + ")" + // the meat
          "⍗" +                          // end of superscript
        ")";
    protected final String citeDelimiter = separatorRe;

    BracketNumber(Class[] c) {
      super(c);
    }

    public String getCiteRegex() {
      return citeRegex;
    }
    
    public String getCiteDelimiter() {
      return citeDelimiter;
    }

    public List<BibRecord> parse(String line) {
      line = line.replaceAll("<bb>", "<lb>");
      int i = 0;
      String tag = "[" + (++i) + "]";
      List<String> cites = new ArrayList<String>();
      int st = line.indexOf(tag);
      while (st >= 0) {
        tag = "<lb>[" + (++i) + "]";
        int end = line.indexOf(tag, st);
        if (end > 0) {
          cites.add(line.substring(st, end));
        } else {
          cites.add(line.substring(st));
        }
        st = end;
      }
      List<BibRecord> out = new ArrayList<BibRecord>();
      boolean first = true;
      for (String s : cites) {
        s = s.replaceAll("-<lb>(\\p{Ll})", "$1").replaceAll("<lb>", " ").trim();
        for(int j=0; j<recParser.length;j++) {
          BibRecord br = this.recParser[j].parseRecord(s);
          if(br!=null) {
            out.add(br);
            break;
          }
        }
      }
      out = removeNulls(out);
      return out;
    }
  }

  private class NumberCloseParen extends BibStractor {
    NumberCloseParen(Class[] c) {
      super(c);
    }

    private final String citeRegex = "\\(([0-9, \\p{Pd}]+)\\)";
    public String getCiteRegex() {
      return citeRegex;
    }

    private final String citeDelimiter = "(,| |;)+";
    public String getCiteDelimiter() {
      return citeDelimiter;
    }

    public List<BibRecord> parse(String line) {
      line = line.replaceAll("<bb>", "<lb>");
      int i = 0;
      String tag = (++i) + ")";
      List<String> cites = new ArrayList<String>();
      int st = line.indexOf(tag);
      while (st >= 0) {
        tag = "<lb>" + (++i) + ")";
        int end = line.indexOf(tag, st);
        if (end > 0) {
          cites.add(line.substring(st, end));
        } else {
          cites.add(line.substring(st));
        }
        st = end;
      }
      List<BibRecord> out = new ArrayList<BibRecord>();
      boolean first = true;
      for (String s : cites) {
        s = s.replaceAll("-<lb>(\\p{Ll})", "$1").replaceAll("<lb>", " ").trim();
        for(int j=0; j<recParser.length;j++) {
          BibRecord br = this.recParser[j].parseRecord(s);
          if(br!=null) {
            out.add(br);
            break;
          }
        }
      }
      out = removeNulls(out);
      return out;
    }
  }

  private class BracketName extends BibStractor {
    protected final String citeRegex = "\\[([^\\]]+)\\]";
    protected final String citeDelimiter = "; ?";

    BracketName(Class [] c) {
      super(c);
    }
    
    public String getCiteRegex() {
      return citeRegex;
    }

    public String getCiteDelimiter() {
      return citeDelimiter;
    }

    public List<BibRecord> parse(String line) {
      if (line.startsWith("<bb>"))
        line = line.substring(4);
      String[] citesa = line.split("<bb>");
      List<String> cites = Arrays.asList(citesa);
      List<BibRecord> out = new ArrayList<BibRecord>();
      boolean first = true;
      for (String s : cites) {
        s = s.replaceAll("-<lb>", "").replaceAll("<lb>", " ").trim();
        if(first) { //don't overwrite number-bracket or number-dot
          if(s.length() > 0)
              if(RegexWithTimeout.matcher(pBracket, s).matches() ||
            RegexWithTimeout.matcher(pDot, s).matches()) {
                return removeNulls(out);
              }
              else {
                first = false;
              }
        }
        for(int i=0; i<recParser.length;i++) {
          BibRecord br = this.recParser[i].parseRecord(s);
          if(br!=null) {
            out.add(br);
            break;
          }
        }
      }
      out = removeNulls(out);
      return out;
    }
  }
}
