HOL-master/examples/lassie/sempre/src/edu/stanford/nlp/sempre/freebase/LexiconFn.java
package edu.stanford.nlp.sempre.freebase;

import com.google.common.base.Joiner;
import edu.stanford.nlp.sempre.*;
import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry;
import edu.stanford.nlp.sempre.freebase.lexicons.LexicalEntry.BinaryLexicalEntry;
import fig.basic.*;
import org.apache.lucene.queryparser.classic.ParseException;

import java.io.IOException;
import java.util.*;

/**
 * Looks up a string into the lexicon, interfacing with the fbalignment code.
 * @author Percy Liang
 */
public class LexiconFn extends SemanticFn {
  public static class Options {
    // Note: we filter here and not in entity lexicon so that don't need different cache for different numbers.
    @Option(gloss = "Number of entities to return from entity lexicon")
    public int maxEntityEntries = 100;
    @Option public int maxUnaryEntries = Integer.MAX_VALUE;
    @Option public int maxBinaryEntries = Integer.MAX_VALUE;

    @Option(gloss = "Verbose") public int verbose = 0;
    @Option(gloss = "Class name for lexicon") public String lexiconClassName;
    @Option public boolean useHistogramFeatures = true;
  }

  public static Options opts = new Options();
  private static Lexicon lexicon;
  public static Evaluation lexEval = new Evaluation();

  private String mode;  // unary, binary, or entity
  private EntityLexicon.SearchStrategy entitySearchStrategy;  // For entities, how to search
  private TextToTextMatcher textToTextMatcher = new TextToTextMatcher();
  private FbFormulasInfo fbFormulaInfo;

  public static final Map<Pair<String, Formula>, Set<Integer>> correctEntryToExampleIds =
          new HashMap<>();

  public LexiconFn() throws IOException {
    lexicon = Lexicon.getSingleton();
    fbFormulaInfo = FbFormulasInfo.getSingleton();
  }

  public void init(LispTree tree) {
    super.init(tree);
    for (int i = 1; i < tree.children.size(); i++) {
      String value = tree.child(i).value;

      // mode
      if (value.equals("unary")) this.mode = "unary";
      else if (value.equals("binary")) this.mode = "binary";
      else if (value.equals("entity")) this.mode = "entity";
      // entity search strategy
      // TODO(joberant): aren't we saying these should be set by a flag so the cache doesn't get messed up?
      else if (value.equals("inexact")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.inexact;
      else if (value.equals("exact")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.exact;
      else if (value.equals("fbsearch")) this.entitySearchStrategy = EntityLexicon.SearchStrategy.fbsearch;
      else throw new RuntimeException("Invalid argument: " + value);
    }
  }

  public static void getEntityEntryFeatures(LexicalEntry.EntityLexicalEntry eEntry, FeatureVector features,
                                            Callable c, Example ex) {

    if (FeatureExtractor.containsDomain("basicStats")) {
      if (opts.useHistogramFeatures)
        features.addHistogram("basicStats", "entity.popularity ", eEntry.getPopularity());
      else
        features.addWithBias("basicStats", "entity.popularity", Math.log(eEntry.getPopularity() + 1));
    }
    if (FeatureExtractor.containsDomain("entityFeatures")) {
      for (String feature : eEntry.entityFeatures.keySet()) {
        double value = eEntry.entityFeatures.getCount(feature);
        features.addWithBias("entityFeatures", "entity." + feature, value);
      }
      features.add("entityFeatures", "entity.pos=" + ex.languageInfo.posSeq(c.getStart(), c.getEnd()));
      features.add("entityFeatures", "entity.mention_length=" + (c.getEnd() - c.getStart()));
    }
    if (FeatureExtractor.containsDomain("lexAlign"))
      features.add("lexAlign", eEntry.textDescription + " --- " + eEntry.formula);
  }

  public static void getUnaryEntryFeatures(LexicalEntry.UnaryLexicalEntry uEntry, FeatureVector features) {
    if (FeatureExtractor.containsDomain("basicStats")) {
      if (opts.useHistogramFeatures)
        features.addHistogram("basicStats", "unary.popularity ", uEntry.getPopularity() + 1);
      else
        features.addWithBias("basicStats", "unary.popularity", Math.log(uEntry.getPopularity() + 1));
    }
    // Alignment scores features
    if (FeatureExtractor.containsDomain("alignmentScores")) {
      for (String feature : uEntry.alignmentScores.keySet()) {
        features.addWithBias("alignmentScores",  "unary." + feature, Math.log(MapUtils.getDouble(uEntry.alignmentScores, feature, 0.0) + 1));
      }
    }

    if (FeatureExtractor.containsDomain("basicStats")) {
      if (uEntry.getDistance() < 0.0001)
        features.add("basicStats", "unary.equal");

      // adding the source of the lexical entry as a feature
      features.add("basicStats", "unary.source=" + uEntry.source);
    }
    if (FeatureExtractor.containsDomain("lexAlign"))
      features.add("lexAlign", uEntry.textDescription + " --- " + uEntry.formula);
  }

  public static void getBinaryEntryFeatures(BinaryLexicalEntry bEntry, FeatureVector features) {
    if (FeatureExtractor.containsDomain("basicStats")) {
      if (opts.useHistogramFeatures)
        features.addHistogram("basicStats", "binary.popularity ", bEntry.getPopularity() + 1);
      else
        features.addWithBias("basicStats", "binary.popularity", Math.log(bEntry.getPopularity() + 1));
      // adding the source of the lexical entry as a feature
      features.add("basicStats", "binary." + bEntry.source);
    }
    // Alignment scores features
    if (FeatureExtractor.containsDomain("alignmentScores")) {
      for (String feature : bEntry.alignmentScores.keySet()) {
        features.addWithBias("alignmentScores", "binary." + feature, Math.log(MapUtils.getDouble(bEntry.alignmentScores, feature, 0.0) + 1));
      }
    }
    if (FeatureExtractor.containsDomain("lexAlign"))
      features.add("lexAlign", bEntry.textDescription + " --- " + bEntry.formula);
  }

  // Convert LexicalEntry into a form consumable by the semantic parser.
  private Derivation convert(Example ex,
                             Callable c,
                             String mode,
                             String word,
                             LexicalEntry entry) {
    FeatureVector features = new FeatureVector();
    SemType type;

    switch (mode) {
      case "entity":
        // Entities
        LexicalEntry.EntityLexicalEntry eEntry = (LexicalEntry.EntityLexicalEntry) entry;
        getEntityEntryFeatures(eEntry, features, c, ex);
        type = ((LexicalEntry.EntityLexicalEntry) entry).type;
        break;
      case "unary":
        // Unaries
        LexicalEntry.UnaryLexicalEntry uEntry = (LexicalEntry.UnaryLexicalEntry) entry;
        getUnaryEntryFeatures(uEntry, features);
        type = ((LexicalEntry.UnaryLexicalEntry) entry).type;
        break;
      case "binary":
        // Binaries
        BinaryLexicalEntry bEntry = (BinaryLexicalEntry) entry;
        getBinaryEntryFeatures(bEntry, features);
        // features that depend on entry but also on the example
        features.add(
                textToTextMatcher.extractFeatures(
                        ex.languageInfo.tokens.subList(c.getStart(), c.getEnd()),
                        ex.languageInfo.posTags.subList(c.getStart(), c.getEnd()),
                        ex.languageInfo.lemmaTokens.subList(c.getStart(), c.getEnd()),
                        bEntry.fbDescriptions));

        // Note that expectedType2 is the argument type, expectedType1 is the return type.
        type = SemType.newFuncSemType(bEntry.getExpectedType2(), bEntry.getExpectedType1());
        break;
      default:
        throw new RuntimeException("Invalid mode: " + mode);
    }

    Derivation newDeriv = new Derivation.Builder()
            .withCallable(c)
            .formula(entry.formula)
            .type(type)
            .localFeatureVector(features)
            .createDerivation();

    if (SemanticFn.opts.trackLocalChoices)
      newDeriv.addLocalChoice("LexiconFn " + newDeriv.startEndString(ex.getTokens()) + " " + entry);

    if (opts.verbose >= 3) {
      LogInfo.logs(
              "LexiconFn: %s [%s => %s ~ %s | %s]: popularity = %s, distance = %s, type = %s, source=%s",
              mode, word, entry.normalizedTextDesc, entry.fbDescriptions, newDeriv.formula,
              entry.getPopularity(), entry.getDistance(), newDeriv.type, entry.source);
    }
    return newDeriv;
  }

  public DerivationStream call(Example ex, Callable c) {

    if (opts.verbose >= 5) LogInfo.begin_track("LexicalFn.call: %s", c.childStringValue(0));

    String query = c.childStringValue(0);
    DerivationStream res;

    try {
      switch (mode) {
        // Entities
        case "entity": {
          // if (opts.verbose >= 2)
          // LogInfo.log("LexiconFn: querying for entity: " + query);

          List<? extends LexicalEntry> entries = lexicon.lookupEntities(query, entitySearchStrategy);
          lexEval.add("entity", !entries.isEmpty());
          entries = entries.subList(0, Math.min(opts.maxEntityEntries, entries.size()));
          res = new LazyLexiconFnDerivs(ex, c, entries, query);
          break;
        }
        // Unaries
        case "unary": {
            List<? extends LexicalEntry> entries = lexicon.lookupUnaryPredicates(query);
          lexEval.add("unary", !entries.isEmpty());
          entries = entries.subList(0, Math.min(opts.maxUnaryEntries, entries.size()));
          res = new LazyLexiconFnDerivs(ex, c, entries, query);
          break;
        }
        // Binaries
        case "binary": {
          List<? extends LexicalEntry> entries = lexicon.lookupBinaryPredicates(query);
          lexEval.add("binary", !entries.isEmpty());
          List<LexicalEntry> filteredEntries = new ArrayList<>();
          // filter cvt entries (TODO(joberant): remove this hack)
          for (LexicalEntry entry : entries) {
            if (!fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType1)
                    && !fbFormulaInfo.isCvt(((BinaryLexicalEntry) entry).expectedType2))
              filteredEntries.add(entry);
          }
          filteredEntries = filteredEntries.subList(0, Math.min(opts.maxBinaryEntries, filteredEntries.size()));
          res = new LazyLexiconFnDerivs(ex, c, filteredEntries, query);
          break;
        }
        default:
          throw new RuntimeException("Illegal mode: " + mode);
      }
    } catch (IOException | ParseException e) {
      throw new RuntimeException(e);
    }

    if (opts.verbose >= 5) LogInfo.end_track();
    return res;
  }

  // if there was bridging then have a rule from tokens to binary
  @Override
  public void addFeedback(Example ex) {
    LogInfo.begin_track("LexiconFn.addFeedback");
    Set<Pair<String, Formula>> correctLexemeFormulaMatches = collectLexemeFormulaPairs(ex);

    for (Pair<String, Formula> pair: correctLexemeFormulaMatches) {
      LogInfo.logs("LexiconFn.addFeedback: %s => %s", pair.getFirst(), pair.getSecond());
      // TODO(joberant): hack to get id
      MapUtils.addToSet(correctEntryToExampleIds, pair, Integer.parseInt(ex.id.substring(ex.id.lastIndexOf(':') + 1)));
      BinaryLexicon.getInstance().updateLexicon(pair, correctEntryToExampleIds.get(pair).size());
    }
    LogInfo.end_track();
  }

  private static Set<Pair<String, Formula>> collectLexemeFormulaPairs(Example ex) {
    Set<Pair<String, Formula>> res = new HashSet<>();
    Set<Pair<String, Formula>> temp = new HashSet<>();
    for (Derivation correctDerivation : ex.getCorrectDerivations()) {
      // get all join formulas
      List<Formula> relations =
              correctDerivation.formula.mapToList(formula -> {
                List<Formula> res1 = new ArrayList<>();
                if (formula instanceof JoinFormula)
                  res1.add(((JoinFormula) formula).relation);
                return res1;
              }, false);
      Set<Integer> validIndices = new HashSet<>();
      findValidIndices(correctDerivation, validIndices);
      // match formulas
      for (Formula relation : relations) {
        for (int i = 0; i < ex.numTokens(); ++i) {
          if (LanguageInfo.isContentWord(ex.posTag(i)) &&
                  validIndices.contains(i)) {
            temp.add(Pair.newPair(ex.languageInfo.lemmaTokens.get(i), relation));
          }
        }
      }
    }
    // reverse invalid relations
    for (Pair<String, Formula> pair: temp) {
      if (!BinaryLexicon.getInstance().validBinaryFormula(pair.getSecond())) {
        pair.setSecond(FbFormulasInfo.getSingleton().equivalentFormula(pair.getSecond()));
      }
      res.add(pair);
    }
    return res;
  }

  @Override
  public void sortOnFeedback(Params params) {
    LogInfo.begin_track("Learner.sortLexiconOnFeedback");
    BinaryLexicon.getInstance().sortLexiconByFeedback(params);
    UnaryLexicon.getInstance().sortLexiconByFeedback(params);
    LogInfo.end_track();
  }

  // todo - this method is grammar specific and that is bad
  private static void findValidIndices(Derivation deriv, Set<Integer> indices) {
    if (deriv.cat.equals("$Entity"))
      return;
    if (deriv.children.size() == 0) {
      for (int i = deriv.start; i < deriv.end; ++i)
        indices.add(i);
      return;
    }
    for (Derivation child : deriv.children)
      findValidIndices(child, indices);
  }

  /** For now this ignores stemming!!! */
  private static boolean doesContextMatch(Example ex, Derivation deriv, BinaryLexicalEntry bEntry) {

    if (bEntry.isFullLexemeEqualToNormalizedText())
      return true;
    // get the left and right context surrounding the core (normalized text)
    String[] leftContext = bEntry.getLeftContext();
    String[] rightContext = bEntry.getRightContext();
    // match right context
    for (int i = 0; i < rightContext.length; ++i) {
      // in this case all context words were matched and some were dropped but there was no mismatch
      if (deriv.end + i >= ex.numTokens() || ex.token(deriv.end + i).equals("?"))
        break;

      if (!rightContext[0].equals(ex.lemmaToken(deriv.end + i))) {
        if (opts.verbose >= 4) {
          LogInfo.logs(
                  "RIGHT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s",
                  bEntry.fullLexeme,
                  bEntry.normalizedTextDesc,
                  Joiner.on(' ').join(leftContext),
                  Joiner.on(' ').join(rightContext),
                  Joiner.on(' ').join(ex.languageInfo.tokens),
                  bEntry.formula);
        }
        return false;
      }
    }

    // match right context
    for (int i = 0; i < leftContext.length; ++i) {
      if (deriv.start - i - 1 < 0) // in this case all context words were matched and some were dropped but there was no mismatch
        break;
      if (!leftContext[leftContext.length - i - 1].equals(ex.lemmaToken(deriv.start - i - 1))) {
        if (opts.verbose >= 2) {
          LogInfo.logs(
                  "LEFT CONTEXT MISMATCH: full lexeme=%s, normalized text=%s left context=%s, right context=%s example=%s, formula=%s",
                  bEntry.fullLexeme,
                  bEntry.normalizedTextDesc,
                  Joiner.on(' ').join(leftContext),
                  Joiner.on(' ').join(rightContext),
                  Joiner.on(' ').join(ex.languageInfo.tokens),
                  bEntry.formula);
        }
        return false;
      }
    }
    return true;
  }

  public class LazyLexiconFnDerivs extends MultipleDerivationStream {

    private Example ex;
    private Callable callable;
    private List<? extends LexicalEntry> entries; // we get one derivation from each entry
    private String query;
    private int currIndex = 0;

    public LazyLexiconFnDerivs(Example ex, Callable c, List<? extends LexicalEntry> entries, String query) {
      this.ex = ex;
      this.callable = c;
      this.entries = entries;
      this.query = query;
    }

    @Override
    public int estimatedSize() {
      return entries.size() - currIndex;
    }

    @Override
    public Derivation createDerivation() {
      if (currIndex == entries.size())
        return null;
      LexicalEntry currEntry = entries.get(currIndex++);
      Derivation res;
      switch (mode) {
        case "entity":
          res = convert(ex, callable, "entity", query, currEntry);
          break;
        case "unary":
          res = convert(ex, callable, "unary", query, currEntry);
          break;
        case "binary":
          res = convert(ex, callable, "binary", query, currEntry);
          // add context matching feature
          if (FeatureExtractor.containsDomain("context")) {
            if (!doesContextMatch(ex, res, (BinaryLexicalEntry) currEntry))
              res.addFeature("context", "binary.contextMismatch");
          }
          break;
        default:
          throw new RuntimeException("Illegal mode: " + mode);
      }
      return res;
    }
  }
}

