public final ParserData getParserDataFromTreebank(Treebank trainTreebank,
      GrammarCompactor compactor,
      Treebank tuneTreebank) {
    System.err.println("Currently " + new Date());
    printOptions(true, op);
    Pair<List<Tree>,List<Tree>> pair = getAnnotatedBinaryTreebankFromTreebank(trainTreebank, tuneTreebank, op);
    List<Tree> binaryTrainTrees = pair.first();
    List<Tree> binaryTuneTrees = pair.second();

    // extract grammars
    Extractor<Pair<UnaryGrammar,BinaryGrammar>> bgExtractor = new BinaryGrammarExtractor();
    // Extractor lexExtractor = new LexiconExtractor();
    Extractor<DependencyGrammar> dgExtractor = op.tlpParams.dependencyGrammarExtractor(op);
    //TreeExtractor uwmExtractor = new UnknownWordModelExtractor(binaryTrainTrees.size());
    System.err.print("Extracting PCFG...");
    Pair<UnaryGrammar,BinaryGrammar> bgug = bgExtractor.extract(binaryTrainTrees);
    Timing.tick("done.");
    
    //TODO: wsg2011 Not sure if this should come before or after grammar compaction
    if(Train.ruleSmoothing) {
      System.err.print("Smoothing PCFG...");
      Function<Pair<UnaryGrammar,BinaryGrammar>,Pair<UnaryGrammar,BinaryGrammar>> smoother = new LinearGrammarSmoother();
      bgug = smoother.apply(bgug);
      Timing.tick("done.");
    }
    
    if (compactor != null) {
      System.err.print("Compacting grammar...");
      bgug = compactor.compactGrammar(bgug);
      Timing.tick("done.");
    }
    
    System.err.print("Compiling grammar...");
    BinaryGrammar bg = bgug.second;
    bg.splitRules();
    UnaryGrammar ug = bgug.first;
    // System.err.println("\nUnary grammar built by BinaryGrammarExtractor");
    // ug.writeAllData(new OutputStreamWriter(System.err));
    ug.purgeRules();
    // System.err.println("Unary grammar after purgeRules");
    // ug.writeAllData(new OutputStreamWriter(System.err));
    Timing.tick("done");
    
    System.err.print("Extracting Lexicon...");
    Lexicon lex = op.tlpParams.lex(op.lexOptions);
    lex.train(binaryTrainTrees);
    Timing.tick("done.");

    DependencyGrammar dg = null;
    if (op.doDep) {
      System.err.print("Extracting Dependencies...");
      dg = dgExtractor.extract(binaryTrainTrees);
      //      ((ChineseSimWordAvgDepGrammar)dg).setLex(lex);
      Timing.tick("done.");
      //System.out.println(dg);
      //System.err.print("Extracting Unknown Word Model...");
      //UnknownWordModel uwm = (UnknownWordModel)uwmExtractor.extract(binaryTrainTrees);
      //Timing.tick("done.");
      if (tuneTreebank != null) {
        System.err.print("Tuning Dependency Model...");
        dg.setLexicon(lex); // MG2008: needed if using PwGt model
        dg.tune(binaryTuneTrees);
        Timing.tick("done.");
      }
    }
    Map<String,Numberer> numbs = Numberer.getNumberers();

    System.err.println("Done training parser.");
    if (Train.trainTreeFile!=null) {
      try {
        System.err.print("Writing out binary trees to "+ Train.trainTreeFile+"...");
        IOUtils.writeObjectToFile(binaryTrainTrees, Train.trainTreeFile);
        Timing.tick("done.");
      } catch (Exception e) {
        System.err.println("Problem writing out binary trees.");
      }
    }
    return new ParserData(lex, bg, ug, dg, numbs, op);
  }