@Test public void testTreeInsertParserTraining() throws Exception { ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData(); HeadRules headRules = ParserTestUtil.createTestHeadRules(); ParserModel model = Parser.train("eng", parseSamples, headRules, 100, 0); opennlp.tools.parser.Parser parser = ParserFactory.create(model); parser.parse(Parse.parseParse("She was just another freighter from the " + "States and she seemed as commonplace as her name .")); ByteArrayOutputStream outArray = new ByteArrayOutputStream(); model.serialize(outArray); outArray.close(); new ParserModel(new ByteArrayInputStream(outArray.toByteArray())); }
public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream( parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream( parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, checkReportMap, "check"); parseSamples.reset(); System.err.println("Training attacher"); ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.ATTACH); Map<String, String> attachReportMap = new HashMap<>(); EventTrainer attachTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("attach"), attachReportMap); MaxentModel attachModel = attachTrainer.train(attachEvents); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, attachReportMap, "attach"); return new ParserModel(languageCode, buildModel, checkModel, attachModel, posModel, chunkModel, rules, ParserType.TREEINSERT, manifestInfoEntries); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream( parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream( parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, checkReportMap, "check"); parseSamples.reset(); System.err.println("Training attacher"); ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.ATTACH); Map<String, String> attachReportMap = new HashMap<>(); EventTrainer attachTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("attach"), attachReportMap); MaxentModel attachModel = attachTrainer.train(attachEvents); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, attachReportMap, "attach"); return new ParserModel(languageCode, buildModel, checkModel, attachModel, posModel, chunkModel, rules, ParserType.TREEINSERT, manifestInfoEntries); } }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream( parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream( parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, checkReportMap, "check"); parseSamples.reset(); System.err.println("Training attacher"); ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.ATTACH); Map<String, String> attachReportMap = new HashMap<>(); EventTrainer attachTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("attach"), attachReportMap); MaxentModel attachModel = attachTrainer.train(attachEvents); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, attachReportMap, "attach"); return new ParserModel(languageCode, buildModel, checkModel, attachModel, posModel, chunkModel, rules, ParserType.TREEINSERT, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel,
                 POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize,
                 double advancePercentage); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream( parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream( parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, checkReportMap, "check"); parseSamples.reset(); System.err.println("Training attacher"); ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.ATTACH); Map<String, String> attachReportMap = new HashMap<>(); EventTrainer attachTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("attach"), attachReportMap); MaxentModel attachModel = attachTrainer.train(attachEvents); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, attachReportMap, "attach"); return new ParserModel(languageCode, buildModel, checkModel, attachModel, posModel, chunkModel, rules, ParserType.TREEINSERT, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel,
                 POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize,
                 double advancePercentage); static List<Parse> getRightFrontier(Parse root,Set<String> punctSet); static ParserModel train(String languageCode,
      ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams); static ParserModel train(String languageCode,
      ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream( parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream( parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, checkReportMap, "check"); parseSamples.reset(); System.err.println("Training attacher"); ObjectStream<Event> attachEvents = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.ATTACH); Map<String, String> attachReportMap = new HashMap<>(); EventTrainer attachTrainer = TrainerFactory.getEventTrainer( mlParams.getParameters("attach"), attachReportMap); MaxentModel attachModel = attachTrainer.train(attachEvents); opennlp.tools.parser.chunking.Parser.mergeReportIntoManifest( manifestInfoEntries, attachReportMap, "attach"); return new ParserModel(languageCode, buildModel, checkModel, attachModel, posModel, chunkModel, rules, ParserType.TREEINSERT, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel attachModel, MaxentModel checkModel,
                 POSTagger tagger, Chunker chunker, HeadRules headRules, int beamSize,
                 double advancePercentage); static List<Parse> getRightFrontier(Parse root,Set<String> punctSet); static ParserModel train(String languageCode,
      ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams); static ParserModel train(String languageCode,
      ObjectStream<Parse> parseSamples, HeadRules rules, int iterations, int cut); static final String DONE; static final String ATTACH_SISTER; static final String ATTACH_DAUGHTER; static final String NON_ATTACH; static final String BUILT; }