@Test public void testChunkingParserTraining() throws Exception { ObjectStream<Parse> parseSamples = ParserTestUtil.openTestTrainingData(); HeadRules headRules = ParserTestUtil.createTestHeadRules(); ParserModel model = Parser.train("eng", parseSamples, headRules, TrainingParameters.defaultParams()); opennlp.tools.parser.Parser parser = ParserFactory.create(model); ByteArrayOutputStream outArray = new ByteArrayOutputStream(); model.serialize(outArray); outArray.close(); ParserModel outputModel = new ParserModel(new ByteArrayInputStream(outArray.toByteArray())); }
public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); TrainingParameters posTaggerParams = mlParams.getParameters("tagger"); if (!posTaggerParams.getObjectSettings().containsKey(BeamSearch.BEAM_SIZE_PARAMETER)) { mlParams.put("tagger", BeamSearch.BEAM_SIZE_PARAMETER, 10); } POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check"); return new ParserModel(languageCode, buildModel, checkModel, posModel, chunkModel, rules, ParserType.CHUNKING, manifestInfoEntries); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); TrainingParameters posTaggerParams = mlParams.getParameters("tagger"); if (!posTaggerParams.getObjectSettings().containsKey(BeamSearch.BEAM_SIZE_PARAMETER)) { mlParams.put("tagger", BeamSearch.BEAM_SIZE_PARAMETER, 10); } POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check"); return new ParserModel(languageCode, buildModel, checkModel, posModel, chunkModel, rules, ParserType.CHUNKING, manifestInfoEntries); } }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); TrainingParameters posTaggerParams = mlParams.getParameters("tagger"); if (!posTaggerParams.getObjectSettings().containsKey(BeamSearch.BEAM_SIZE_PARAMETER)) { mlParams.put("tagger", BeamSearch.BEAM_SIZE_PARAMETER, 10); } POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check"); return new ParserModel(languageCode, buildModel, checkModel, posModel, chunkModel, rules, ParserType.CHUNKING, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker,
                 HeadRules headRules, int beamSize, double advancePercentage); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); TrainingParameters posTaggerParams = mlParams.getParameters("tagger"); if (!posTaggerParams.getObjectSettings().containsKey(BeamSearch.BEAM_SIZE_PARAMETER)) { mlParams.put("tagger", BeamSearch.BEAM_SIZE_PARAMETER, 10); } POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check"); return new ParserModel(languageCode, buildModel, checkModel, posModel, chunkModel, rules, ParserType.CHUNKING, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker,
                 HeadRules headRules, int beamSize, double advancePercentage); static void mergeReportIntoManifest(Map<String, String> manifest,
      Map<String, String> report, String namespace); static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples,
                                  HeadRules rules, TrainingParameters mlParams); }
Parser extends AbstractBottomUpParser { public static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples, HeadRules rules, TrainingParameters mlParams) throws IOException { System.err.println("Building dictionary"); Dictionary mdict = buildDictionary(parseSamples, rules, mlParams); parseSamples.reset(); Map<String, String> manifestInfoEntries = new HashMap<>(); System.err.println("Training builder"); ObjectStream<Event> bes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.BUILD, mdict); Map<String, String> buildReportMap = new HashMap<>(); EventTrainer buildTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("build"), buildReportMap); MaxentModel buildModel = buildTrainer.train(bes); mergeReportIntoManifest(manifestInfoEntries, buildReportMap, "build"); parseSamples.reset(); TrainingParameters posTaggerParams = mlParams.getParameters("tagger"); if (!posTaggerParams.getObjectSettings().containsKey(BeamSearch.BEAM_SIZE_PARAMETER)) { mlParams.put("tagger", BeamSearch.BEAM_SIZE_PARAMETER, 10); } POSModel posModel = POSTaggerME.train(languageCode, new PosSampleStream(parseSamples), mlParams.getParameters("tagger"), new POSTaggerFactory()); parseSamples.reset(); ChunkerModel chunkModel = ChunkerME.train(languageCode, new ChunkSampleStream(parseSamples), mlParams.getParameters("chunker"), new ParserChunkerFactory()); parseSamples.reset(); System.err.println("Training checker"); ObjectStream<Event> kes = new ParserEventStream(parseSamples, rules, ParserEventTypeEnum.CHECK); Map<String, String> checkReportMap = new HashMap<>(); EventTrainer checkTrainer = TrainerFactory.getEventTrainer(mlParams.getParameters("check"), checkReportMap); MaxentModel checkModel = checkTrainer.train(kes); mergeReportIntoManifest(manifestInfoEntries, checkReportMap, "check"); return new ParserModel(languageCode, buildModel, checkModel, posModel, chunkModel, rules, ParserType.CHUNKING, manifestInfoEntries); } Parser(ParserModel model, int beamSize, double advancePercentage);  Parser(ParserModel model); private  Parser(MaxentModel buildModel, MaxentModel checkModel, POSTagger tagger, Chunker chunker,
                 HeadRules headRules, int beamSize, double advancePercentage); static void mergeReportIntoManifest(Map<String, String> manifest,
      Map<String, String> report, String namespace); static ParserModel train(String languageCode, ObjectStream<Parse> parseSamples,
                                  HeadRules rules, TrainingParameters mlParams);  }