@Test(expected = InsufficientTrainingDataException.class) public void testInsufficientData() throws IOException { ResourceAsStreamFactory in = new ResourceAsStreamFactory(getClass(), "/opennlp/tools/chunker/test-insufficient.txt"); ObjectStream<ChunkSample> sampleStream = new ChunkSampleStream( new PlainTextByLineStream(in, StandardCharsets.UTF_8)); TrainingParameters params = new TrainingParameters(); params.put(TrainingParameters.ITERATIONS_PARAM, 70); params.put(TrainingParameters.CUTOFF_PARAM, 1); ChunkerME.train("eng", sampleStream, params, new ChunkerFactory()); }
public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in, TrainingParameters mlParams, ChunkerFactory factory) throws IOException { int beamSize = mlParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, ChunkerME.DEFAULT_BEAM_SIZE); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(mlParams); MaxentModel chunkerModel = null; SequenceClassificationModel<String> seqChunkerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries); chunkerModel = trainer.train(es); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( mlParams, manifestInfoEntries); ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator()); seqChunkerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (chunkerModel != null) { return new ChunkerModel(lang, chunkerModel, beamSize, manifestInfoEntries, factory); } else { return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory); } }
ChunkerME implements Chunker { public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in, TrainingParameters mlParams, ChunkerFactory factory) throws IOException { int beamSize = mlParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, ChunkerME.DEFAULT_BEAM_SIZE); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(mlParams); MaxentModel chunkerModel = null; SequenceClassificationModel<String> seqChunkerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries); chunkerModel = trainer.train(es); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( mlParams, manifestInfoEntries); ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator()); seqChunkerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (chunkerModel != null) { return new ChunkerModel(lang, chunkerModel, beamSize, manifestInfoEntries, factory); } else { return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory); } } }
ChunkerME implements Chunker { public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in, TrainingParameters mlParams, ChunkerFactory factory) throws IOException { int beamSize = mlParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, ChunkerME.DEFAULT_BEAM_SIZE); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(mlParams); MaxentModel chunkerModel = null; SequenceClassificationModel<String> seqChunkerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries); chunkerModel = trainer.train(es); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( mlParams, manifestInfoEntries); ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator()); seqChunkerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (chunkerModel != null) { return new ChunkerModel(lang, chunkerModel, beamSize, manifestInfoEntries, factory); } else { return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory); } } @Deprecated private  ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<TokenTag> sequenceValidator,
      ChunkerContextGenerator contextGenerator); @Deprecated private  ChunkerME(ChunkerModel model, int beamSize);  ChunkerME(ChunkerModel model); }
ChunkerME implements Chunker { public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in, TrainingParameters mlParams, ChunkerFactory factory) throws IOException { int beamSize = mlParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, ChunkerME.DEFAULT_BEAM_SIZE); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(mlParams); MaxentModel chunkerModel = null; SequenceClassificationModel<String> seqChunkerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries); chunkerModel = trainer.train(es); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( mlParams, manifestInfoEntries); ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator()); seqChunkerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (chunkerModel != null) { return new ChunkerModel(lang, chunkerModel, beamSize, manifestInfoEntries, factory); } else { return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory); } } @Deprecated private  ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<TokenTag> sequenceValidator,
      ChunkerContextGenerator contextGenerator); @Deprecated private  ChunkerME(ChunkerModel model, int beamSize);  ChunkerME(ChunkerModel model); String[] chunk(String[] toks, String[] tags); Span[] chunkAsSpans(String[] toks, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore); void probs(double[] probs); double[] probs(); static ChunkerModel train(String lang, ObjectStream<ChunkSample> in,
      TrainingParameters mlParams, ChunkerFactory factory); }
ChunkerME implements Chunker { public static ChunkerModel train(String lang, ObjectStream<ChunkSample> in, TrainingParameters mlParams, ChunkerFactory factory) throws IOException { int beamSize = mlParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, ChunkerME.DEFAULT_BEAM_SIZE); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(mlParams); MaxentModel chunkerModel = null; SequenceClassificationModel<String> seqChunkerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new ChunkerEventStream(in, factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer(mlParams, manifestInfoEntries); chunkerModel = trainer.train(es); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( mlParams, manifestInfoEntries); ChunkSampleSequenceStream ss = new ChunkSampleSequenceStream(in, factory.getContextGenerator()); seqChunkerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (chunkerModel != null) { return new ChunkerModel(lang, chunkerModel, beamSize, manifestInfoEntries, factory); } else { return new ChunkerModel(lang, seqChunkerModel, manifestInfoEntries, factory); } } @Deprecated private  ChunkerME(ChunkerModel model, int beamSize, SequenceValidator<TokenTag> sequenceValidator,
      ChunkerContextGenerator contextGenerator); @Deprecated private  ChunkerME(ChunkerModel model, int beamSize);  ChunkerME(ChunkerModel model); String[] chunk(String[] toks, String[] tags); Span[] chunkAsSpans(String[] toks, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore); void probs(double[] probs); double[] probs(); static ChunkerModel train(String lang, ObjectStream<ChunkSample> in,
      TrainingParameters mlParams, ChunkerFactory factory); static final int DEFAULT_BEAM_SIZE; }