@Test(expected = InsufficientTrainingDataException.class) public void testInsufficientData() throws IOException { ObjectStream<LemmaSample> sampleStream = new LemmaSampleStream( new PlainTextByLineStream(new MockInputStreamFactory( new File("opennlp/tools/lemmatizer/trial.old-insufficient.tsv")), StandardCharsets.UTF_8)); TrainingParameters params = new TrainingParameters(); params.put(TrainingParameters.ITERATIONS_PARAM, 100); params.put(TrainingParameters.CUTOFF_PARAM, 5); LemmatizerME.train("eng", sampleStream, params, new LemmatizerFactory()); }
public static LemmatizerModel train(String languageCode, ObjectStream<LemmaSample> samples, TrainingParameters trainParams, LemmatizerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, LemmatizerME.DEFAULT_BEAM_SIZE); LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel lemmatizerModel = null; SequenceClassificationModel<String> seqLemmatizerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new LemmaSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); seqLemmatizerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (lemmatizerModel != null) { return new LemmatizerModel(languageCode, lemmatizerModel, beamSize, manifestInfoEntries, posFactory); } else { return new LemmatizerModel(languageCode, seqLemmatizerModel, manifestInfoEntries, posFactory); } }
LemmatizerME implements Lemmatizer { public static LemmatizerModel train(String languageCode, ObjectStream<LemmaSample> samples, TrainingParameters trainParams, LemmatizerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, LemmatizerME.DEFAULT_BEAM_SIZE); LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel lemmatizerModel = null; SequenceClassificationModel<String> seqLemmatizerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new LemmaSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); seqLemmatizerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (lemmatizerModel != null) { return new LemmatizerModel(languageCode, lemmatizerModel, beamSize, manifestInfoEntries, posFactory); } else { return new LemmatizerModel(languageCode, seqLemmatizerModel, manifestInfoEntries, posFactory); } } }
LemmatizerME implements Lemmatizer { public static LemmatizerModel train(String languageCode, ObjectStream<LemmaSample> samples, TrainingParameters trainParams, LemmatizerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, LemmatizerME.DEFAULT_BEAM_SIZE); LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel lemmatizerModel = null; SequenceClassificationModel<String> seqLemmatizerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new LemmaSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); seqLemmatizerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (lemmatizerModel != null) { return new LemmatizerModel(languageCode, lemmatizerModel, beamSize, manifestInfoEntries, posFactory); } else { return new LemmatizerModel(languageCode, seqLemmatizerModel, manifestInfoEntries, posFactory); } } LemmatizerME(LemmatizerModel model); }
LemmatizerME implements Lemmatizer { public static LemmatizerModel train(String languageCode, ObjectStream<LemmaSample> samples, TrainingParameters trainParams, LemmatizerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, LemmatizerME.DEFAULT_BEAM_SIZE); LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel lemmatizerModel = null; SequenceClassificationModel<String> seqLemmatizerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new LemmaSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); seqLemmatizerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (lemmatizerModel != null) { return new LemmatizerModel(languageCode, lemmatizerModel, beamSize, manifestInfoEntries, posFactory); } else { return new LemmatizerModel(languageCode, seqLemmatizerModel, manifestInfoEntries, posFactory); } } LemmatizerME(LemmatizerModel model); String[] lemmatize(String[] toks, String[] tags); @Override List<List<String>> lemmatize(List<String> toks,
      List<String> tags); String[] predictSES(String[] toks, String[] tags); String[][] predictLemmas(int numLemmas, String[] toks, String[] tags); static String[] decodeLemmas(String[] toks, String[] preds); static String[] encodeLemmas(String[] toks, String[] lemmas); Sequence[] topKSequences(String[] sentence, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore); void probs(double[] probs); double[] probs(); static LemmatizerModel train(String languageCode,
      ObjectStream<LemmaSample> samples, TrainingParameters trainParams,
      LemmatizerFactory posFactory); Sequence[] topKLemmaClasses(String[] sentence, String[] tags); Sequence[] topKLemmaClasses(String[] sentence, String[] tags, double minSequenceScore); }
LemmatizerME implements Lemmatizer { public static LemmatizerModel train(String languageCode, ObjectStream<LemmaSample> samples, TrainingParameters trainParams, LemmatizerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, LemmatizerME.DEFAULT_BEAM_SIZE); LemmatizerContextGenerator contextGenerator = posFactory.getContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel lemmatizerModel = null; SequenceClassificationModel<String> seqLemmatizerModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new LemmaSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); lemmatizerModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); LemmaSampleSequenceStream ss = new LemmaSampleSequenceStream(samples, contextGenerator); seqLemmatizerModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (lemmatizerModel != null) { return new LemmatizerModel(languageCode, lemmatizerModel, beamSize, manifestInfoEntries, posFactory); } else { return new LemmatizerModel(languageCode, seqLemmatizerModel, manifestInfoEntries, posFactory); } } LemmatizerME(LemmatizerModel model); String[] lemmatize(String[] toks, String[] tags); @Override List<List<String>> lemmatize(List<String> toks,
      List<String> tags); String[] predictSES(String[] toks, String[] tags); String[][] predictLemmas(int numLemmas, String[] toks, String[] tags); static String[] decodeLemmas(String[] toks, String[] preds); static String[] encodeLemmas(String[] toks, String[] lemmas); Sequence[] topKSequences(String[] sentence, String[] tags); Sequence[] topKSequences(String[] sentence, String[] tags, double minSequenceScore); void probs(double[] probs); double[] probs(); static LemmatizerModel train(String languageCode,
      ObjectStream<LemmaSample> samples, TrainingParameters trainParams,
      LemmatizerFactory posFactory); Sequence[] topKLemmaClasses(String[] sentence, String[] tags); Sequence[] topKLemmaClasses(String[] sentence, String[] tags, double minSequenceScore); static final int LEMMA_NUMBER; static final int DEFAULT_BEAM_SIZE; }