@Test(expected = InsufficientTrainingDataException.class) public void insufficientTestData() throws IOException { InputStreamFactory in = new ResourceAsStreamFactory(POSTaggerMETest.class, "/opennlp/tools/postag/AnnotatedSentencesInsufficient.txt"); ObjectStream<POSSample> stream = new WordTagSampleStream( new PlainTextByLineStream(in, StandardCharsets.UTF_8)); TrainingParameters params = new TrainingParameters(); params.put(TrainingParameters.ALGORITHM_PARAM, ModelType.MAXENT.name()); params.put(TrainingParameters.ITERATIONS_PARAM, 100); params.put(TrainingParameters.CUTOFF_PARAM, 5); POSTaggerME.train("eng", stream, params, new POSTaggerFactory()); }
public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams, POSTaggerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE); POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel posModel = null; SequenceClassificationModel<String> seqPosModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); seqPosModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (posModel != null) { return new POSModel(languageCode, posModel, beamSize, manifestInfoEntries, posFactory); } else { return new POSModel(languageCode, seqPosModel, manifestInfoEntries, posFactory); } }
POSTaggerME implements POSTagger { public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams, POSTaggerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE); POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel posModel = null; SequenceClassificationModel<String> seqPosModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); seqPosModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (posModel != null) { return new POSModel(languageCode, posModel, beamSize, manifestInfoEntries, posFactory); } else { return new POSModel(languageCode, seqPosModel, manifestInfoEntries, posFactory); } } }
POSTaggerME implements POSTagger { public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams, POSTaggerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE); POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel posModel = null; SequenceClassificationModel<String> seqPosModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); seqPosModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (posModel != null) { return new POSModel(languageCode, posModel, beamSize, manifestInfoEntries, posFactory); } else { return new POSModel(languageCode, seqPosModel, manifestInfoEntries, posFactory); } } POSTaggerME(POSModel model); }
POSTaggerME implements POSTagger { public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams, POSTaggerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE); POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel posModel = null; SequenceClassificationModel<String> seqPosModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); seqPosModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (posModel != null) { return new POSModel(languageCode, posModel, beamSize, manifestInfoEntries, posFactory); } else { return new POSModel(languageCode, seqPosModel, manifestInfoEntries, posFactory); } } POSTaggerME(POSModel model); String[] getAllPosTags(); String[] tag(String[] sentence); String[] tag(String[] sentence, Object[] additionaContext); String[][] tag(int numTaggings, String[] sentence); Sequence[] topKSequences(String[] sentence); Sequence[] topKSequences(String[] sentence, Object[] additionaContext); void probs(double[] probs); double[] probs(); String[] getOrderedTags(List<String> words, List<String> tags, int index); String[] getOrderedTags(List<String> words, List<String> tags, int index,double[] tprobs); static POSModel train(String languageCode,
      ObjectStream<POSSample> samples, TrainingParameters trainParams,
      POSTaggerFactory posFactory); static Dictionary buildNGramDictionary(ObjectStream<POSSample> samples, int cutoff); static void populatePOSDictionary(ObjectStream<POSSample> samples,
      MutableTagDictionary dict, int cutoff); }
POSTaggerME implements POSTagger { public static POSModel train(String languageCode, ObjectStream<POSSample> samples, TrainingParameters trainParams, POSTaggerFactory posFactory) throws IOException { int beamSize = trainParams.getIntParameter(BeamSearch.BEAM_SIZE_PARAMETER, POSTaggerME.DEFAULT_BEAM_SIZE); POSContextGenerator contextGenerator = posFactory.getPOSContextGenerator(); Map<String, String> manifestInfoEntries = new HashMap<>(); TrainerType trainerType = TrainerFactory.getTrainerType(trainParams); MaxentModel posModel = null; SequenceClassificationModel<String> seqPosModel = null; if (TrainerType.EVENT_MODEL_TRAINER.equals(trainerType)) { ObjectStream<Event> es = new POSSampleEventStream(samples, contextGenerator); EventTrainer trainer = TrainerFactory.getEventTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(es); } else if (TrainerType.EVENT_MODEL_SEQUENCE_TRAINER.equals(trainerType)) { POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); EventModelSequenceTrainer trainer = TrainerFactory.getEventModelSequenceTrainer(trainParams, manifestInfoEntries); posModel = trainer.train(ss); } else if (TrainerType.SEQUENCE_TRAINER.equals(trainerType)) { SequenceTrainer trainer = TrainerFactory.getSequenceModelTrainer( trainParams, manifestInfoEntries); POSSampleSequenceStream ss = new POSSampleSequenceStream(samples, contextGenerator); seqPosModel = trainer.train(ss); } else { throw new IllegalArgumentException("Trainer type is not supported: " + trainerType); } if (posModel != null) { return new POSModel(languageCode, posModel, beamSize, manifestInfoEntries, posFactory); } else { return new POSModel(languageCode, seqPosModel, manifestInfoEntries, posFactory); } } POSTaggerME(POSModel model); String[] getAllPosTags(); String[] tag(String[] sentence); String[] tag(String[] sentence, Object[] additionaContext); String[][] tag(int numTaggings, String[] sentence); Sequence[] topKSequences(String[] sentence); Sequence[] topKSequences(String[] sentence, Object[] additionaContext); void probs(double[] probs); double[] probs(); String[] getOrderedTags(List<String> words, List<String> tags, int index); String[] getOrderedTags(List<String> words, List<String> tags, int index,double[] tprobs); static POSModel train(String languageCode,
      ObjectStream<POSSample> samples, TrainingParameters trainParams,
      POSTaggerFactory posFactory); static Dictionary buildNGramDictionary(ObjectStream<POSSample> samples, int cutoff); static void populatePOSDictionary(ObjectStream<POSSample> samples,
      MutableTagDictionary dict, int cutoff); static final int DEFAULT_BEAM_SIZE; }