@Test(expected = InsufficientTrainingDataException.class) public void testInsufficientData() throws IOException { InputStreamFactory trainDataIn = new ResourceAsStreamFactory( TokenizerModel.class, "/opennlp/tools/tokenize/token-insufficient.train"); ObjectStream<TokenSample> samples = new TokenSampleStream( new PlainTextByLineStream(trainDataIn, StandardCharsets.UTF_8)); TrainingParameters mlParams = new TrainingParameters(); mlParams.put(TrainingParameters.ITERATIONS_PARAM, 100); mlParams.put(TrainingParameters.CUTOFF_PARAM, 5); TokenizerME.train(samples, TokenizerFactory.create(null, "eng", null, true, null), mlParams); }
public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); ObjectStream<Event> eventStream = new TokSpanEventStream(samples, factory.isUseAlphaNumericOptmization(), factory.getAlphaNumericPattern(), factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel maxentModel = trainer.train(eventStream); return new TokenizerModel(maxentModel, manifestInfoEntries, factory); }
TokenizerME extends AbstractTokenizer { public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); ObjectStream<Event> eventStream = new TokSpanEventStream(samples, factory.isUseAlphaNumericOptmization(), factory.getAlphaNumericPattern(), factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel maxentModel = trainer.train(eventStream); return new TokenizerModel(maxentModel, manifestInfoEntries, factory); } }
TokenizerME extends AbstractTokenizer { public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); ObjectStream<Event> eventStream = new TokSpanEventStream(samples, factory.isUseAlphaNumericOptmization(), factory.getAlphaNumericPattern(), factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel maxentModel = trainer.train(eventStream); return new TokenizerModel(maxentModel, manifestInfoEntries, factory); } TokenizerME(TokenizerModel model);  TokenizerME(TokenizerModel model, Factory factory); }
TokenizerME extends AbstractTokenizer { public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); ObjectStream<Event> eventStream = new TokSpanEventStream(samples, factory.isUseAlphaNumericOptmization(), factory.getAlphaNumericPattern(), factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel maxentModel = trainer.train(eventStream); return new TokenizerModel(maxentModel, manifestInfoEntries, factory); } TokenizerME(TokenizerModel model);  TokenizerME(TokenizerModel model, Factory factory); double[] getTokenProbabilities(); Span[] tokenizePos(String d); static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory,
      TrainingParameters mlParams); boolean useAlphaNumericOptimization(); }
TokenizerME extends AbstractTokenizer { public static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory, TrainingParameters mlParams) throws IOException { Map<String, String> manifestInfoEntries = new HashMap<>(); ObjectStream<Event> eventStream = new TokSpanEventStream(samples, factory.isUseAlphaNumericOptmization(), factory.getAlphaNumericPattern(), factory.getContextGenerator()); EventTrainer trainer = TrainerFactory.getEventTrainer( mlParams, manifestInfoEntries); MaxentModel maxentModel = trainer.train(eventStream); return new TokenizerModel(maxentModel, manifestInfoEntries, factory); } TokenizerME(TokenizerModel model);  TokenizerME(TokenizerModel model, Factory factory); double[] getTokenProbabilities(); Span[] tokenizePos(String d); static TokenizerModel train(ObjectStream<TokenSample> samples, TokenizerFactory factory,
      TrainingParameters mlParams); boolean useAlphaNumericOptimization(); static final String SPLIT; static final String NO_SPLIT; @Deprecated
  static final Pattern alphaNumeric; }