@Test public void testTrain() throws Exception { final Tokenizer tokenizer = new Tokenizer(); final List<String> trainDataORG = readLinesFromFile(FILENAME_TRAIN_DATA_ORG); final List<String> trainDataTOK = readLinesFromFile(FILENAME_TRAIN_DATA_TOK); final InstanceList trainData = tokenizer.makeTrainingData(trainDataORG, trainDataTOK); final Pipe trainPipe = trainData.getPipe(); tokenizer.train(trainData, trainPipe); tokenizer.writeModel(FILENAME_TRAIN_MODEL_OUTPUT); assertTrue(new File(FILENAME_TRAIN_MODEL_OUTPUT + ".gz").isFile()); }
void train(final InstanceList instList, final Pipe myPipe) { final long s1 = System.currentTimeMillis(); model = new CRF(myPipe, null); model.addStatesForLabelsConnectedAsIn(instList); final CRFTrainerByLabelLikelihood crfTrainer = new CRFTrainerByLabelLikelihood( model); final boolean b = crfTrainer.train(instList); LOGGER.info("Tokenizer training: model converged: " + b); final long s2 = System.currentTimeMillis(); model.getInputPipe().getDataAlphabet().stopGrowth(); trained = true; LOGGER.debug("train() - training time: " + ((s2 - s1) / 1000) + " sec"); }
Tokenizer { void train(final InstanceList instList, final Pipe myPipe) { final long s1 = System.currentTimeMillis(); model = new CRF(myPipe, null); model.addStatesForLabelsConnectedAsIn(instList); final CRFTrainerByLabelLikelihood crfTrainer = new CRFTrainerByLabelLikelihood( model); final boolean b = crfTrainer.train(instList); LOGGER.info("Tokenizer training: model converged: " + b); final long s2 = System.currentTimeMillis(); model.getInputPipe().getDataAlphabet().stopGrowth(); trained = true; LOGGER.debug("train() - training time: " + ((s2 - s1) / 1000) + " sec"); } }
Tokenizer { void train(final InstanceList instList, final Pipe myPipe) { final long s1 = System.currentTimeMillis(); model = new CRF(myPipe, null); model.addStatesForLabelsConnectedAsIn(instList); final CRFTrainerByLabelLikelihood crfTrainer = new CRFTrainerByLabelLikelihood( model); final boolean b = crfTrainer.train(instList); LOGGER.info("Tokenizer training: model converged: " + b); final long s2 = System.currentTimeMillis(); model.getInputPipe().getDataAlphabet().stopGrowth(); trained = true; LOGGER.debug("train() - training time: " + ((s2 - s1) / 1000) + " sec"); } Tokenizer(); }
Tokenizer { void train(final InstanceList instList, final Pipe myPipe) { final long s1 = System.currentTimeMillis(); model = new CRF(myPipe, null); model.addStatesForLabelsConnectedAsIn(instList); final CRFTrainerByLabelLikelihood crfTrainer = new CRFTrainerByLabelLikelihood( model); final boolean b = crfTrainer.train(instList); LOGGER.info("Tokenizer training: model converged: " + b); final long s2 = System.currentTimeMillis(); model.getInputPipe().getDataAlphabet().stopGrowth(); trained = true; LOGGER.debug("train() - training time: " + ((s2 - s1) / 1000) + " sec"); } Tokenizer(); CRF getModel(); ArrayList<Unit> predict(final String sentence); void readModel(final File file); void readModel(InputStream is); }
Tokenizer { void train(final InstanceList instList, final Pipe myPipe) { final long s1 = System.currentTimeMillis(); model = new CRF(myPipe, null); model.addStatesForLabelsConnectedAsIn(instList); final CRFTrainerByLabelLikelihood crfTrainer = new CRFTrainerByLabelLikelihood( model); final boolean b = crfTrainer.train(instList); LOGGER.info("Tokenizer training: model converged: " + b); final long s2 = System.currentTimeMillis(); model.getInputPipe().getDataAlphabet().stopGrowth(); trained = true; LOGGER.debug("train() - training time: " + ((s2 - s1) / 1000) + " sec"); } Tokenizer(); CRF getModel(); ArrayList<Unit> predict(final String sentence); void readModel(final File file); void readModel(InputStream is);  }