VowpalWabbitFormat implements InputFormat { @Override public void parse(LineBytesBuffer line, SparseItem item, long lineNo) { item.clear(); feature.clear(); namespace.clear(); number.clear(); item.id(Long.toString(lineNo)); State state = State.BEFORE_LABEL; for (int i = 0; i < line.size(); i++) { byte b = line.get(i); char c = b > 0 ? (char) b : NAME_CHAR; switch (state) { case BEFORE_LABEL: if (NUMBER_MATCHER.matches(c)) { number.append(b); state = State.LABEL; } break; case LABEL: if (NUMBER_MATCHER.matches(c)) { number.append(b); } else { item.label(Util.doublesEqual(1., Double.parseDouble(number.toAsciiString())) ? 1. : 0.); number.clear(); state = State.AFTER_LABEL; } break; case AFTER_LABEL: if (PIPE_MATCHER.matches(c)) { state = State.BEFORE_NAMESPACE; } break; case BEFORE_NAMESPACE: if (NAME_MATCHER.matches(c)) { namespace.append(b); state = State.NAMESPACE; } break; case NAMESPACE: if (NAME_MATCHER.matches(c)) { namespace.append(b); } else { state = State.BEFORE_FEATURE; } break; case BEFORE_FEATURE: if (NAME_MATCHER.matches(c)) { feature.append(b); state = State.FEATURE; } else if (PIPE_MATCHER.matches(c)) { state = State.BEFORE_NAMESPACE; namespace.clear(); } break; case FEATURE: if (NAME_MATCHER.matches(c)) { feature.append(b); } else if (PIPE_MATCHER.matches(c)) { addCategoricalIndex(item); state = State.BEFORE_NAMESPACE; namespace.clear(); } else if (COLON_MATCHER.matches(c)) { state = State.FEATURE_VALUE; } else { addCategoricalIndex(item); state = State.BEFORE_FEATURE; } break; case FEATURE_VALUE: if (NUMBER_MATCHER.matches(c)) { number.append(b); } else if (PIPE_MATCHER.matches(c)) { addNumericalIndex(item, Double.parseDouble(number.toAsciiString())); state = State.BEFORE_NAMESPACE; namespace.clear(); } else { addNumericalIndex(item, Double.parseDouble(number.toAsciiString())); state = State.BEFORE_FEATURE; } } } if (state == State.FEATURE) { addCategoricalIndex(item); } if (state == State.FEATURE_VALUE) { addNumericalIndex(item, Double.parseDouble(number.toAsciiString())); } featuresProcessor.finalize(item); }  @Override void parse(LineBytesBuffer line, SparseItem item, long lineNo); @Inject VowpalWabbitFormat featruresProcessor(FeaturesProcessor featuresProcessor);  }
@Test public void testParse() throws Exception { String line1 = "-1 |C1 1005 |banner_pos 1 |site_id 0a742914 |site_domain 510bd839 |site_category f028772b " + "|app_id ecad2386 |app_domain 7801e8d9 |app_category 07d7df22 |device_id a99f214a |device_ip 0cff710f " + "|device_model 76dc4769 |device_type 1 |device_conn_type 0 |C14 8330 |C15 320 |C16 50 |C17 761 |C18 3 " + "|C19 175 |C20 100075"; InputFormat format = new VowpalWabbitFormat() .featruresProcessor(new FeaturesProcessor()); LineBytesBuffer line = new LineBytesBuffer(line1); SparseItem item = new SparseItem(); format.parse(line, item, 0); assertEquals(0., item.label(), 0.000001); assertEquals(20, item.indexes().stream().distinct().count()); }