@Test public void hugeFileTest() throws IOException { GZIPInputStream gzipInputStream = new GZIPInputStream(this.getClass() .getClassLoader().getResourceAsStream("ner.train.gz")); BufferedReader testReader = new BufferedReader(new InputStreamReader( gzipInputStream)); GZIPInputStream gzipInputStreamForTestSubject = new GZIPInputStream( this.getClass().getClassLoader() .getResourceAsStream("ner.train.gz")); PlainTextExamplesMessageBodyReader toTest = new PlainTextExamplesMessageBodyReader(); MediaType mediaType = MediaType.TEXT_PLAIN_TYPE; Iterable<Example> theIterableOfExamples = toTest.readFrom(null, null, null, mediaType, null, gzipInputStreamForTestSubject); int numExamples = 0; boolean dumpExamples = false; for (Example example : theIterableOfExamples) { String expectedExample = testReader.readLine(); Assert.assertEquals(expectedExample, example.getVWStringRepresentation()); numExamples++; if (dumpExamples && numExamples % 21 == 0) { LOGGER.debug("expected example: {}", expectedExample); LOGGER.debug("read example : {}", example); LOGGER.debug(""); } } Assert.assertTrue(testReader.readLine() == null); }
@Override public ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { if (LOGGER.isDebugEnabled()) if (httpHeaders != null && httpHeaders.size() > 0) { LOGGER.debug("Rec'd HTTP headers: "); for (Entry<String, List<String>> entry : httpHeaders.entrySet()) { LOGGER.debug("{}:{}", entry.getKey(), StringUtils.join(entry.getValue(), ',')); } } Charset charset = ReaderWriter.getCharset(mediaType); LOGGER.debug("Reading examples using charset: {}", charset.displayName()); StringExampleIterator theIterator = new StringExampleIterator(entityStream, charset); return new ExamplesIterableImpl(Integer.MAX_VALUE, null, theIterator); }
PlainTextExamplesMessageBodyReader implements MessageBodyReader<ExamplesIterable> { @Override public ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { if (LOGGER.isDebugEnabled()) if (httpHeaders != null && httpHeaders.size() > 0) { LOGGER.debug("Rec'd HTTP headers: "); for (Entry<String, List<String>> entry : httpHeaders.entrySet()) { LOGGER.debug("{}:{}", entry.getKey(), StringUtils.join(entry.getValue(), ',')); } } Charset charset = ReaderWriter.getCharset(mediaType); LOGGER.debug("Reading examples using charset: {}", charset.displayName()); StringExampleIterator theIterator = new StringExampleIterator(entityStream, charset); return new ExamplesIterableImpl(Integer.MAX_VALUE, null, theIterator); } }
PlainTextExamplesMessageBodyReader implements MessageBodyReader<ExamplesIterable> { @Override public ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { if (LOGGER.isDebugEnabled()) if (httpHeaders != null && httpHeaders.size() > 0) { LOGGER.debug("Rec'd HTTP headers: "); for (Entry<String, List<String>> entry : httpHeaders.entrySet()) { LOGGER.debug("{}:{}", entry.getKey(), StringUtils.join(entry.getValue(), ',')); } } Charset charset = ReaderWriter.getCharset(mediaType); LOGGER.debug("Reading examples using charset: {}", charset.displayName()); StringExampleIterator theIterator = new StringExampleIterator(entityStream, charset); return new ExamplesIterableImpl(Integer.MAX_VALUE, null, theIterator); } PlainTextExamplesMessageBodyReader(); }
PlainTextExamplesMessageBodyReader implements MessageBodyReader<ExamplesIterable> { @Override public ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { if (LOGGER.isDebugEnabled()) if (httpHeaders != null && httpHeaders.size() > 0) { LOGGER.debug("Rec'd HTTP headers: "); for (Entry<String, List<String>> entry : httpHeaders.entrySet()) { LOGGER.debug("{}:{}", entry.getKey(), StringUtils.join(entry.getValue(), ',')); } } Charset charset = ReaderWriter.getCharset(mediaType); LOGGER.debug("Reading examples using charset: {}", charset.displayName()); StringExampleIterator theIterator = new StringExampleIterator(entityStream, charset); return new ExamplesIterableImpl(Integer.MAX_VALUE, null, theIterator); } PlainTextExamplesMessageBodyReader(); @Override boolean isReadable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType); @Override ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream); }
PlainTextExamplesMessageBodyReader implements MessageBodyReader<ExamplesIterable> { @Override public ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream) throws IOException, WebApplicationException { if (LOGGER.isDebugEnabled()) if (httpHeaders != null && httpHeaders.size() > 0) { LOGGER.debug("Rec'd HTTP headers: "); for (Entry<String, List<String>> entry : httpHeaders.entrySet()) { LOGGER.debug("{}:{}", entry.getKey(), StringUtils.join(entry.getValue(), ',')); } } Charset charset = ReaderWriter.getCharset(mediaType); LOGGER.debug("Reading examples using charset: {}", charset.displayName()); StringExampleIterator theIterator = new StringExampleIterator(entityStream, charset); return new ExamplesIterableImpl(Integer.MAX_VALUE, null, theIterator); } PlainTextExamplesMessageBodyReader(); @Override boolean isReadable(Class<?> type, Type genericType, Annotation[] annotations, MediaType mediaType); @Override ExamplesIterable readFrom(Class<ExamplesIterable> type, Type genericType, Annotation[] annotations, MediaType mediaType, MultivaluedMap<String, String> httpHeaders, InputStream entityStream);  }