@Test public void testCustomDelimiter() throws Exception { Delimiter="</entity>"; String CurrentBufferTailToken= "</entity><entity><id>Gelesh</"; String NextBufferHeadToken= "id><name>Omathil</name></entity>"; String Expected = (CurrentBufferTailToken+NextBufferHeadToken) .replace(Delimiter, ""); String TestPartOfInput = CurrentBufferTailToken+NextBufferHeadToken; int BufferSize=64 * 1024; int numberOfCharToFillTheBuffer = BufferSize - CurrentBufferTailToken.length(); StringBuilder fillerString=new StringBuilder(); for (int i=0; i<numberOfCharToFillTheBuffer; i++) { fillerString.append('a'); } TestData = fillerString + TestPartOfInput; lineReader = new LineReader( new ByteArrayInputStream(TestData.getBytes()), Delimiter.getBytes()); line = new Text(); lineReader.readLine(line); Assert.assertEquals(fillerString.toString(), line.toString()); lineReader.readLine(line); Assert.assertEquals(Expected, line.toString()); Delimiter = "record"; StringBuilder TestStringBuilder = new StringBuilder(); TestStringBuilder.append(Delimiter + "Kerala "); TestStringBuilder.append(Delimiter + "Bangalore"); TestStringBuilder.append(Delimiter + " North Korea"); TestStringBuilder.append(Delimiter + Delimiter+ "Guantanamo"); TestStringBuilder.append(Delimiter + "ecord" + "recor" + "core"); TestData=TestStringBuilder.toString(); lineReader = new LineReader( new ByteArrayInputStream(TestData.getBytes()), Delimiter.getBytes()); lineReader.readLine(line); Assert.assertEquals("", line.toString()); lineReader.readLine(line); Assert.assertEquals("Kerala ", line.toString()); lineReader.readLine(line); Assert.assertEquals("Bangalore", line.toString()); lineReader.readLine(line); Assert.assertEquals(" North Korea", line.toString()); lineReader.readLine(line); Assert.assertEquals("", line.toString()); lineReader.readLine(line); Assert.assertEquals("Guantanamo", line.toString()); lineReader.readLine(line); Assert.assertEquals(("ecord"+"recor"+"core"), line.toString()); TestData = "aaaabccc"; Delimiter = "aaab"; lineReader = new LineReader( new ByteArrayInputStream(TestData.getBytes()), Delimiter.getBytes()); lineReader.readLine(line); Assert.assertEquals("a", line.toString()); lineReader.readLine(line); Assert.assertEquals("ccc", line.toString()); }
public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { if (this.recordDelimiterBytes != null) { return readCustomLine(str, maxLineLength, maxBytesToConsume); } else { return readDefaultLine(str, maxLineLength, maxBytesToConsume); } }
LineReader implements Closeable { public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { if (this.recordDelimiterBytes != null) { return readCustomLine(str, maxLineLength, maxBytesToConsume); } else { return readDefaultLine(str, maxLineLength, maxBytesToConsume); } } }
LineReader implements Closeable { public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { if (this.recordDelimiterBytes != null) { return readCustomLine(str, maxLineLength, maxBytesToConsume); } else { return readDefaultLine(str, maxLineLength, maxBytesToConsume); } } LineReader(InputStream in);  LineReader(InputStream in, int bufferSize);  LineReader(InputStream in, Configuration conf);  LineReader(InputStream in, byte[] recordDelimiterBytes);  LineReader(InputStream in, int bufferSize,
      byte[] recordDelimiterBytes);  LineReader(InputStream in, Configuration conf,
      byte[] recordDelimiterBytes);  LineReader(MallocedMappedDataBuffer sgxBuffer, byte[] recordDelimiterBytes); }
LineReader implements Closeable { public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { if (this.recordDelimiterBytes != null) { return readCustomLine(str, maxLineLength, maxBytesToConsume); } else { return readDefaultLine(str, maxLineLength, maxBytesToConsume); } } LineReader(InputStream in);  LineReader(InputStream in, int bufferSize);  LineReader(InputStream in, Configuration conf);  LineReader(InputStream in, byte[] recordDelimiterBytes);  LineReader(InputStream in, int bufferSize,
      byte[] recordDelimiterBytes);  LineReader(InputStream in, Configuration conf,
      byte[] recordDelimiterBytes);  LineReader(MallocedMappedDataBuffer sgxBuffer, byte[] recordDelimiterBytes); void close(); int readLine(Text str, int maxLineLength,
                      int maxBytesToConsume); int readLine(Text str, int maxLineLength); int readLine(Text str); int getBufferSize(); long getBufferOffset(); byte[] getRecordDelimiterBytes(); int fillBuffer(boolean inDelimiter); }
LineReader implements Closeable { public int readLine(Text str, int maxLineLength, int maxBytesToConsume) throws IOException { if (this.recordDelimiterBytes != null) { return readCustomLine(str, maxLineLength, maxBytesToConsume); } else { return readDefaultLine(str, maxLineLength, maxBytesToConsume); } } LineReader(InputStream in);  LineReader(InputStream in, int bufferSize);  LineReader(InputStream in, Configuration conf);  LineReader(InputStream in, byte[] recordDelimiterBytes);  LineReader(InputStream in, int bufferSize,
      byte[] recordDelimiterBytes);  LineReader(InputStream in, Configuration conf,
      byte[] recordDelimiterBytes);  LineReader(MallocedMappedDataBuffer sgxBuffer, byte[] recordDelimiterBytes); void close(); int readLine(Text str, int maxLineLength,
                      int maxBytesToConsume); int readLine(Text str, int maxLineLength); int readLine(Text str); int getBufferSize(); long getBufferOffset(); byte[] getRecordDelimiterBytes(); int fillBuffer(boolean inDelimiter);  }