BlockRecordReader extends RecordReader { public byte[][] readTextRecord() throws IOException { if (isLastLine) { return null; } currentLine = readLine(); isLastLine = startPos + readBytes > blockInfo.getStartPos() + blockInfo.getLength(); return splitLine(currentLine); } BlockRecordReader(BlockInfo blockInfo, String fd, String rd, boolean ignoreHeader); byte[][] readTextRecord(); String getCurrentLine(); void close(); static int indexOf(byte[] src, int offset, int length, byte[] search);  }
@Test public void testReadMultiBlocksNormalCase() throws Exception { { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 0L, 8L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); byte[][] secondLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(secondLine[0])); assertEquals("not equal", "bbb", new String(secondLine[1])); BlockInfo blockInfo2 = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 8L, 7L); reader = new BlockRecordReader(blockInfo2, ",", "\n", false); secondLine = reader.readTextRecord(); assertNull("not null", secondLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 0L, 9L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); byte[][] secondLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(secondLine[0])); assertEquals("not equal", "bbb", new String(secondLine[1])); BlockInfo blockInfo2 = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 9L, 6L); reader = new BlockRecordReader(blockInfo2, ",", "\n", false); secondLine = reader.readTextRecord(); assertNull("not null", secondLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 0L, 7L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); firstLine = reader.readTextRecord(); assertNull("not null", firstLine); BlockInfo blockInfo2 = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 7L, 8L); reader = new BlockRecordReader(blockInfo2, ",", "\n", false); byte[][] secondLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(secondLine[0])); assertEquals("not equal", "bbb", new String(secondLine[1])); secondLine = reader.readTextRecord(); assertNull("not null", secondLine); } }
@Test public void testReadMultiBlocksAbnormalCase() throws Exception { { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 100L, 8L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", false); byte[][] secondLine = reader.readTextRecord(); assertNull("not null", secondLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 0L, 100L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); byte[][] secondLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(secondLine[0])); assertEquals("not equal", "bbb", new String(secondLine[1])); secondLine = reader.readTextRecord(); assertNull("not null", secondLine); } }
@Test public void testReadMultiBlocksWithMoreCharSplit() throws Exception { { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split.txt"), 0L, 16L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); byte[][] nextLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(nextLine[0])); assertEquals("not equal", "bbb", new String(nextLine[1])); byte[][] tLine = reader.readTextRecord(); assertNull("not null", tLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split.txt"), 16L, 16L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "333", new String(firstLine[0])); assertEquals("not equal", "ccc", new String(firstLine[1])); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split.txt"), 0L, 21L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0])); assertEquals("not equal", "abc", new String(firstLine[1])); byte[][] nextLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(nextLine[0])); assertEquals("not equal", "bbb", new String(nextLine[1])); byte[][] tLine = reader.readTextRecord(); assertNull("not null", tLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split.txt"), 21L, 16L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "333", new String(firstLine[0])); assertEquals("not equal", "ccc", new String(firstLine[1])); } }
@Test public void testReadMultiBlocksWithBom() throws Exception { { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split_chinese_bom.txt"), 0L, 27L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123", new String(firstLine[0], "utf8")); assertEquals("not equal", "ab测试c", new String(firstLine[1], "utf8")); byte[][] nextLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(nextLine[0], "utf8")); assertEquals("not equal", "bb你好b", new String(nextLine[1], "utf8")); byte[][] tLine = reader.readTextRecord(); assertNull("not null", tLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split_chinese_bom.txt"), 27L, 54L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", ",,,", false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "33啊啊3", new String(firstLine[0], "utf8")); assertEquals("not equal", "ccc", new String(firstLine[1], "utf8")); } }
@Test public void testReadIgnoreHeader() throws Exception { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/one_char_split.txt"), 0L, 15L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",", "\n", true); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "234", new String(firstLine[0])); assertEquals("not equal", "bbb", new String(firstLine[1])); byte[][] secondLine = reader.readTextRecord(); assertNull("not null", secondLine); }
@Test public void testRDConflictWithBom() throws Exception { { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split_chinese_bom.txt"), 0L, 54L); BlockRecordReader reader = new BlockRecordReader(blockInfo, ",,,,", Character.toString((char)0xFF), false); byte[][] firstLine = reader.readTextRecord(); assertEquals("not equal", "123||ab测试c,,,234||bb你好b,,,33啊啊3||ccc,,,", new String(firstLine[0], "utf8")); byte[][] tLine = reader.readTextRecord(); assertNull("not null", tLine); } { BlockInfo blockInfo = new BlockInfo(1L, new File("src/test/resources/file/reader/more_char_split_chinese_bom.txt"), 0L, 54L); BlockRecordReader reader = new BlockRecordReader(blockInfo, "||", Character.toString((char)0xFF), true); byte[][] firstLine = reader.readTextRecord(); assertNull("not null", firstLine); } }