HadoopArchives implements Tool { public Configuration getConf() { return this.conf; } HadoopArchives(Configuration conf); void setConf(Configuration conf); Configuration getConf(); int run(String[] args); static void main(String[] args); static final int VERSION; }
@Test public void testReadFileContent() throws Exception { fileList.add(createFile(inputPath, fs, "c c")); final Path sub1 = new Path(inputPath, "sub 1"); fs.mkdirs(sub1); fileList.add(createFile(inputPath, fs, sub1.getName(), "file x y z")); fileList.add(createFile(inputPath, fs, sub1.getName(), "file")); fileList.add(createFile(inputPath, fs, sub1.getName(), "x")); fileList.add(createFile(inputPath, fs, sub1.getName(), "y")); fileList.add(createFile(inputPath, fs, sub1.getName(), "z")); final Path sub2 = new Path(inputPath, "sub 1 with suffix"); fs.mkdirs(sub2); fileList.add(createFile(inputPath, fs, sub2.getName(), "z")); final byte[] binContent = prepareBin(); fileList.add(createFile(inputPath, fs, binContent, sub2.getName(), "bin")); fileList.add(createFile(inputPath, fs, new byte[0], sub2.getName(), "zero-length")); final String fullHarPathStr = makeArchive(); final HarFileSystem harFileSystem = new HarFileSystem(fs); try { final URI harUri = new URI(fullHarPathStr); harFileSystem.initialize(harUri, fs.getConf()); int readFileCount = 0; for (final String pathStr0 : fileList) { final Path path = new Path(fullHarPathStr + Path.SEPARATOR + pathStr0); final String baseName = path.getName(); final FileStatus status = harFileSystem.getFileStatus(path); if (status.isFile()) { final byte[] actualContentSimple = readAllSimple( harFileSystem.open(path), true); final byte[] actualContentBuffer = readAllWithBuffer( harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentBuffer); final byte[] actualContentFully = readAllWithReadFully( actualContentSimple.length, harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentFully); final byte[] actualContentSeek = readAllWithSeek( actualContentSimple.length, harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentSeek); final byte[] actualContentRead4 = readAllWithRead4(harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentRead4); final byte[] actualContentSkip = readAllWithSkip( actualContentSimple.length, harFileSystem.open(path), harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentSkip); if ("bin".equals(baseName)) { assertArrayEquals(binContent, actualContentSimple); } else if ("zero-length".equals(baseName)) { assertEquals(0, actualContentSimple.length); } else { String actual = new String(actualContentSimple, "UTF-8"); assertEquals(baseName, actual); } readFileCount++; } } assertThat(fileList.size()).isEqualTo(readFileCount); } finally { harFileSystem.close(); } }
@Test public void testReadFileContent() throws Exception { fileList.add(createFile(inputPath, fs, "c c")); final Path sub1 = new Path(inputPath, "sub 1"); fs.mkdirs(sub1); fileList.add(createFile(inputPath, fs, sub1.getName(), "file x y z")); fileList.add(createFile(inputPath, fs, sub1.getName(), "file")); fileList.add(createFile(inputPath, fs, sub1.getName(), "x")); fileList.add(createFile(inputPath, fs, sub1.getName(), "y")); fileList.add(createFile(inputPath, fs, sub1.getName(), "z")); final Path sub2 = new Path(inputPath, "sub 1 with suffix"); fs.mkdirs(sub2); fileList.add(createFile(inputPath, fs, sub2.getName(), "z")); final byte[] binContent = prepareBin(); fileList.add(createFile(inputPath, fs, binContent, sub2.getName(), "bin")); fileList.add(createFile(inputPath, fs, new byte[0], sub2.getName(), "zero-length")); final String fullHarPathStr = makeArchive(); final HarFileSystem harFileSystem = new HarFileSystem(fs); try { final URI harUri = new URI(fullHarPathStr); harFileSystem.initialize(harUri, fs.getConf()); int readFileCount = 0; for (final String pathStr0 : fileList) { final Path path = new Path(fullHarPathStr + Path.SEPARATOR + pathStr0); final String baseName = path.getName(); final FileStatus status = harFileSystem.getFileStatus(path); if (status.isFile()) { final byte[] actualContentSimple = readAllSimple( harFileSystem.open(path), true); final byte[] actualContentBuffer = readAllWithBuffer( harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentBuffer); final byte[] actualContentFully = readAllWithReadFully( actualContentSimple.length, harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentFully); final byte[] actualContentSeek = readAllWithSeek( actualContentSimple.length, harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentSeek); final byte[] actualContentRead4 = readAllWithRead4(harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentRead4); final byte[] actualContentSkip = readAllWithSkip( actualContentSimple.length, harFileSystem.open(path), harFileSystem.open(path), true); assertArrayEquals(actualContentSimple, actualContentSkip); if ("bin".equals(baseName)) { assertArrayEquals(binContent, actualContentSimple); } else if ("zero-length".equals(baseName)) { assertEquals(0, actualContentSimple.length); } else { String actual = new String(actualContentSimple, "UTF-8"); assertEquals(baseName, actual); } readFileCount++; } } assertEquals(fileList.size(), readFileCount); } finally { harFileSystem.close(); } }
@Test public void testCopyToLocal() throws Exception { final String fullHarPathStr = makeArchive(); final String tmpDir = System.getProperty("test.build.data","build/test/data") + "/work-dir/har-fs-tmp"; final Path tmpPath = new Path(tmpDir); final LocalFileSystem localFs = FileSystem.getLocal(new Configuration()); localFs.delete(tmpPath, true); localFs.mkdirs(tmpPath); assertTrue(localFs.exists(tmpPath)); final HarFileSystem harFileSystem = new HarFileSystem(fs); try { final URI harUri = new URI(fullHarPathStr); harFileSystem.initialize(harUri, fs.getConf()); final Path sourcePath = new Path(fullHarPathStr + Path.SEPARATOR + "a"); final Path targetPath = new Path(tmpPath, "straus"); harFileSystem.copyToLocalFile(false, sourcePath, targetPath); FileStatus straus = localFs.getFileStatus(targetPath); assertEquals(1, straus.getLen()); } finally { harFileSystem.close(); localFs.delete(tmpPath, true); } }