@Test public void testSplitStoreFile() throws IOException { Path dir = util.getDataTestDirOnTestFS("testSplitHFile"); FileSystem fs = util.getTestFileSystem(); Path testIn = new Path(dir, "testhfile"); HColumnDescriptor familyDesc = new HColumnDescriptor(FAMILY); HFileTestUtil.createHFile(util.getConfiguration(), fs, testIn, FAMILY, QUALIFIER, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), 1000); Path bottomOut = new Path(dir, "bottom.out"); Path topOut = new Path(dir, "top.out"); LoadIncrementalHFiles.splitStoreFile( util.getConfiguration(), testIn, familyDesc, Bytes.toBytes("ggg"), bottomOut, topOut); int rowCount = verifyHFile(bottomOut); rowCount += verifyHFile(topOut); assertEquals(1000, rowCount); }
protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item, final HTable table, byte[] startKey, byte[] splitKey) throws IOException { final Path hfilePath = item.hfilePath; final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); String uniqueName = getUniqueName(table.getName()); HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); Path botOut = new Path(tmpDir, uniqueName + ".bottom"); Path topOut = new Path(tmpDir, uniqueName + ".top"); splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); FileSystem fs = tmpDir.getFileSystem(getConf()); fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx")); fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx")); List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2); lqis.add(new LoadQueueItem(item.family, botOut)); lqis.add(new LoadQueueItem(item.family, topOut)); LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); return lqis; }
LoadIncrementalHFiles extends Configured implements Tool { protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item, final HTable table, byte[] startKey, byte[] splitKey) throws IOException { final Path hfilePath = item.hfilePath; final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); String uniqueName = getUniqueName(table.getName()); HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); Path botOut = new Path(tmpDir, uniqueName + ".bottom"); Path topOut = new Path(tmpDir, uniqueName + ".top"); splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); FileSystem fs = tmpDir.getFileSystem(getConf()); fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx")); fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx")); List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2); lqis.add(new LoadQueueItem(item.family, botOut)); lqis.add(new LoadQueueItem(item.family, topOut)); LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); return lqis; } }
LoadIncrementalHFiles extends Configured implements Tool { protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item, final HTable table, byte[] startKey, byte[] splitKey) throws IOException { final Path hfilePath = item.hfilePath; final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); String uniqueName = getUniqueName(table.getName()); HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); Path botOut = new Path(tmpDir, uniqueName + ".bottom"); Path topOut = new Path(tmpDir, uniqueName + ".top"); splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); FileSystem fs = tmpDir.getFileSystem(getConf()); fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx")); fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx")); List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2); lqis.add(new LoadQueueItem(item.family, botOut)); lqis.add(new LoadQueueItem(item.family, topOut)); LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); return lqis; } LoadIncrementalHFiles(Configuration conf); }
LoadIncrementalHFiles extends Configured implements Tool { protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item, final HTable table, byte[] startKey, byte[] splitKey) throws IOException { final Path hfilePath = item.hfilePath; final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); String uniqueName = getUniqueName(table.getName()); HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); Path botOut = new Path(tmpDir, uniqueName + ".bottom"); Path topOut = new Path(tmpDir, uniqueName + ".top"); splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); FileSystem fs = tmpDir.getFileSystem(getConf()); fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx")); fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx")); List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2); lqis.add(new LoadQueueItem(item.family, botOut)); lqis.add(new LoadQueueItem(item.family, topOut)); LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); return lqis; } LoadIncrementalHFiles(Configuration conf); @SuppressWarnings("deprecation") void doBulkLoad(Path hfofDir, final HTable table); static byte[][] inferBoundaries(TreeMap<byte[], Integer> bdryMap); @Override int run(String[] args); static void main(String[] args); }
LoadIncrementalHFiles extends Configured implements Tool { protected List<LoadQueueItem> splitStoreFile(final LoadQueueItem item, final HTable table, byte[] startKey, byte[] splitKey) throws IOException { final Path hfilePath = item.hfilePath; final Path tmpDir = new Path(item.hfilePath.getParent(), "_tmp"); LOG.info("HFile at " + hfilePath + " no longer fits inside a single " + "region. Splitting..."); String uniqueName = getUniqueName(table.getName()); HColumnDescriptor familyDesc = table.getTableDescriptor().getFamily(item.family); Path botOut = new Path(tmpDir, uniqueName + ".bottom"); Path topOut = new Path(tmpDir, uniqueName + ".top"); splitStoreFile(getConf(), hfilePath, familyDesc, splitKey, botOut, topOut); FileSystem fs = tmpDir.getFileSystem(getConf()); fs.setPermission(tmpDir, FsPermission.valueOf("-rwxrwxrwx")); fs.setPermission(botOut, FsPermission.valueOf("-rwxrwxrwx")); List<LoadQueueItem> lqis = new ArrayList<LoadQueueItem>(2); lqis.add(new LoadQueueItem(item.family, botOut)); lqis.add(new LoadQueueItem(item.family, topOut)); LOG.info("Successfully split into new HFiles " + botOut + " and " + topOut); return lqis; } LoadIncrementalHFiles(Configuration conf); @SuppressWarnings("deprecation") void doBulkLoad(Path hfofDir, final HTable table); static byte[][] inferBoundaries(TreeMap<byte[], Integer> bdryMap); @Override int run(String[] args); static void main(String[] args); static final String NAME; static final String MAX_FILES_PER_REGION_PER_FAMILY; }