@Test public void testRegionBoundariesCheck() throws Exception { HBaseFsck hbck = doFsck(conf, false); assertNoErrors(hbck); try { hbck.checkRegionBoundaries(); } catch (IllegalArgumentException e) { if (e.getMessage().endsWith("not a valid DFS filename.")) { fail("Table directory path is not valid." + e.getMessage()); } } }
public void checkRegionBoundaries() { try { ByteArrayComparator comparator = new ByteArrayComparator(); List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false); final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation(); Path hbaseRoot = FSUtils.getRootDir(getConf()); for (HRegionInfo regionInfo : regions) { Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); Path path = new Path(tableDir, regionInfo.getEncodedName()); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); byte[] storeFirstKey = null; byte[] storeLastKey = null; for (FileStatus file : files) { String fileName = file.getPath().toString(); fileName = fileName.substring(fileName.lastIndexOf("/") + 1); if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { FileStatus[] storeFiles = fs.listStatus(file.getPath()); for (FileStatus storeFile : storeFiles) { HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( getConf()), getConf()); if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, reader.getFirstKey()) > 0))) { storeFirstKey = reader.getFirstKey(); } if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, reader.getLastKey())) < 0)) { storeLastKey = reader.getLastKey(); } reader.close(); } } } currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); if (currentRegionBoundariesInformation.metaFirstKey.length == 0) currentRegionBoundariesInformation.metaFirstKey = null; if (currentRegionBoundariesInformation.metaLastKey.length == 0) currentRegionBoundariesInformation.metaLastKey = null; boolean valid = true; if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0; } if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0; } if (!valid) { errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable())); LOG.warn("Region's boundaries not alligned between stores and META for:"); LOG.warn(currentRegionBoundariesInformation); } } } catch (IOException e) { LOG.error(e); } }
HBaseFsck extends Configured { public void checkRegionBoundaries() { try { ByteArrayComparator comparator = new ByteArrayComparator(); List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false); final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation(); Path hbaseRoot = FSUtils.getRootDir(getConf()); for (HRegionInfo regionInfo : regions) { Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); Path path = new Path(tableDir, regionInfo.getEncodedName()); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); byte[] storeFirstKey = null; byte[] storeLastKey = null; for (FileStatus file : files) { String fileName = file.getPath().toString(); fileName = fileName.substring(fileName.lastIndexOf("/") + 1); if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { FileStatus[] storeFiles = fs.listStatus(file.getPath()); for (FileStatus storeFile : storeFiles) { HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( getConf()), getConf()); if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, reader.getFirstKey()) > 0))) { storeFirstKey = reader.getFirstKey(); } if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, reader.getLastKey())) < 0)) { storeLastKey = reader.getLastKey(); } reader.close(); } } } currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); if (currentRegionBoundariesInformation.metaFirstKey.length == 0) currentRegionBoundariesInformation.metaFirstKey = null; if (currentRegionBoundariesInformation.metaLastKey.length == 0) currentRegionBoundariesInformation.metaLastKey = null; boolean valid = true; if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0; } if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0; } if (!valid) { errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable())); LOG.warn("Region's boundaries not alligned between stores and META for:"); LOG.warn(currentRegionBoundariesInformation); } } } catch (IOException e) { LOG.error(e); } } }
HBaseFsck extends Configured { public void checkRegionBoundaries() { try { ByteArrayComparator comparator = new ByteArrayComparator(); List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false); final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation(); Path hbaseRoot = FSUtils.getRootDir(getConf()); for (HRegionInfo regionInfo : regions) { Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); Path path = new Path(tableDir, regionInfo.getEncodedName()); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); byte[] storeFirstKey = null; byte[] storeLastKey = null; for (FileStatus file : files) { String fileName = file.getPath().toString(); fileName = fileName.substring(fileName.lastIndexOf("/") + 1); if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { FileStatus[] storeFiles = fs.listStatus(file.getPath()); for (FileStatus storeFile : storeFiles) { HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( getConf()), getConf()); if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, reader.getFirstKey()) > 0))) { storeFirstKey = reader.getFirstKey(); } if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, reader.getLastKey())) < 0)) { storeLastKey = reader.getLastKey(); } reader.close(); } } } currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); if (currentRegionBoundariesInformation.metaFirstKey.length == 0) currentRegionBoundariesInformation.metaFirstKey = null; if (currentRegionBoundariesInformation.metaLastKey.length == 0) currentRegionBoundariesInformation.metaLastKey = null; boolean valid = true; if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0; } if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0; } if (!valid) { errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable())); LOG.warn("Region's boundaries not alligned between stores and META for:"); LOG.warn(currentRegionBoundariesInformation); } } } catch (IOException e) { LOG.error(e); } } HBaseFsck(Configuration conf);  HBaseFsck(Configuration conf, ExecutorService exec); }
HBaseFsck extends Configured { public void checkRegionBoundaries() { try { ByteArrayComparator comparator = new ByteArrayComparator(); List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false); final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation(); Path hbaseRoot = FSUtils.getRootDir(getConf()); for (HRegionInfo regionInfo : regions) { Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); Path path = new Path(tableDir, regionInfo.getEncodedName()); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); byte[] storeFirstKey = null; byte[] storeLastKey = null; for (FileStatus file : files) { String fileName = file.getPath().toString(); fileName = fileName.substring(fileName.lastIndexOf("/") + 1); if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { FileStatus[] storeFiles = fs.listStatus(file.getPath()); for (FileStatus storeFile : storeFiles) { HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( getConf()), getConf()); if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, reader.getFirstKey()) > 0))) { storeFirstKey = reader.getFirstKey(); } if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, reader.getLastKey())) < 0)) { storeLastKey = reader.getLastKey(); } reader.close(); } } } currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); if (currentRegionBoundariesInformation.metaFirstKey.length == 0) currentRegionBoundariesInformation.metaFirstKey = null; if (currentRegionBoundariesInformation.metaLastKey.length == 0) currentRegionBoundariesInformation.metaLastKey = null; boolean valid = true; if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0; } if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0; } if (!valid) { errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable())); LOG.warn("Region's boundaries not alligned between stores and META for:"); LOG.warn(currentRegionBoundariesInformation); } } } catch (IOException e) { LOG.error(e); } } HBaseFsck(Configuration conf);  HBaseFsck(Configuration conf, ExecutorService exec); void connect(); void offlineHdfsIntegrityRepair(); int onlineConsistencyRepair(); int onlineHbck(); static byte[] keyOnly(byte[] b); void checkRegionBoundaries(); ErrorReporter getErrors(); void fixEmptyMetaCells(); void fixOrphanTables(); boolean rebuildMeta(boolean fix); void loadHdfsRegionDirs(); int mergeRegionDirs(Path targetRegionDir, HbckInfo contained); void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions); void dumpSidelinedRegions(Map<Path, HbckInfo> regions); Multimap<byte[], HbckInfo> getOverlapGroups(
      TableName table); static void setDisplayFullReport(); void setFixTableLocks(boolean shouldFix); void setFixAssignments(boolean shouldFix); void setFixMeta(boolean shouldFix); void setFixEmptyMetaCells(boolean shouldFix); void setCheckHdfs(boolean checking); void setFixHdfsHoles(boolean shouldFix); void setFixTableOrphans(boolean shouldFix); void setFixHdfsOverlaps(boolean shouldFix); void setFixHdfsOrphans(boolean shouldFix); void setFixVersionFile(boolean shouldFix); boolean shouldFixVersionFile(); void setSidelineBigOverlaps(boolean sbo); boolean shouldSidelineBigOverlaps(); void setFixSplitParents(boolean shouldFix); void setFixReferenceFiles(boolean shouldFix); boolean shouldIgnorePreCheckPermission(); void setIgnorePreCheckPermission(boolean ignorePreCheckPermission); void setMaxMerge(int mm); int getMaxMerge(); void setMaxOverlapsToSideline(int mo); int getMaxOverlapsToSideline(); void includeTable(TableName table); void setTimeLag(long seconds); void setSidelineDir(String sidelineDir); HFileCorruptionChecker getHFilecorruptionChecker(); void setHFileCorruptionChecker(HFileCorruptionChecker hfcc); void setRetCode(int code); int getRetCode(); static void main(String[] args); HBaseFsck exec(ExecutorService exec, String[] args); static void debugLsr(Configuration conf,
      Path p); static void debugLsr(Configuration conf,
      Path p, ErrorReporter errors); }
HBaseFsck extends Configured { public void checkRegionBoundaries() { try { ByteArrayComparator comparator = new ByteArrayComparator(); List<HRegionInfo> regions = MetaScanner.listAllRegions(getConf(), false); final RegionBoundariesInformation currentRegionBoundariesInformation = new RegionBoundariesInformation(); Path hbaseRoot = FSUtils.getRootDir(getConf()); for (HRegionInfo regionInfo : regions) { Path tableDir = FSUtils.getTableDir(hbaseRoot, regionInfo.getTable()); currentRegionBoundariesInformation.regionName = regionInfo.getRegionName(); Path path = new Path(tableDir, regionInfo.getEncodedName()); FileSystem fs = path.getFileSystem(getConf()); FileStatus[] files = fs.listStatus(path); byte[] storeFirstKey = null; byte[] storeLastKey = null; for (FileStatus file : files) { String fileName = file.getPath().toString(); fileName = fileName.substring(fileName.lastIndexOf("/") + 1); if (!fileName.startsWith(".") && !fileName.endsWith("recovered.edits")) { FileStatus[] storeFiles = fs.listStatus(file.getPath()); for (FileStatus storeFile : storeFiles) { HFile.Reader reader = HFile.createReader(fs, storeFile.getPath(), new CacheConfig( getConf()), getConf()); if ((reader.getFirstKey() != null) && ((storeFirstKey == null) || (comparator.compare(storeFirstKey, reader.getFirstKey()) > 0))) { storeFirstKey = reader.getFirstKey(); } if ((reader.getLastKey() != null) && ((storeLastKey == null) || (comparator.compare(storeLastKey, reader.getLastKey())) < 0)) { storeLastKey = reader.getLastKey(); } reader.close(); } } } currentRegionBoundariesInformation.metaFirstKey = regionInfo.getStartKey(); currentRegionBoundariesInformation.metaLastKey = regionInfo.getEndKey(); currentRegionBoundariesInformation.storesFirstKey = keyOnly(storeFirstKey); currentRegionBoundariesInformation.storesLastKey = keyOnly(storeLastKey); if (currentRegionBoundariesInformation.metaFirstKey.length == 0) currentRegionBoundariesInformation.metaFirstKey = null; if (currentRegionBoundariesInformation.metaLastKey.length == 0) currentRegionBoundariesInformation.metaLastKey = null; boolean valid = true; if ((currentRegionBoundariesInformation.storesFirstKey != null) && (currentRegionBoundariesInformation.metaFirstKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesFirstKey, currentRegionBoundariesInformation.metaFirstKey) >= 0; } if ((currentRegionBoundariesInformation.storesLastKey != null) && (currentRegionBoundariesInformation.metaLastKey != null)) { valid = valid && comparator.compare(currentRegionBoundariesInformation.storesLastKey, currentRegionBoundariesInformation.metaLastKey) < 0; } if (!valid) { errors.reportError(ERROR_CODE.BOUNDARIES_ERROR, "Found issues with regions boundaries", tablesInfo.get(regionInfo.getTable())); LOG.warn("Region's boundaries not alligned between stores and META for:"); LOG.warn(currentRegionBoundariesInformation); } } } catch (IOException e) { LOG.error(e); } } HBaseFsck(Configuration conf);  HBaseFsck(Configuration conf, ExecutorService exec); void connect(); void offlineHdfsIntegrityRepair(); int onlineConsistencyRepair(); int onlineHbck(); static byte[] keyOnly(byte[] b); void checkRegionBoundaries(); ErrorReporter getErrors(); void fixEmptyMetaCells(); void fixOrphanTables(); boolean rebuildMeta(boolean fix); void loadHdfsRegionDirs(); int mergeRegionDirs(Path targetRegionDir, HbckInfo contained); void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions); void dumpSidelinedRegions(Map<Path, HbckInfo> regions); Multimap<byte[], HbckInfo> getOverlapGroups(
      TableName table); static void setDisplayFullReport(); void setFixTableLocks(boolean shouldFix); void setFixAssignments(boolean shouldFix); void setFixMeta(boolean shouldFix); void setFixEmptyMetaCells(boolean shouldFix); void setCheckHdfs(boolean checking); void setFixHdfsHoles(boolean shouldFix); void setFixTableOrphans(boolean shouldFix); void setFixHdfsOverlaps(boolean shouldFix); void setFixHdfsOrphans(boolean shouldFix); void setFixVersionFile(boolean shouldFix); boolean shouldFixVersionFile(); void setSidelineBigOverlaps(boolean sbo); boolean shouldSidelineBigOverlaps(); void setFixSplitParents(boolean shouldFix); void setFixReferenceFiles(boolean shouldFix); boolean shouldIgnorePreCheckPermission(); void setIgnorePreCheckPermission(boolean ignorePreCheckPermission); void setMaxMerge(int mm); int getMaxMerge(); void setMaxOverlapsToSideline(int mo); int getMaxOverlapsToSideline(); void includeTable(TableName table); void setTimeLag(long seconds); void setSidelineDir(String sidelineDir); HFileCorruptionChecker getHFilecorruptionChecker(); void setHFileCorruptionChecker(HFileCorruptionChecker hfcc); void setRetCode(int code); int getRetCode(); static void main(String[] args); HBaseFsck exec(ExecutorService exec, String[] args); static void debugLsr(Configuration conf,
      Path p); static void debugLsr(Configuration conf,
      Path p, ErrorReporter errors); static final long DEFAULT_TIME_LAG; static final long DEFAULT_SLEEP_BEFORE_RERUN; }