@Test public void testgetHDFSBlocksDistribution() throws Exception { HBaseTestingUtility htu = new HBaseTestingUtility(); htu.getConfiguration().setInt("dfs.replication", 2); MiniHBaseCluster cluster = null; String dataNodeHosts[] = new String[] { "host1", "host2", "host3" }; int regionServersCount = 3; try { cluster = htu.startMiniCluster(1, regionServersCount, dataNodeHosts); byte[][] families = { fam1, fam2 }; HTable ht = htu.createTable(Bytes.toBytes(this.getName()), families); byte row[] = Bytes.toBytes("row1"); byte col[] = Bytes.toBytes("col1"); Put put = new Put(row); put.add(fam1, col, 1, Bytes.toBytes("test1")); put.add(fam2, col, 1, Bytes.toBytes("test2")); ht.put(put); HRegion firstRegion = htu.getHBaseCluster().getRegions(TableName.valueOf(this.getName())) .get(0); firstRegion.flushcache(); HDFSBlocksDistribution blocksDistribution1 = firstRegion.getHDFSBlocksDistribution(); long uniqueBlocksWeight1 = blocksDistribution1.getUniqueBlocksTotalWeight(); StringBuilder sb = new StringBuilder(); for (String host: blocksDistribution1.getTopHosts()) { if (sb.length() > 0) sb.append(", "); sb.append(host); sb.append("="); sb.append(blocksDistribution1.getWeight(host)); } String topHost = blocksDistribution1.getTopHosts().get(0); long topHostWeight = blocksDistribution1.getWeight(topHost); String msg = "uniqueBlocksWeight=" + uniqueBlocksWeight1 + ", topHostWeight=" + topHostWeight + ", topHost=" + topHost + "; " + sb.toString(); LOG.info(msg); assertTrue(msg, uniqueBlocksWeight1 == topHostWeight); HDFSBlocksDistribution blocksDistribution2 = HRegion.computeHDFSBlocksDistribution( htu.getConfiguration(), firstRegion.getTableDesc(), firstRegion.getRegionInfo()); long uniqueBlocksWeight2 = blocksDistribution2.getUniqueBlocksTotalWeight(); assertTrue(uniqueBlocksWeight1 == uniqueBlocksWeight2); ht.close(); } finally { if (cluster != null) { htu.shutdownMiniCluster(); } } }
public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); synchronized (this.stores) { for (Store store : this.stores.values()) { for (StoreFile sf : store.getStorefiles()) { HDFSBlocksDistribution storeFileBlocksDistribution = sf.getHDFSBlockDistribution(); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } } } return hdfsBlocksDistribution; }
HRegion implements HeapSize { public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); synchronized (this.stores) { for (Store store : this.stores.values()) { for (StoreFile sf : store.getStorefiles()) { HDFSBlocksDistribution storeFileBlocksDistribution = sf.getHDFSBlockDistribution(); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } } } return hdfsBlocksDistribution; } }
HRegion implements HeapSize { public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); synchronized (this.stores) { for (Store store : this.stores.values()) { for (StoreFile sf : store.getStorefiles()) { HDFSBlocksDistribution storeFileBlocksDistribution = sf.getHDFSBlockDistribution(); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } } } return hdfsBlocksDistribution; } @Deprecated  HRegion(final Path tableDir, final HLog log, final FileSystem fs,
      final Configuration confParam, final HRegionInfo regionInfo,
      final HTableDescriptor htd, final RegionServerServices rsServices);  HRegion(final HRegionFileSystem fs, final HLog log, final Configuration confParam,
      final HTableDescriptor htd, final RegionServerServices rsServices); }
HRegion implements HeapSize { public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); synchronized (this.stores) { for (Store store : this.stores.values()) { for (StoreFile sf : store.getStorefiles()) { HDFSBlocksDistribution storeFileBlocksDistribution = sf.getHDFSBlockDistribution(); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } } } return hdfsBlocksDistribution; } @Deprecated  HRegion(final Path tableDir, final HLog log, final FileSystem fs,
      final Configuration confParam, final HRegionInfo regionInfo,
      final HTableDescriptor htd, final RegionServerServices rsServices);  HRegion(final HRegionFileSystem fs, final HLog log, final Configuration confParam,
      final HTableDescriptor htd, final RegionServerServices rsServices); long getSmallestReadPoint(); @Deprecated long initialize(); boolean hasReferences(); HDFSBlocksDistribution getHDFSBlocksDistribution(); static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
      final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo); static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
      final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo,  Path tablePath); AtomicLong getMemstoreSize(); long addAndGetGlobalMemstoreSize(long memStoreSize); HRegionInfo getRegionInfo(); boolean isClosed(); boolean isClosing(); void setRecovering(boolean newState); boolean isRecovering(); boolean isAvailable(); boolean isSplittable(); boolean isMergeable(); boolean areWritesEnabled(); MultiVersionConsistencyControl getMVCC(); long getReadpoint(IsolationLevel isolationLevel); boolean isLoadingCfsOnDemandDefault(); Map<byte[], List<StoreFile>> close(); Map<byte[], List<StoreFile>> close(final boolean abort); void waitForFlushesAndCompactions(); byte [] getStartKey(); byte [] getEndKey(); long getRegionId(); byte [] getRegionName(); String getRegionNameAsString(); HTableDescriptor getTableDesc(); HLog getLog(); FileSystem getFilesystem(); HRegionFileSystem getRegionFileSystem(); long getLastFlushTime(); long getLargestHStoreSize(); KeyValue.KVComparator getComparator(); void compactStores(final boolean majorCompaction); void compactStores(); boolean compact(CompactionContext compaction, Store store); FlushResult flushcache(); Result getClosestRowBefore(final byte [] row, final byte [] family); RegionScanner getScanner(Scan scan); void delete(Delete delete); void put(Put put); OperationStatus[] batchMutate(
      Mutation[] mutations, long nonceGroup, long nonce); OperationStatus[] batchMutate(Mutation[] mutations); OperationStatus[] batchReplay(HLogSplitter.MutationReplay[] mutations); boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
      CompareOp compareOp, ByteArrayComparable comparator, Mutation w,
      boolean writeToWAL); void addRegionToSnapshot(SnapshotDescription desc,
      ForeignExceptionSnare exnSnare); Store getStore(final byte[] column); Map<byte[], Store> getStores(); List<String> getStoreFileList(final byte [][] columns); RowLock getRowLock(byte[] row, boolean waitForLock); RowLock getRowLock(byte[] row); void releaseRowLocks(List<RowLock> rowLocks); boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
                                boolean assignSeqId); boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, boolean assignSeqId,
      BulkLoadListener bulkLoadListener); @Override boolean equals(Object o); @Override int hashCode(); @Override String toString(); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
      final Configuration conf, final HTableDescriptor hTableDescriptor); static void closeHRegion(final HRegion r); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize, final boolean ignoreHLog); static HRegion createHRegion(final HRegionInfo info, final Path rootDir, final Path tableDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize, final boolean ignoreHLog); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog); static HRegion openHRegion(final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal,
      final Configuration conf); static HRegion openHRegion(final HRegionInfo info,
    final HTableDescriptor htd, final HLog wal, final Configuration conf,
    final RegionServerServices rsServices,
    final CancelableProgressable reporter); static HRegion openHRegion(Path rootDir, final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal, final Configuration conf); static HRegion openHRegion(final Path rootDir, final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal, final Configuration conf,
      final RegionServerServices rsServices,
      final CancelableProgressable reporter); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal,
      final RegionServerServices rsServices, final CancelableProgressable reporter); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final Path tableDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal,
      final RegionServerServices rsServices, final CancelableProgressable reporter); static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter); static void addRegionToMETA(final HRegion meta, final HRegion r); @Deprecated static Path getRegionDir(final Path tabledir, final String name); @Deprecated static Path getRegionDir(final Path rootdir, final HRegionInfo info); static boolean rowIsInRange(HRegionInfo info, final byte [] row); static HRegion mergeAdjacent(final HRegion srcA, final HRegion srcB); static HRegion merge(final HRegion a, final HRegion b); Result get(final Get get); void mutateRow(RowMutations rm); void mutateRowsWithLocks(Collection<Mutation> mutations,
      Collection<byte[]> rowsToLock); void mutateRowsWithLocks(Collection<Mutation> mutations,
      Collection<byte[]> rowsToLock, long nonceGroup, long nonce); void processRowsWithLocks(RowProcessor<?,?> processor, long nonceGroup, long nonce); void processRowsWithLocks(RowProcessor<?,?> processor, long timeout,
      long nonceGroup, long nonce); Result append(Append append); Result append(Append append, long nonceGroup, long nonce); Result increment(Increment increment); Result increment(Increment increment, long nonceGroup, long nonce); @Override long heapSize(); boolean registerService(Service instance); Message execService(RpcController controller, CoprocessorServiceCall call); byte[] checkSplit(); int getCompactPriority(); RegionCoprocessorHost getCoprocessorHost(); void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost); void startRegionOperation(); void closeRegionOperation(); void closeRegionOperation(Operation operation); static void main(String[] args); long getOpenSeqNum(); Map<byte[], Long> getMaxStoreSeqIdForLogReplay(); CompactionState getCompactionState(); void reportCompactionRequestStart(boolean isMajor); void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted); @VisibleForTesting AtomicLong getSequenceId(); }
HRegion implements HeapSize { public HDFSBlocksDistribution getHDFSBlocksDistribution() { HDFSBlocksDistribution hdfsBlocksDistribution = new HDFSBlocksDistribution(); synchronized (this.stores) { for (Store store : this.stores.values()) { for (StoreFile sf : store.getStorefiles()) { HDFSBlocksDistribution storeFileBlocksDistribution = sf.getHDFSBlockDistribution(); hdfsBlocksDistribution.add(storeFileBlocksDistribution); } } } return hdfsBlocksDistribution; } @Deprecated  HRegion(final Path tableDir, final HLog log, final FileSystem fs,
      final Configuration confParam, final HRegionInfo regionInfo,
      final HTableDescriptor htd, final RegionServerServices rsServices);  HRegion(final HRegionFileSystem fs, final HLog log, final Configuration confParam,
      final HTableDescriptor htd, final RegionServerServices rsServices); long getSmallestReadPoint(); @Deprecated long initialize(); boolean hasReferences(); HDFSBlocksDistribution getHDFSBlocksDistribution(); static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
      final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo); static HDFSBlocksDistribution computeHDFSBlocksDistribution(final Configuration conf,
      final HTableDescriptor tableDescriptor, final HRegionInfo regionInfo,  Path tablePath); AtomicLong getMemstoreSize(); long addAndGetGlobalMemstoreSize(long memStoreSize); HRegionInfo getRegionInfo(); boolean isClosed(); boolean isClosing(); void setRecovering(boolean newState); boolean isRecovering(); boolean isAvailable(); boolean isSplittable(); boolean isMergeable(); boolean areWritesEnabled(); MultiVersionConsistencyControl getMVCC(); long getReadpoint(IsolationLevel isolationLevel); boolean isLoadingCfsOnDemandDefault(); Map<byte[], List<StoreFile>> close(); Map<byte[], List<StoreFile>> close(final boolean abort); void waitForFlushesAndCompactions(); byte [] getStartKey(); byte [] getEndKey(); long getRegionId(); byte [] getRegionName(); String getRegionNameAsString(); HTableDescriptor getTableDesc(); HLog getLog(); FileSystem getFilesystem(); HRegionFileSystem getRegionFileSystem(); long getLastFlushTime(); long getLargestHStoreSize(); KeyValue.KVComparator getComparator(); void compactStores(final boolean majorCompaction); void compactStores(); boolean compact(CompactionContext compaction, Store store); FlushResult flushcache(); Result getClosestRowBefore(final byte [] row, final byte [] family); RegionScanner getScanner(Scan scan); void delete(Delete delete); void put(Put put); OperationStatus[] batchMutate(
      Mutation[] mutations, long nonceGroup, long nonce); OperationStatus[] batchMutate(Mutation[] mutations); OperationStatus[] batchReplay(HLogSplitter.MutationReplay[] mutations); boolean checkAndMutate(byte [] row, byte [] family, byte [] qualifier,
      CompareOp compareOp, ByteArrayComparable comparator, Mutation w,
      boolean writeToWAL); void addRegionToSnapshot(SnapshotDescription desc,
      ForeignExceptionSnare exnSnare); Store getStore(final byte[] column); Map<byte[], Store> getStores(); List<String> getStoreFileList(final byte [][] columns); RowLock getRowLock(byte[] row, boolean waitForLock); RowLock getRowLock(byte[] row); void releaseRowLocks(List<RowLock> rowLocks); boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths,
                                boolean assignSeqId); boolean bulkLoadHFiles(List<Pair<byte[], String>> familyPaths, boolean assignSeqId,
      BulkLoadListener bulkLoadListener); @Override boolean equals(Object o); @Override int hashCode(); @Override String toString(); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
      final Configuration conf, final HTableDescriptor hTableDescriptor); static void closeHRegion(final HRegion r); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize, final boolean ignoreHLog); static HRegion createHRegion(final HRegionInfo info, final Path rootDir, final Path tableDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog,
                                      final boolean initialize, final boolean ignoreHLog); static HRegion createHRegion(final HRegionInfo info, final Path rootDir,
                                      final Configuration conf,
                                      final HTableDescriptor hTableDescriptor,
                                      final HLog hlog); static HRegion openHRegion(final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal,
      final Configuration conf); static HRegion openHRegion(final HRegionInfo info,
    final HTableDescriptor htd, final HLog wal, final Configuration conf,
    final RegionServerServices rsServices,
    final CancelableProgressable reporter); static HRegion openHRegion(Path rootDir, final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal, final Configuration conf); static HRegion openHRegion(final Path rootDir, final HRegionInfo info,
      final HTableDescriptor htd, final HLog wal, final Configuration conf,
      final RegionServerServices rsServices,
      final CancelableProgressable reporter); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal,
      final RegionServerServices rsServices, final CancelableProgressable reporter); static HRegion openHRegion(final Configuration conf, final FileSystem fs,
      final Path rootDir, final Path tableDir, final HRegionInfo info, final HTableDescriptor htd, final HLog wal,
      final RegionServerServices rsServices, final CancelableProgressable reporter); static HRegion openHRegion(final HRegion other, final CancelableProgressable reporter); static void addRegionToMETA(final HRegion meta, final HRegion r); @Deprecated static Path getRegionDir(final Path tabledir, final String name); @Deprecated static Path getRegionDir(final Path rootdir, final HRegionInfo info); static boolean rowIsInRange(HRegionInfo info, final byte [] row); static HRegion mergeAdjacent(final HRegion srcA, final HRegion srcB); static HRegion merge(final HRegion a, final HRegion b); Result get(final Get get); void mutateRow(RowMutations rm); void mutateRowsWithLocks(Collection<Mutation> mutations,
      Collection<byte[]> rowsToLock); void mutateRowsWithLocks(Collection<Mutation> mutations,
      Collection<byte[]> rowsToLock, long nonceGroup, long nonce); void processRowsWithLocks(RowProcessor<?,?> processor, long nonceGroup, long nonce); void processRowsWithLocks(RowProcessor<?,?> processor, long timeout,
      long nonceGroup, long nonce); Result append(Append append); Result append(Append append, long nonceGroup, long nonce); Result increment(Increment increment); Result increment(Increment increment, long nonceGroup, long nonce); @Override long heapSize(); boolean registerService(Service instance); Message execService(RpcController controller, CoprocessorServiceCall call); byte[] checkSplit(); int getCompactPriority(); RegionCoprocessorHost getCoprocessorHost(); void setCoprocessorHost(final RegionCoprocessorHost coprocessorHost); void startRegionOperation(); void closeRegionOperation(); void closeRegionOperation(Operation operation); static void main(String[] args); long getOpenSeqNum(); Map<byte[], Long> getMaxStoreSeqIdForLogReplay(); CompactionState getCompactionState(); void reportCompactionRequestStart(boolean isMajor); void reportCompactionRequestEnd(boolean isMajor, int numFiles, long filesSizeCompacted); @VisibleForTesting AtomicLong getSequenceId(); static final Log LOG; static final String LOAD_CFS_ON_DEMAND_CONFIG_KEY; final AtomicLong memstoreSize; static final String MEMSTORE_PERIODIC_FLUSH_INTERVAL; static final int DEFAULT_CACHE_FLUSH_INTERVAL; static final String MEMSTORE_FLUSH_PER_CHANGES; static final long DEFAULT_FLUSH_PER_CHANGES; static final long MAX_FLUSH_PER_CHANGES; static final long FIXED_OVERHEAD; static final long DEEP_OVERHEAD; }