@Test public void testGetPartitions() { Map<String, Long> comparisons1 = new HashMap<String, Long>() { { IntStream.range(0, 100000).forEach(f -> put("f" + f, 100L)); } }; BucketizedBloomCheckPartitioner p = new BucketizedBloomCheckPartitioner(1000, comparisons1, 10); IntStream.range(0, 100000).forEach(f -> { int partition = p.getPartition(Pair.of("f" + f, "value")); assertTrue(0 <= partition && partition <= 1000, "partition is out of range: " + partition); }); }
@Override public int getPartition(Object key) { final Pair<String, String> parts = (Pair<String, String>) key; final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight()); final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft()); final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size()); assert idx >= 0; return candidatePartitions.get(idx); }
BucketizedBloomCheckPartitioner extends Partitioner { @Override public int getPartition(Object key) { final Pair<String, String> parts = (Pair<String, String>) key; final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight()); final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft()); final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size()); assert idx >= 0; return candidatePartitions.get(idx); } }
BucketizedBloomCheckPartitioner extends Partitioner { @Override public int getPartition(Object key) { final Pair<String, String> parts = (Pair<String, String>) key; final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight()); final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft()); final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size()); assert idx >= 0; return candidatePartitions.get(idx); } BucketizedBloomCheckPartitioner(int targetPartitions, Map<String, Long> fileGroupToComparisons,
      int keysPerBucket); }
BucketizedBloomCheckPartitioner extends Partitioner { @Override public int getPartition(Object key) { final Pair<String, String> parts = (Pair<String, String>) key; final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight()); final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft()); final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size()); assert idx >= 0; return candidatePartitions.get(idx); } BucketizedBloomCheckPartitioner(int targetPartitions, Map<String, Long> fileGroupToComparisons,
      int keysPerBucket); @Override int numPartitions(); @Override int getPartition(Object key); }
BucketizedBloomCheckPartitioner extends Partitioner { @Override public int getPartition(Object key) { final Pair<String, String> parts = (Pair<String, String>) key; final long hashOfKey = NumericUtils.getMessageDigestHash("MD5", parts.getRight()); final List<Integer> candidatePartitions = fileGroupToPartitions.get(parts.getLeft()); final int idx = (int) Math.floorMod((int) hashOfKey, candidatePartitions.size()); assert idx >= 0; return candidatePartitions.get(idx); } BucketizedBloomCheckPartitioner(int targetPartitions, Map<String, Long> fileGroupToComparisons,
      int keysPerBucket); @Override int numPartitions(); @Override int getPartition(Object key);  }