@Test public void testSampleLG() throws IOException { int num = 5000; for (int sampleBufferSize : new int[] {1 << 10, 1 << 20}) { RFile.setSampleBufferSize(sampleBufferSize); for (int modulus : new int[] {19, 103, 1019}) { List<Entry<Key,Value>> sampleDataLG1 = new ArrayList<>(); List<Entry<Key,Value>> sampleDataLG2 = new ArrayList<>(); ConfigurationCopy sampleConf = new ConfigurationCopy(conf == null ? DefaultConfiguration.getInstance() : conf); sampleConf.set(Property.TABLE_SAMPLER, RowSampler.class.getName()); sampleConf.set(Property.TABLE_SAMPLER_OPTS + "hasher", "murmur3_32"); sampleConf.set(Property.TABLE_SAMPLER_OPTS + "modulus", modulus + ""); Sampler sampler = SamplerFactory .newSampler(SamplerConfigurationImpl.newSamplerConfig(sampleConf), sampleConf); TestRFile trf = new TestRFile(sampleConf); trf.openWriter(false, 1000); trf.writer.startNewLocalityGroup("meta-lg", newColFamByteSequence("metaA", "metaB")); for (int r = 0; r < num; r++) { String row = String.format("r%06d", r); Key k1 = new Key(row, "metaA", "q9", 7); Key k2 = new Key(row, "metaB", "q8", 7); Key k3 = new Key(row, "metaB", "qA", 7); Value v1 = new Value("" + r); Value v2 = new Value("" + r * 93); Value v3 = new Value("" + r * 113); if (sampler.accept(k1)) { sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1)); sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k2, v2)); sampleDataLG1.add(new AbstractMap.SimpleImmutableEntry<>(k3, v3)); } trf.writer.append(k1, v1); trf.writer.append(k2, v2); trf.writer.append(k3, v3); } trf.writer.startDefaultLocalityGroup(); for (int r = 0; r < num; r++) { String row = String.format("r%06d", r); Key k1 = new Key(row, "dataA", "q9", 7); Value v1 = new Value("" + r); if (sampler.accept(k1)) { sampleDataLG2.add(new AbstractMap.SimpleImmutableEntry<>(k1, v1)); } trf.writer.append(k1, v1); } trf.closeWriter(); assertTrue(!sampleDataLG1.isEmpty()); assertTrue(!sampleDataLG2.isEmpty()); trf.openReader(false); FileSKVIterator sample = trf.reader.getSample(SamplerConfigurationImpl.newSamplerConfig(sampleConf)); checkSample(sample, sampleDataLG1, newColFamByteSequence("metaA", "metaB"), true); checkSample(sample, sampleDataLG1, newColFamByteSequence("metaA"), true); checkSample(sample, sampleDataLG1, newColFamByteSequence("metaB"), true); checkSample(sample, sampleDataLG1, newColFamByteSequence("dataA"), false); checkSample(sample, sampleDataLG2, newColFamByteSequence("metaA", "metaB"), false); checkSample(sample, sampleDataLG2, newColFamByteSequence("dataA"), true); ArrayList<Entry<Key,Value>> allSampleData = new ArrayList<>(); allSampleData.addAll(sampleDataLG1); allSampleData.addAll(sampleDataLG2); allSampleData.sort(Comparator.comparing(Entry::getKey)); checkSample(sample, allSampleData, newColFamByteSequence("dataA", "metaA"), true); checkSample(sample, allSampleData, EMPTY_COL_FAMS, false); trf.closeReader(); } } }
@VisibleForTesting public static void setSampleBufferSize(int bufferSize) { sampleBufferSize = bufferSize; }
RFile { @VisibleForTesting public static void setSampleBufferSize(int bufferSize) { sampleBufferSize = bufferSize; } }
RFile { @VisibleForTesting public static void setSampleBufferSize(int bufferSize) { sampleBufferSize = bufferSize; } private  RFile(); }
RFile { @VisibleForTesting public static void setSampleBufferSize(int bufferSize) { sampleBufferSize = bufferSize; } private  RFile(); @VisibleForTesting static void setSampleBufferSize(int bufferSize); }
RFile { @VisibleForTesting public static void setSampleBufferSize(int bufferSize) { sampleBufferSize = bufferSize; } private  RFile(); @VisibleForTesting static void setSampleBufferSize(int bufferSize); static final String EXTENSION; }