@Test public void testBigDataSet() { ArrayList<String> strs = new ArrayList<>(); Iterator<String> it = new RandomStrings(10 * 10000).iterator(); int totalSize = 0; final StringBytesConverter converter = new StringBytesConverter(); while (it.hasNext()) { String str = it.next(); byte[] data = converter.convertToBytes(str); if (data != null) { totalSize += data.length; } strs.add(str); } Collections.sort(strs); int baseId = 20; int maxTreeSize = totalSize / 10; System.out.println("data size:" + totalSize / 1024 + "KB max tree size:" + maxTreeSize / 1024 + "KB"); Map<String, Integer> idMap = rightIdMap(baseId, strs); TrieDictionaryForestBuilder<String> builder = newDictBuilder(strs, baseId, maxTreeSize); TrieDictionaryForest<String> dict = builder.build(); System.out.println("tree num:" + dict.getTrees().size()); for (Map.Entry<String, Integer> entry : idMap.entrySet()) { assertEquals(0, dict.getIdFromValue(entry.getKey()) - entry.getValue()); assertEquals(entry.getKey(), dict.getValueFromId(entry.getValue())); } }
List<TrieDictionary<T>> getTrees() { return Collections.unmodifiableList(this.trees); }
TrieDictionaryForest extends CacheDictionary<T> { List<TrieDictionary<T>> getTrees() { return Collections.unmodifiableList(this.trees); } }
TrieDictionaryForest extends CacheDictionary<T> { List<TrieDictionary<T>> getTrees() { return Collections.unmodifiableList(this.trees); } TrieDictionaryForest();  TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, //
                                ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId); }
TrieDictionaryForest extends CacheDictionary<T> { List<TrieDictionary<T>> getTrees() { return Collections.unmodifiableList(this.trees); } TrieDictionaryForest();  TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, //
                                ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId); @Override int getMinId(); @Override int getMaxId(); @Override int getSizeOfId(); @Override int getSizeOfValue(); @Override void dump(PrintStream out); @Override void write(DataOutput out); @Override void readFields(DataInput in); @Override boolean contains(Dictionary other); @Override int hashCode(); @Override boolean equals(Object obj); }
TrieDictionaryForest extends CacheDictionary<T> { List<TrieDictionary<T>> getTrees() { return Collections.unmodifiableList(this.trees); } TrieDictionaryForest();  TrieDictionaryForest(ArrayList<TrieDictionary<T>> trees, ArrayList<ByteArray> valueDivide, //
                                ArrayList<Integer> accuOffset, BytesConverter<T> bytesConverter, int baseId); @Override int getMinId(); @Override int getMaxId(); @Override int getSizeOfId(); @Override int getSizeOfValue(); @Override void dump(PrintStream out); @Override void write(DataOutput out); @Override void readFields(DataInput in); @Override boolean contains(Dictionary other); @Override int hashCode(); @Override boolean equals(Object obj);  }