@Test public void testLexicon() throws Exception { Lexicon lexicon = new Lexicon(true, true); List<String> set = populateLexicon(lexicon, maxLen, N); logger.info(set.size() + " unique features"); Lexicon prunedLexicon = lexicon.getPrunedLexicon(threshold); for (String s : set) { if (prunedLexicon.contains(s)) { int originalId = lexicon.lookupId(s); assertEquals(prunedLexicon.lookupId(s), originalId); assertEquals(true, lexicon.featureCounts.get(originalId) > threshold); } } Map<String, Float> features = new HashMap<>(); for (int i = 0; i < 10; i++) features.put(set.get(random.nextInt(set.size())), 1f); Pair<int[], float[]> feats = lexicon.getFeatureVector(features); int[] f1 = lexicon.pruneFeaturesByCount(feats.getFirst(), feats.getSecond(), threshold) .getFirst(); int[] f2 = prunedLexicon.getFeatureVector(features).getFirst(); logger.info(Arrays.toString(feats.getFirst())); for (int id : feats.getFirst()) { logger.info(id + "\t" + lexicon.lookupName(id) + "\t" + lexicon.featureCounts.get(id)); } logger.info(Arrays.toString(f1)); logger.info(Arrays.toString(f2)); assertEquals(true, Arrays.equals(f1, f2)); }
public Lexicon(boolean hasBias, boolean storeStrings) { feature2Id = new TIntIntHashMap(); nextFeatureId = 0; if (hasBias) this.previewFeature(GLOBAL_BIAS); if (storeStrings) featureNames = new ArrayList<>(); else featureNames = null; featureCounts = new TIntIntHashMap(); }
Lexicon { public Lexicon(boolean hasBias, boolean storeStrings) { feature2Id = new TIntIntHashMap(); nextFeatureId = 0; if (hasBias) this.previewFeature(GLOBAL_BIAS); if (storeStrings) featureNames = new ArrayList<>(); else featureNames = null; featureCounts = new TIntIntHashMap(); } }
Lexicon { public Lexicon(boolean hasBias, boolean storeStrings) { feature2Id = new TIntIntHashMap(); nextFeatureId = 0; if (hasBias) this.previewFeature(GLOBAL_BIAS); if (storeStrings) featureNames = new ArrayList<>(); else featureNames = null; featureCounts = new TIntIntHashMap(); } Lexicon(boolean hasBias, boolean storeStrings);  Lexicon(InputStream in);  Lexicon(InputStream in, boolean loadStrings); }
Lexicon { public Lexicon(boolean hasBias, boolean storeStrings) { feature2Id = new TIntIntHashMap(); nextFeatureId = 0; if (hasBias) this.previewFeature(GLOBAL_BIAS); if (storeStrings) featureNames = new ArrayList<>(); else featureNames = null; featureCounts = new TIntIntHashMap(); } Lexicon(boolean hasBias, boolean storeStrings);  Lexicon(InputStream in);  Lexicon(InputStream in, boolean loadStrings); int lookupId(String featureName); String lookupName(int id); synchronized void countFeature(int featureId); synchronized int getFeatureId(String feature); synchronized void previewFeature(String f); boolean contains(String f); int size(); Pair<int[], float[]> getFeatureVector(Map<String, Float> featureMap); Pair<int[], float[]> pruneFeaturesByCount(int[] idx, float[] fs, int threshold); void writeIntegerToFeatureStringFormat(PrintStream out); void save(String file); Lexicon getPrunedLexicon(final int threshold); }
Lexicon { public Lexicon(boolean hasBias, boolean storeStrings) { feature2Id = new TIntIntHashMap(); nextFeatureId = 0; if (hasBias) this.previewFeature(GLOBAL_BIAS); if (storeStrings) featureNames = new ArrayList<>(); else featureNames = null; featureCounts = new TIntIntHashMap(); } Lexicon(boolean hasBias, boolean storeStrings);  Lexicon(InputStream in);  Lexicon(InputStream in, boolean loadStrings); int lookupId(String featureName); String lookupName(int id); synchronized void countFeature(int featureId); synchronized int getFeatureId(String feature); synchronized void previewFeature(String f); boolean contains(String f); int size(); Pair<int[], float[]> getFeatureVector(Map<String, Float> featureMap); Pair<int[], float[]> pruneFeaturesByCount(int[] idx, float[] fs, int threshold); void writeIntegerToFeatureStringFormat(PrintStream out); void save(String file); Lexicon getPrunedLexicon(final int threshold); static final String GLOBAL_BIAS; final TIntIntHashMap featureCounts; }