@Test @Seeds(value = { @Seed, @Seed("deadbeef") }) public void testSameResultWithTrimmedLanguages() { final Set<String> allowed = Sets.newHashSet("en", "de", "es", "fr", "it", "pl"); LangIdV3 v1 = new LangIdV3(); LangIdV3 v2 = new LangIdV3(Model.detectOnly(allowed)); for (int i = 0; i < 10000; i++) { String in = randomRealisticUnicodeOfCodepointLengthBetween(1, 300); DetectedLanguage c1 = v1.classify(in, true); DetectedLanguage c2 = v2.classify(in, true); if (allowed.contains(c1.langCode)) { assertEquals(c1.langCode, c2.langCode); } } }
public static Model detectOnly(Set<String> langCodes) { final Model source = defaultModel(); Set<String> newClasses = new LinkedHashSet<String>(Arrays.asList(source.langClasses)); newClasses.retainAll(langCodes); if (newClasses.size() < 2) { throw new IllegalArgumentException("A model must contain at least two languages."); } String [] trimmed_nb_classes = newClasses.toArray(new String[newClasses.size()]); float[] trimmed_nb_pc = new float [newClasses.size()]; float[] trimmed_nb_ptc = new float [newClasses.size() * source.numFeatures]; for (int i = 0, j = 0; i < source.numClasses; i++) { if (newClasses.contains(source.langClasses[i])) { trimmed_nb_pc[j] = source.nb_pc[i]; for (int f = 0; f < source.numFeatures; f++) { int iFrom = source.numFeatures * i + f; int iTo = source.numFeatures * j + f; trimmed_nb_ptc[iTo] = source.nb_ptc[iFrom]; } j++; } } return new Model( trimmed_nb_classes, trimmed_nb_ptc, trimmed_nb_pc, source.dsa, source.dsaOutput); }
Model { public static Model detectOnly(Set<String> langCodes) { final Model source = defaultModel(); Set<String> newClasses = new LinkedHashSet<String>(Arrays.asList(source.langClasses)); newClasses.retainAll(langCodes); if (newClasses.size() < 2) { throw new IllegalArgumentException("A model must contain at least two languages."); } String [] trimmed_nb_classes = newClasses.toArray(new String[newClasses.size()]); float[] trimmed_nb_pc = new float [newClasses.size()]; float[] trimmed_nb_ptc = new float [newClasses.size() * source.numFeatures]; for (int i = 0, j = 0; i < source.numClasses; i++) { if (newClasses.contains(source.langClasses[i])) { trimmed_nb_pc[j] = source.nb_pc[i]; for (int f = 0; f < source.numFeatures; f++) { int iFrom = source.numFeatures * i + f; int iTo = source.numFeatures * j + f; trimmed_nb_ptc[iTo] = source.nb_ptc[iFrom]; } j++; } } return new Model( trimmed_nb_classes, trimmed_nb_ptc, trimmed_nb_pc, source.dsa, source.dsaOutput); } }
Model { public static Model detectOnly(Set<String> langCodes) { final Model source = defaultModel(); Set<String> newClasses = new LinkedHashSet<String>(Arrays.asList(source.langClasses)); newClasses.retainAll(langCodes); if (newClasses.size() < 2) { throw new IllegalArgumentException("A model must contain at least two languages."); } String [] trimmed_nb_classes = newClasses.toArray(new String[newClasses.size()]); float[] trimmed_nb_pc = new float [newClasses.size()]; float[] trimmed_nb_ptc = new float [newClasses.size() * source.numFeatures]; for (int i = 0, j = 0; i < source.numClasses; i++) { if (newClasses.contains(source.langClasses[i])) { trimmed_nb_pc[j] = source.nb_pc[i]; for (int f = 0; f < source.numFeatures; f++) { int iFrom = source.numFeatures * i + f; int iTo = source.numFeatures * j + f; trimmed_nb_ptc[iTo] = source.nb_ptc[iFrom]; } j++; } } return new Model( trimmed_nb_classes, trimmed_nb_ptc, trimmed_nb_pc, source.dsa, source.dsaOutput); } Model(String [] langClasses, float [] ptc, float [] pc, short [] dsa, int[][] dsaOutput); }
Model { public static Model detectOnly(Set<String> langCodes) { final Model source = defaultModel(); Set<String> newClasses = new LinkedHashSet<String>(Arrays.asList(source.langClasses)); newClasses.retainAll(langCodes); if (newClasses.size() < 2) { throw new IllegalArgumentException("A model must contain at least two languages."); } String [] trimmed_nb_classes = newClasses.toArray(new String[newClasses.size()]); float[] trimmed_nb_pc = new float [newClasses.size()]; float[] trimmed_nb_ptc = new float [newClasses.size() * source.numFeatures]; for (int i = 0, j = 0; i < source.numClasses; i++) { if (newClasses.contains(source.langClasses[i])) { trimmed_nb_pc[j] = source.nb_pc[i]; for (int f = 0; f < source.numFeatures; f++) { int iFrom = source.numFeatures * i + f; int iTo = source.numFeatures * j + f; trimmed_nb_ptc[iTo] = source.nb_ptc[iFrom]; } j++; } } return new Model( trimmed_nb_classes, trimmed_nb_ptc, trimmed_nb_pc, source.dsa, source.dsaOutput); } Model(String [] langClasses, float [] ptc, float [] pc, short [] dsa, int[][] dsaOutput); static Model readExternal(ObjectInput in); static Model detectOnly(Set<String> langCodes); Set<String> getDetectedLanguages(); static synchronized Model defaultModel(); }
Model { public static Model detectOnly(Set<String> langCodes) { final Model source = defaultModel(); Set<String> newClasses = new LinkedHashSet<String>(Arrays.asList(source.langClasses)); newClasses.retainAll(langCodes); if (newClasses.size() < 2) { throw new IllegalArgumentException("A model must contain at least two languages."); } String [] trimmed_nb_classes = newClasses.toArray(new String[newClasses.size()]); float[] trimmed_nb_pc = new float [newClasses.size()]; float[] trimmed_nb_ptc = new float [newClasses.size() * source.numFeatures]; for (int i = 0, j = 0; i < source.numClasses; i++) { if (newClasses.contains(source.langClasses[i])) { trimmed_nb_pc[j] = source.nb_pc[i]; for (int f = 0; f < source.numFeatures; f++) { int iFrom = source.numFeatures * i + f; int iTo = source.numFeatures * j + f; trimmed_nb_ptc[iTo] = source.nb_ptc[iFrom]; } j++; } } return new Model( trimmed_nb_classes, trimmed_nb_ptc, trimmed_nb_pc, source.dsa, source.dsaOutput); } Model(String [] langClasses, float [] ptc, float [] pc, short [] dsa, int[][] dsaOutput); static Model readExternal(ObjectInput in); static Model detectOnly(Set<String> langCodes); Set<String> getDetectedLanguages(); static synchronized Model defaultModel();  }