LexiconReader { public List<PosTaggerLexicon> readLexicons(File lexiconPropsFile) throws IOException, TalismaneException { LOG.debug("Serializing from " + lexiconPropsFile.getPath()); List<PosTaggerLexicon> lexicons = new ArrayList<>(); File lexiconDir = lexiconPropsFile.getParentFile(); Map<String, String> properties = StringUtils.getArgMap(lexiconPropsFile, "UTF-8"); String[] lexiconList = properties.get("lexicons").split(","); List<String> knownPropertyList = Arrays.asList("file", "regex", "categories", "exclusions", "encoding", "uniqueKey"); Set<String> knownProperties = new HashSet<String>(knownPropertyList); for (String property : properties.keySet()) { if (property.equals("lexicons")) { } else { boolean foundLexicon = false; for (String lexiconName : lexiconList) { if (property.startsWith(lexiconName + ".")) { foundLexicon = true; String remainder = property.substring(lexiconName.length() + 1); if (!knownProperties.contains(remainder)) { throw new TalismaneException("Unknown property: " + property); } } if (foundLexicon) break; } if (!foundLexicon) throw new TalismaneException("Unknown lexicon in property: " + property); } } for (String lexiconName : lexiconList) { LOG.debug("Lexicon: " + lexiconName); String lexiconFilePath = properties.get(lexiconName + ".file"); String lexiconRegexPath = properties.get(lexiconName + ".regex"); String lexiconExclusionPath = properties.get(lexiconName + ".exclusions"); String categoryString = properties.get(lexiconName + ".categories"); String lexiconEncoding = properties.get(lexiconName + ".encoding"); String lexiconUniqueKey = properties.get(lexiconName + ".uniqueKey"); File lexiconRegexFile = new File(lexiconDir, lexiconRegexPath); Scanner regexScanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(lexiconRegexFile), "UTF-8"))); File lexiconInputFile = new File(lexiconDir, lexiconFilePath); InputStream inputStream = null; if (lexiconInputFile.getName().endsWith(".zip")) { InputStream inputStream2 = new FileInputStream(lexiconInputFile); @SuppressWarnings("resource") ZipInputStream zis = new ZipInputStream(inputStream2); zis.getNextEntry(); inputStream = zis; } else { inputStream = new FileInputStream(lexiconInputFile); } Charset lexiconCharset = Charset.defaultCharset(); if (lexiconEncoding != null) lexiconCharset = Charset.forName(lexiconEncoding); Reader reader = new BufferedReader(new InputStreamReader(inputStream, lexiconCharset)); Scanner lexiconScanner = new Scanner(reader); RegexLexicalEntryReader lexicalEntryReader = new RegexLexicalEntryReader(regexScanner); Set<String> categories = null; if (categoryString != null) { categories = new HashSet<String>(); String[] cats = categoryString.split(","); for (String cat : cats) categories.add(cat); } List<String> exclusionAttributes = null; List<List<String>> exclusions = null; if (lexiconExclusionPath != null) { exclusions = new ArrayList<List<String>>(); File lexiconExclusionFile = new File(lexiconDir, lexiconExclusionPath); Scanner exclusionScanner = new Scanner(new BufferedReader(new InputStreamReader(new FileInputStream(lexiconExclusionFile), "UTF-8"))); while (exclusionScanner.hasNextLine()) { String line = exclusionScanner.nextLine(); if (line.length() == 0 || line.startsWith("#")) continue; String[] parts = line.split("\t"); if (exclusionAttributes == null) { exclusionAttributes = new ArrayList<String>(); for (String part : parts) { exclusionAttributes.add(part); } } else { List<String> exclusion = new ArrayList<String>(); for (String part : parts) { exclusion.add(part); } exclusions.add(exclusion); } } exclusionScanner.close(); } List<LexicalAttribute> uniqueAttributes = null; if (lexiconUniqueKey != null) { uniqueAttributes = new ArrayList<LexicalAttribute>(); String[] uniqueKeyElements = lexiconUniqueKey.split(","); for (String uniqueKeyElement : uniqueKeyElements) { try { LexicalAttribute attribute = LexicalAttribute.valueOf(uniqueKeyElement); uniqueAttributes.add(attribute); } catch (IllegalArgumentException e) { lexiconScanner.close(); throw new TalismaneException("Unknown attribute in " + lexiconName + ".uniqueKey: " + uniqueKeyElement); } } } LOG.debug("Serializing: " + lexiconFilePath); LexiconFile lexiconFile = new LexiconFile(lexiconName, lexiconScanner, lexicalEntryReader, sessionId); if (categories != null) lexiconFile.setCategories(categories); if (exclusionAttributes != null) lexiconFile.setExclusionAttributes(exclusionAttributes); if (exclusions != null) lexiconFile.setExclusions(exclusions); if (uniqueAttributes != null) lexiconFile.setUniqueKeyAttributes(uniqueAttributes); lexiconFile.load(); inputStream.close(); lexicons.add(lexiconFile); } return lexicons; } LexiconReader(String sessionId); static void main(String[] args); List<PosTaggerLexicon> readLexicons(File lexiconPropsFile); void serializeLexicons(List<PosTaggerLexicon> lexicons, File outFile); List<PosTaggerLexicon> deserializeLexicons(File lexiconFile); List<PosTaggerLexicon> deserializeLexicons(ZipInputStream zis);  }
@Test public void testReadLexicons() throws Exception { System.setProperty("config.file", "src/test/resources/testWithLex.conf"); ConfigFactory.invalidateCaches(); final Config config = ConfigFactory.load(); final String sessionId = "test"; PosTaggerLexicon lexicon = TalismaneSession.get(sessionId).getMergedLexicon(); List<LexicalEntry> entries = lexicon.getEntries("dame"); for (LexicalEntry entry : entries) { System.out.println(entry); } assertEquals(9, entries.size()); PosTagSet posTagSet = TalismaneSession.get(sessionId).getPosTagSet(); entries = lexicon.findLexicalEntries("dame", posTagSet.getPosTag("NC")); for (LexicalEntry entry : entries) { System.out.println(entry); } assertEquals(2, entries.size()); Set<PosTag> posTags = lexicon.findPossiblePosTags("dame"); System.out.println(posTags); assertEquals(4, posTags.size()); System.clearProperty("config.file"); ConfigFactory.invalidateCaches(); }