@Test public void testLindenWordDelimiterAnalyzer() throws Exception { LindenWordDelimiterAnalyzerFactory wordDelimiterAnalyzerFactory = new LindenWordDelimiterAnalyzerFactory(); Map<String, String> args = new HashMap<>(); Map<String, String> lastargs = new HashMap<>(); args.put("luceneMatchVersion", "LUCENE_4_10_0"); lastargs.putAll(args); Analyzer analyzer = wordDelimiterAnalyzerFactory.getInstance(args); TokenStream stream = analyzer.tokenStream("", new StringReader("Hello, this is a test case. " + "你好，这是一个测试的实例。" + "created2018by sls sun-li-shun SunLiShun")); String expected = "[hello][test][case][你][好][这][是][一][个][测][试][的][实][例][created][2018][sls][sun][li][shun][sun][li][shun]"; String out = ""; stream.reset(); while (stream.incrementToken()) { out += "[" + stream.getAttribute(CharTermAttribute.class).toString() + "]"; } Assert.assertEquals(expected, out); args.put("lower.case", "false"); args.putAll(lastargs); lastargs.putAll(args); analyzer = wordDelimiterAnalyzerFactory.getInstance(args); stream = analyzer.tokenStream("", new StringReader("Hello, this is a test case. " + "你好，这是一个测试的实例。" + "created2018by sls on 20140707")); expected = "[Hello][test][case][你][好][这][是][一][个][测][试][的][实][例][created][2018][sls][20140707]"; out = ""; stream.reset(); while (stream.incrementToken()) { out += "[" + stream.getAttribute(CharTermAttribute.class).toString() + "]"; } Assert.assertEquals(expected, out); args.put("set.stopwords", "false"); args.putAll(lastargs); lastargs.putAll(args); analyzer = wordDelimiterAnalyzerFactory.getInstance(args); stream = analyzer.tokenStream("", new StringReader("Hello, this is a test case. " + "你好，这是一个测试的实例。" + "created2018by sls on 20140707")); expected = "[Hello][this][is][a][test][case][你][好][这][是][一][个][测][试][的][实][例][created][2018][by][sls][on][20140707]"; out = ""; stream.reset(); while (stream.incrementToken()) { out += "[" + stream.getAttribute(CharTermAttribute.class).toString() + "]"; } Assert.assertEquals(expected, out); args.putAll(lastargs); args.put("splitOnCaseChange", "0"); args.put("set.stopwords", "false"); args.put("lower.case", "true"); lastargs.putAll(args); analyzer = wordDelimiterAnalyzerFactory.getInstance(args); stream = analyzer.tokenStream("", new StringReader("Hello, this is a test case. " + "你好，这是一个测试的实例。" + "created2018by sls sun-li-shun SunLiShun")); expected = "[hello][this][is][a][test][case][你][好][这][是][一][个][测][试][的][实][例][created][2018][by][sls][sun][li][shun][sunlishun]"; out = ""; stream.reset(); while (stream.incrementToken()) { out += "[" + stream.getAttribute(CharTermAttribute.class).toString() + "]"; } Assert.assertEquals(expected, out); }
public LindenWordDelimiterAnalyzer(Map<String, String> params) { if (params.containsKey(SET_STOP_WORDS)) { this.setStopWords = Boolean.parseBoolean(params.get(SET_STOP_WORDS)); params.remove(SET_STOP_WORDS); } if (params.containsKey(TO_LOWER_CASE)) { this.toLowerCase = Boolean.parseBoolean(params.get(TO_LOWER_CASE)); params.remove(TO_LOWER_CASE); } factoryDefault = new WordDelimiterFilterFactory(params); }
LindenWordDelimiterAnalyzer extends Analyzer { public LindenWordDelimiterAnalyzer(Map<String, String> params) { if (params.containsKey(SET_STOP_WORDS)) { this.setStopWords = Boolean.parseBoolean(params.get(SET_STOP_WORDS)); params.remove(SET_STOP_WORDS); } if (params.containsKey(TO_LOWER_CASE)) { this.toLowerCase = Boolean.parseBoolean(params.get(TO_LOWER_CASE)); params.remove(TO_LOWER_CASE); } factoryDefault = new WordDelimiterFilterFactory(params); } }
LindenWordDelimiterAnalyzer extends Analyzer { public LindenWordDelimiterAnalyzer(Map<String, String> params) { if (params.containsKey(SET_STOP_WORDS)) { this.setStopWords = Boolean.parseBoolean(params.get(SET_STOP_WORDS)); params.remove(SET_STOP_WORDS); } if (params.containsKey(TO_LOWER_CASE)) { this.toLowerCase = Boolean.parseBoolean(params.get(TO_LOWER_CASE)); params.remove(TO_LOWER_CASE); } factoryDefault = new WordDelimiterFilterFactory(params); } LindenWordDelimiterAnalyzer(Map<String, String> params); }
LindenWordDelimiterAnalyzer extends Analyzer { public LindenWordDelimiterAnalyzer(Map<String, String> params) { if (params.containsKey(SET_STOP_WORDS)) { this.setStopWords = Boolean.parseBoolean(params.get(SET_STOP_WORDS)); params.remove(SET_STOP_WORDS); } if (params.containsKey(TO_LOWER_CASE)) { this.toLowerCase = Boolean.parseBoolean(params.get(TO_LOWER_CASE)); params.remove(TO_LOWER_CASE); } factoryDefault = new WordDelimiterFilterFactory(params); } LindenWordDelimiterAnalyzer(Map<String, String> params);  }
LindenWordDelimiterAnalyzer extends Analyzer { public LindenWordDelimiterAnalyzer(Map<String, String> params) { if (params.containsKey(SET_STOP_WORDS)) { this.setStopWords = Boolean.parseBoolean(params.get(SET_STOP_WORDS)); params.remove(SET_STOP_WORDS); } if (params.containsKey(TO_LOWER_CASE)) { this.toLowerCase = Boolean.parseBoolean(params.get(TO_LOWER_CASE)); params.remove(TO_LOWER_CASE); } factoryDefault = new WordDelimiterFilterFactory(params); } LindenWordDelimiterAnalyzer(Map<String, String> params);   }