StopWordsRemoverMapper extends SISOMapper { @Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); } StopWordsRemoverMapper(TableSchema dataSchema, Params params); @Override void open();  }
@Test public void testStopWords() throws Exception { TableSchema schema = new TableSchema(new String[] {"sentence"}, new TypeInformation<?>[] {Types.STRING}); Params params = new Params() .set(StopWordsRemoverParams.SELECTED_COL, "sentence") .set(StopWordsRemoverParams.STOP_WORDS, new String[]{"Test"}); StopWordsRemoverMapper mapper = new StopWordsRemoverMapper(schema, params); mapper.open(); assertEquals(mapper.map(Row.of("This is a unit test for filtering stopWords")).getField(0), "unit filtering stopWords"); assertEquals(mapper.map(Row.of("Filter stopWords test")).getField(0), "Filter stopWords"); assertEquals(mapper.map(Row.of("这 是 停用词 过滤 的 单元 测试")).getField(0), "停用词 过滤 单元 测试"); assertEquals(mapper.getOutputSchema(), schema); }
@Test public void testCaseSensitive() throws Exception { TableSchema schema = new TableSchema(new String[] {"sentence"}, new TypeInformation<?>[] {Types.STRING}); Params params = new Params() .set(StopWordsRemoverParams.SELECTED_COL, "sentence") .set(StopWordsRemoverParams.CASE_SENSITIVE, true) .set(StopWordsRemoverParams.STOP_WORDS, new String[]{"Test"}); StopWordsRemoverMapper mapper = new StopWordsRemoverMapper(schema, params); mapper.open(); assertEquals(mapper.map(Row.of("This is a unit test for filtering stopWords")).getField(0), "This unit test filtering stopWords"); assertEquals(mapper.map(Row.of("Filter stopWords test")).getField(0), "Filter stopWords test"); assertEquals(mapper.getOutputSchema(), schema); }