@Test public void testCaseSensitive() throws Exception { TableSchema schema = new TableSchema(new String[] {"sentence"}, new TypeInformation<?>[] {Types.STRING}); Params params = new Params() .set(StopWordsRemoverParams.SELECTED_COL, "sentence") .set(StopWordsRemoverParams.CASE_SENSITIVE, true) .set(StopWordsRemoverParams.STOP_WORDS, new String[]{"Test"}); StopWordsRemoverMapper mapper = new StopWordsRemoverMapper(schema, params); mapper.open(); assertEquals(mapper.map(Row.of("This is a unit test for filtering stopWords")).getField(0), "This unit test filtering stopWords"); assertEquals(mapper.map(Row.of("Filter stopWords test")).getField(0), "Filter stopWords test"); assertEquals(mapper.getOutputSchema(), schema); }
@Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); }
StopWordsRemoverMapper extends SISOMapper { @Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); } }
StopWordsRemoverMapper extends SISOMapper { @Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); } StopWordsRemoverMapper(TableSchema dataSchema, Params params); }
StopWordsRemoverMapper extends SISOMapper { @Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); } StopWordsRemoverMapper(TableSchema dataSchema, Params params); @Override void open(); }
StopWordsRemoverMapper extends SISOMapper { @Override public void open(){ this.stopWordsSet = new HashSet<>(); String[] stopWords = this.params.get(StopWordsRemoverParams.STOP_WORDS); if (null != stopWords) { for (String stopWord : stopWords) { stopWordsSet.add(caseSensitive ? stopWord : stopWord.toLowerCase()); } } loadDefaultStopWords(); } StopWordsRemoverMapper(TableSchema dataSchema, Params params); @Override void open();  }