HadoopTwitterTokenTool extends Configured implements Tool { public static void main(String[] args) throws Exception { try { ToolRunner.run(new HadoopTwitterTokenTool(args), args); } catch (CmdLineException e) { System.err.print(e); } } HadoopTwitterTokenTool(String[] originalArgs); @Override int run(String[] args); static void main(String[] args);  }
@Test public void testMonthLongDFIDFCSVByTime() throws Exception { String command = String.format(hadoopCommand, monthLongTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); command += " -svbt"; String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp","-m PORTER_STEM" }); System.out.println(Arrays.toString(args)); HadoopTwitterTokenTool.main(args); Path p = new Path(resultsOutputLocation.getAbsolutePath()); System.out.println(p); }
@Test public void testMonthLongDFIDFCSVByTimeAsMatlab() throws Exception { String command = String.format(hadoopCommand, monthLongTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); command += " -svbt -matlab"; String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp","-m PORTER_STEM" }); System.out.println(Arrays.toString(args)); HadoopTwitterTokenTool.main(args); Path p = new Path(resultsOutputLocation.getAbsolutePath()); System.out.println(p); }
@Test public void testResumingIncompleteJob() throws Exception { String command = String.format(hadoopCommand, jsonTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); File[] deletedPartFiles = FileUtils.findRecursive(outputLocation, new FilenameFilter() { @Override public boolean accept(File dir, String name) { if (name.contains("part")) { System.out.println("Removing " + name); FileUtils.deleteRecursive(dir); return true; } return false; } }); FileUtils.deleteRecursive(this.resultsOutputLocation); for (File file : deletedPartFiles) { assertTrue(!file.exists()); } HadoopTwitterTokenTool.main(args); for (File file : deletedPartFiles) { assertTrue(file.exists()); } }
@Test public void testResumingIncompleteJobType2() throws Exception { String command = String.format(hadoopCommand, jsonTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); HadoopTwitterTokenTool.main(args); }
@Test public void testDFIDF() throws Exception { String command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex.readWordCountLines(resultsOutputLocation.getAbsolutePath()); assertTrue(wordLineCounts.get(".").firstObject() == 12); HashMap<Long, IndependentPair<Long, Long>> timeLineCounts = TimeIndex .readTimeCountLines(resultsOutputLocation.getAbsolutePath()); long nenglish = 43; long sum = 0; for (IndependentPair<Long, Long> countLine : timeLineCounts.values()) { sum += countLine.firstObject(); } assertTrue(sum == nenglish); }
@Test public void testStemmingDFIDF() throws Exception { String command = String.format(hadoopCommand, jsonTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex .readWordCountLines(resultsOutputLocation.getAbsolutePath()); assertTrue(wordLineCounts.get(".").firstObject() == 12); }
@Test public void testWordStatsDFIDF() throws Exception { String command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "WORD_STATS", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex .readWordCountLines(resultsOutputLocation.getAbsolutePath()); assertTrue(wordLineCounts.get(".").firstObject() == 12); }
@Test public void testWordJacardDFIDF() throws Exception { String command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "JACARD_INDEX", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); HadoopTwitterTokenTool.main(command.split(" ")); LinkedHashMap<Long, JacardIndex> timejacardIndex = CumulativeTimeWord .readTimeCountLines(resultsOutputLocation.getAbsolutePath()); String srsOut = resultsOutputLocation.getAbsolutePath() + "-srs"; command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "JACARD_INDEX", srsOut, "DFIDF", "analysis.stemmed"); command += " -rm"; HadoopTwitterTokenTool.main(command.split(" ")); LinkedHashMap<Long, JacardIndex> srstimejacardIndex = CumulativeTimeWord .readTimeCountLines(srsOut); for (Long key : srstimejacardIndex.keySet()) { assertTrue(srstimejacardIndex.get(key).equals( timejacardIndex.get(key))); } }
@Test public void testMultiFileInput() throws Exception { hadoopCommand = "-if %s -o %s -om %s -ro %s -m %s -j %s -t 1"; File inputList = folder.newFile("inputs-testMultiFileInput.txt"); PrintWriter listWriter = new PrintWriter(new FileWriter(inputList)); listWriter.println(jsonTweets.getAbsolutePath()); listWriter.println(stemmedTweets.getAbsolutePath()); listWriter.flush(); listWriter.close(); String command = String.format(hadoopCommand, inputList.getAbsolutePath(), outputLocation.getAbsolutePath(), "WORD_STATS", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex .readWordCountLines(resultsOutputLocation.getAbsolutePath()); assertTrue(wordLineCounts.get(".").firstObject() == 24); }
@Test public void testTokenMatchMode() throws Exception { hadoopCommand = "-i %s -o %s -m %s -j %s -r %s"; String command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "MATCH_TERM", "text", ":[)]"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); String[] tweets = HadoopToolsUtil.readlines(outputLocation .getAbsolutePath() + "/" + TokenRegexStage.OUT_NAME); assertTrue(tweets.length == 6); }
@Test public void testTokenMatchModeFiltered() throws Exception { hadoopCommand = "-i %s -o %s -m %s -j %s -r %s -jf %s"; String command = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "MATCH_TERM", "text", "[.]", "analysis.langid.language:==en"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); String[] tweets = HadoopToolsUtil.readlines(outputLocation .getAbsolutePath() + "/" + TokenRegexStage.OUT_NAME); assertTrue(tweets.length == 22); }
@Test public void testMultiFileInputOnlyOutput() throws Exception { String firstOuputLocation = this.outputLocation.getAbsolutePath() + "-first"; String secondOuputLocation = this.outputLocation.getAbsolutePath() + "-second"; String firstResultsOut = resultsOutputLocation.getAbsolutePath() + "-first"; String secondResultsOut = resultsOutputLocation.getAbsolutePath() + "-second"; String commandFirst = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), firstOuputLocation, "WORD_STATS", firstResultsOut, "DFIDF", "analysis.stemmed"); String[] args = commandFirst.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); String commandSecond = String.format(hadoopCommand, stemmedTweets.getAbsolutePath(), secondOuputLocation, "WORD_STATS", secondResultsOut, "DFIDF", "analysis.stemmed"); args = commandSecond.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); hadoopCommand = "-if %s -om %s -ro %s"; File inputList = folder .newFile("inputs-testMultiFileInputOnlyOutput.txt"); PrintWriter listWriter = new PrintWriter(new FileWriter(inputList)); listWriter.println(firstOuputLocation); listWriter.println(secondOuputLocation); listWriter.flush(); listWriter.close(); String command = String.format(hadoopCommand, inputList.getAbsolutePath(), "WORD_STATS", resultsOutputLocation.getAbsolutePath()); HadoopTwitterTokenTool.main(command.split(" ")); HashMap<String, IndependentPair<Long, Long>> wordLineCounts = WordIndex .readWordCountLines(resultsOutputLocation.getAbsolutePath()); assertTrue(wordLineCounts.get(".").firstObject() == 24); }
@Test public void testMonthLongDFIDF() throws Exception { hadoopCommand = "-i %s -o %s -om %s -ro %s -m %s -j %s -t 1 -wt . -wt !"; String command = String.format(hadoopCommand, monthLongTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "SPECIFIC_WORD", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); HadoopTwitterTokenTool.main(args); Path p = new Path(resultsOutputLocation.getAbsolutePath()); p = new Path(p, SpecificWordStageProvider.SPECIFIC_WORD); p = new Path(p, "part-r-00000"); FileSystem fs = HadoopToolsUtil.getFileSystem(p); WordDFIDFTimeSeriesCollection c = IOUtils.read(fs.open(p), WordDFIDFTimeSeriesCollection.class); System.out.println(c); }
@Test public void testMonthLongDFIDFCSV() throws Exception { String command = String.format(hadoopCommand, monthLongTweets.getAbsolutePath(), outputLocation.getAbsolutePath(), "CSV", resultsOutputLocation.getAbsolutePath(), "DFIDF", "analysis.stemmed"); String[] args = command.split(" "); args = (String[]) ArrayUtils.addAll(args, new String[] { "-pp", "-m PORTER_STEM" }); System.out.println(Arrays.toString(args)); HadoopTwitterTokenTool.main(args); Path p = new Path(resultsOutputLocation.getAbsolutePath()); System.out.println(p); }