FeedEvictor extends Configured implements Tool { public static void main(String[] args) throws Exception { Configuration conf = new Configuration(); Path confPath = new Path("file: LOG.info(confPath + " found ? " + confPath.getFileSystem(conf).exists(confPath)); conf.addResource(confPath); int ret = ToolRunner.run(conf, new FeedEvictor(), args); if (ret != 0) { throw new Exception("Unable to perform eviction action args: " + Arrays.toString(args)); } }  static void main(String[] args); @Override int run(String[] args);  }
@Test public void testBadArgs() throws Exception { try { FeedEvictor.main(null); Assert.fail("Expected an exception to be thrown"); } catch (Exception ignore) { } try { FeedEvictor.main(new String[] {"1","2"}); Assert.fail("Expected an exception to be thrown"); } catch (Exception ignore) { } }
@Test public void testEviction1() throws Exception { try { FeedEvictor.main(new String[]{"1", "2", "3", "4", "5","6","7"}); } catch (Exception e) { } }
@Test public void testEviction2() throws Exception { try { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); fs.delete(new Path("/"), true); stream.clear(); Pair<List<String>, List<String>> pair; pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/data"); String dataPath = "/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"; String logFile = "/ivory/staging/feed/instancePaths-2012-01-01-01-00.csv"; FeedEvictor.main(new String[] { "-feedBasePath",cluster.getConf().get("fs.default.name") + dataPath, "-retentionType","instance", "-retentionLimit","days(10)", "-timeZone","UTC", "-frequency","daily", "-logFile",logFile}); assertFailures(fs, pair); compare(map.get("feed1"), stream.getBuffer()); Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath)); } catch (Exception e) { Assert.fail("Unknown exception", e); } }
@Test public void testEviction3() throws Exception { try { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); fs.delete(new Path("/"), true); stream.clear(); Pair<List<String>, List<String>> pair; pair = createTestData("feed2", "yyyyMMddHH/'more'/yyyy", 5, TimeUnit.HOURS,"/data"); String dataPath = "/data/YYYY/feed2/mmHH/dd/MM/" + "?{YEAR}?{MONTH}?{DAY}?{HOUR}/more/?{YEAR}"; String logFile = "/ivory/staging/feed/instancePaths-2012-01-01-02-00.csv"; FeedEvictor.main(new String[] { "-feedBasePath",cluster.getConf().get("fs.default.name") + dataPath, "-retentionType","instance", "-retentionLimit","hours(5)", "-timeZone","UTC", "-frequency","hourly", "-logFile",logFile}); assertFailures(fs, pair); compare(map.get("feed2"), stream.getBuffer()); Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath)); } catch (Exception e) { Assert.fail("Unknown exception", e); } }
@Test public void testEviction4() throws Exception { try { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); fs.delete(new Path("/"), true); stream.clear(); Pair<List<String>, List<String>> pair; pair = createTestData("/data"); FeedEvictor.main(new String[] { "-feedBasePath", cluster.getConf().get("fs.default.name") + "/data/YYYY/feed3/dd/MM/" + "?{MONTH}/more/?{HOUR}", "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", "/ivory/staging/feed/2012-01-01-04-00" }); Assert.assertEquals("instances=NULL", stream.getBuffer()); stream.clear(); String dataPath="/data/YYYY/feed4/dd/MM/" + "02/more/hello"; String logFile = "/ivory/staging/feed/instancePaths-2012-01-01-02-00.csv"; FeedEvictor.main(new String[] { "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile }); Assert.assertEquals("instances=NULL", stream.getBuffer()); Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath)); assertFailures(fs, pair); } catch (Exception e) { Assert.fail("Unknown exception", e); } }
@Test public void testEviction5() throws Exception { try { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); fs.delete(new Path("/"), true); stream.clear(); Pair<List<String>, List<String>> pair,statsPair,metaPair,tmpPair; pair = createTestData("/data"); statsPair = createTestData("/stats"); metaPair = createTestData("/meta"); tmpPair = createTestData("/tmp"); FeedEvictor.main(new String[] { "-feedBasePath", getFeedBasePath(cluster, "/data") + "#" + getFeedBasePath(cluster, "/stats") + "#" + getFeedBasePath(cluster, "/meta") + "#" + getFeedBasePath(cluster, "/tmp"), "-retentionType", "instance", "-retentionLimit", "months(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", "/ivory/staging/feed/2012-01-01-04-00" }); Assert.assertEquals("instances=NULL", stream.getBuffer()); stream.clear(); String dataPath="/data/YYYY/feed4/dd/MM/" + "02/more/hello"; String logFile = "/ivory/staging/feed/instancePaths-2012-01-01-02-00.csv"; FeedEvictor.main(new String[] { "-feedBasePath", cluster.getConf().get("fs.default.name") + dataPath, "-retentionType", "instance", "-retentionLimit", "hours(5)", "-timeZone", "UTC", "-frequency", "hourly", "-logFile", logFile }); Assert.assertEquals("instances=NULL", stream.getBuffer()); Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath)); assertFailures(fs, pair); } catch (Exception e) { Assert.fail("Unknown exception", e); } }
@Test public void testEviction6() throws Exception { try { Configuration conf = cluster.getConf(); FileSystem fs = FileSystem.get(conf); fs.delete(new Path("/"), true); stream.clear(); Pair<List<String>, List<String>> pair, statsPair,metaPair; pair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/data"); statsPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/stats"); metaPair = createTestData("feed1", "yyyy-MM-dd/'more'/yyyy", 10, TimeUnit.DAYS,"/meta"); String dataPath = cluster.getConf().get("fs.default.name")+"/data/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}" + "#" + cluster.getConf().get("fs.default.name")+"/stats/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}" + "#" + cluster.getConf().get("fs.default.name")+"/meta/YYYY/feed1/mmHH/dd/MM/?{YEAR}-?{MONTH}-?{DAY}/more/?{YEAR}"; String logFile = "/ivory/staging/feed/instancePaths-2012-01-01-01-00.csv"; FeedEvictor.main(new String[] { "-feedBasePath", dataPath, "-retentionType","instance", "-retentionLimit","days(10)", "-timeZone","UTC", "-frequency","daily", "-logFile",logFile}); assertFailures(fs, pair); Assert.assertEquals(readLogFile(new Path(logFile)), getExpectedInstancePaths(dataPath.replaceAll(cluster .getConf().get("fs.default.name"), ""))); } catch (Exception e) { Assert.fail("Unknown exception", e); } }