@Test public void testHiveCheck() { ApplicationContext context = new ClassPathXmlApplicationContext("rest-servlet.xml"); HiveCheckService hcs = context.getBean(HiveCheckService.class); hcs.checkHiveJob("test", "test2"); }
public boolean checkHiveJob(String dbName, String tableName) { logger.debug("begin to checkhivejob, table: " + dbName + "." + tableName); try { HiveJobsDo hiveJobsDao = hiveJobsService.getHiveJob(dbName, tableName); if (null == hiveJobsDao) { logger.error("not found table: " + dbName + "." + tableName); return false; } logger.info(hiveJobsDao); if (hiveJobsDao.getCheckrate() <= 0) { return true; } long avgSize = getAvgSize(dbName, tableName, hive_check_avg_partition_num); long lastSize = 0; if (lastPartition != null) { logger.info("last partition: " + lastPartition.getSd().getLocation()); lastSize = getPathSize(lastPartition.getSd().getLocation()); } logger.info("avg size of the table: " + avgSize); logger.info("size of the last partition of the table: " + lastSize); if (avgSize <= 0) { return false; } float rate = lastSize / (float) avgSize; logger.debug("rate = " + rate); StringBuilder content = new StringBuilder(); if (rate < 1 - hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result partition decrease " + (1 - rate) * 100 + "%" + "<br>"); } else if (rate > 1 + hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result parttion increase " + (rate - 1) * 100 + "%" + "<br>"); } else { logger.info("the size of result parttion is OK."); return true; } content.append("Partition: " + lastPartition.getDbName() + "." + lastPartition.getTableName() + lastPartition.getValues() + "<br>"); content.append("avg size of the table: " + avgSize + "<br>"); content.append("size of the last partition of the table: " + lastSize + "<br>"); logger.error(content.toString()); SendAlert(content.toString(), hiveJobsDao); } catch (Exception e) { logger.error("checkHiveJob error....", e); } return false; }
HiveCheckService { public boolean checkHiveJob(String dbName, String tableName) { logger.debug("begin to checkhivejob, table: " + dbName + "." + tableName); try { HiveJobsDo hiveJobsDao = hiveJobsService.getHiveJob(dbName, tableName); if (null == hiveJobsDao) { logger.error("not found table: " + dbName + "." + tableName); return false; } logger.info(hiveJobsDao); if (hiveJobsDao.getCheckrate() <= 0) { return true; } long avgSize = getAvgSize(dbName, tableName, hive_check_avg_partition_num); long lastSize = 0; if (lastPartition != null) { logger.info("last partition: " + lastPartition.getSd().getLocation()); lastSize = getPathSize(lastPartition.getSd().getLocation()); } logger.info("avg size of the table: " + avgSize); logger.info("size of the last partition of the table: " + lastSize); if (avgSize <= 0) { return false; } float rate = lastSize / (float) avgSize; logger.debug("rate = " + rate); StringBuilder content = new StringBuilder(); if (rate < 1 - hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result partition decrease " + (1 - rate) * 100 + "%" + "<br>"); } else if (rate > 1 + hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result parttion increase " + (rate - 1) * 100 + "%" + "<br>"); } else { logger.info("the size of result parttion is OK."); return true; } content.append("Partition: " + lastPartition.getDbName() + "." + lastPartition.getTableName() + lastPartition.getValues() + "<br>"); content.append("avg size of the table: " + avgSize + "<br>"); content.append("size of the last partition of the table: " + lastSize + "<br>"); logger.error(content.toString()); SendAlert(content.toString(), hiveJobsDao); } catch (Exception e) { logger.error("checkHiveJob error....", e); } return false; } }
HiveCheckService { public boolean checkHiveJob(String dbName, String tableName) { logger.debug("begin to checkhivejob, table: " + dbName + "." + tableName); try { HiveJobsDo hiveJobsDao = hiveJobsService.getHiveJob(dbName, tableName); if (null == hiveJobsDao) { logger.error("not found table: " + dbName + "." + tableName); return false; } logger.info(hiveJobsDao); if (hiveJobsDao.getCheckrate() <= 0) { return true; } long avgSize = getAvgSize(dbName, tableName, hive_check_avg_partition_num); long lastSize = 0; if (lastPartition != null) { logger.info("last partition: " + lastPartition.getSd().getLocation()); lastSize = getPathSize(lastPartition.getSd().getLocation()); } logger.info("avg size of the table: " + avgSize); logger.info("size of the last partition of the table: " + lastSize); if (avgSize <= 0) { return false; } float rate = lastSize / (float) avgSize; logger.debug("rate = " + rate); StringBuilder content = new StringBuilder(); if (rate < 1 - hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result partition decrease " + (1 - rate) * 100 + "%" + "<br>"); } else if (rate > 1 + hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result parttion increase " + (rate - 1) * 100 + "%" + "<br>"); } else { logger.info("the size of result parttion is OK."); return true; } content.append("Partition: " + lastPartition.getDbName() + "." + lastPartition.getTableName() + lastPartition.getValues() + "<br>"); content.append("avg size of the table: " + avgSize + "<br>"); content.append("size of the last partition of the table: " + lastSize + "<br>"); logger.error(content.toString()); SendAlert(content.toString(), hiveJobsDao); } catch (Exception e) { logger.error("checkHiveJob error....", e); } return false; }  }
HiveCheckService { public boolean checkHiveJob(String dbName, String tableName) { logger.debug("begin to checkhivejob, table: " + dbName + "." + tableName); try { HiveJobsDo hiveJobsDao = hiveJobsService.getHiveJob(dbName, tableName); if (null == hiveJobsDao) { logger.error("not found table: " + dbName + "." + tableName); return false; } logger.info(hiveJobsDao); if (hiveJobsDao.getCheckrate() <= 0) { return true; } long avgSize = getAvgSize(dbName, tableName, hive_check_avg_partition_num); long lastSize = 0; if (lastPartition != null) { logger.info("last partition: " + lastPartition.getSd().getLocation()); lastSize = getPathSize(lastPartition.getSd().getLocation()); } logger.info("avg size of the table: " + avgSize); logger.info("size of the last partition of the table: " + lastSize); if (avgSize <= 0) { return false; } float rate = lastSize / (float) avgSize; logger.debug("rate = " + rate); StringBuilder content = new StringBuilder(); if (rate < 1 - hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result partition decrease " + (1 - rate) * 100 + "%" + "<br>"); } else if (rate > 1 + hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result parttion increase " + (rate - 1) * 100 + "%" + "<br>"); } else { logger.info("the size of result parttion is OK."); return true; } content.append("Partition: " + lastPartition.getDbName() + "." + lastPartition.getTableName() + lastPartition.getValues() + "<br>"); content.append("avg size of the table: " + avgSize + "<br>"); content.append("size of the last partition of the table: " + lastSize + "<br>"); logger.error(content.toString()); SendAlert(content.toString(), hiveJobsDao); } catch (Exception e) { logger.error("checkHiveJob error....", e); } return false; }  @Scheduled(fixedDelay = 1 * 3600 * 1000) void run(); boolean checkHiveJob(String dbName, String tableName); static void main(String[] args); }
HiveCheckService { public boolean checkHiveJob(String dbName, String tableName) { logger.debug("begin to checkhivejob, table: " + dbName + "." + tableName); try { HiveJobsDo hiveJobsDao = hiveJobsService.getHiveJob(dbName, tableName); if (null == hiveJobsDao) { logger.error("not found table: " + dbName + "." + tableName); return false; } logger.info(hiveJobsDao); if (hiveJobsDao.getCheckrate() <= 0) { return true; } long avgSize = getAvgSize(dbName, tableName, hive_check_avg_partition_num); long lastSize = 0; if (lastPartition != null) { logger.info("last partition: " + lastPartition.getSd().getLocation()); lastSize = getPathSize(lastPartition.getSd().getLocation()); } logger.info("avg size of the table: " + avgSize); logger.info("size of the last partition of the table: " + lastSize); if (avgSize <= 0) { return false; } float rate = lastSize / (float) avgSize; logger.debug("rate = " + rate); StringBuilder content = new StringBuilder(); if (rate < 1 - hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result partition decrease " + (1 - rate) * 100 + "%" + "<br>"); } else if (rate > 1 + hiveJobsDao.getCheckrate()) { content.append("Alert: the size of result parttion increase " + (rate - 1) * 100 + "%" + "<br>"); } else { logger.info("the size of result parttion is OK."); return true; } content.append("Partition: " + lastPartition.getDbName() + "." + lastPartition.getTableName() + lastPartition.getValues() + "<br>"); content.append("avg size of the table: " + avgSize + "<br>"); content.append("size of the last partition of the table: " + lastSize + "<br>"); logger.error(content.toString()); SendAlert(content.toString(), hiveJobsDao); } catch (Exception e) { logger.error("checkHiveJob error....", e); } return false; }  @Scheduled(fixedDelay = 1 * 3600 * 1000) void run(); boolean checkHiveJob(String dbName, String tableName); static void main(String[] args);  }