FsCrawlerCli { @SuppressWarnings("deprecation") public static void main(String[] args) throws Exception { Scanner scanner = new Scanner(System.in); FsCrawlerCommand commands = new FsCrawlerCommand(); JCommander jCommander = new JCommander(commands); jCommander.parse(args); if (commands.debug || commands.trace || commands.silent) { LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); LoggerConfig loggerConfig = config.getLoggerConfig(FsCrawlerCli.class.getPackage().getName()); if (commands.silent) { if (commands.debug || commands.trace) { logger.warn("--debug or --trace can't be used when --silent is set. Only silent mode will be activated."); } if (commands.jobName == null) { logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting."); jCommander.usage(); return; } LoggerConfig rootLogger = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); loggerConfig.setLevel(Level.OFF); rootLogger.setLevel(Level.OFF); } else { loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); } ctx.updateLoggers(); } if (commands.help) { jCommander.usage(); return; } BootstrapChecks.check(); Path configDir; if (commands.configDir == null) { configDir = MetaFileHandler.DEFAULT_ROOT; } else { configDir = Paths.get(commands.configDir); } FsCrawlerUtil.createDirIfMissing(configDir); copyDefaultResources(configDir); FsSettings fsSettings; FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(configDir); String jobName; if (commands.jobName == null) { logger.info("No job specified. Here is the list of existing jobs:"); List<String> files = FsCrawlerJobsUtil.listExistingJobs(configDir); if (!files.isEmpty()) { for (int i = 0; i < files.size(); i++) { logger.info("[{}] - {}", i+1, files.get(i)); } int chosenFile = 0; while (chosenFile <= 0 || chosenFile > files.size()) { logger.info("Choose your job [1-{}]...", files.size()); chosenFile = scanner.nextInt(); } jobName = files.get(chosenFile - 1); } else { logger.info("No job exists in [{}].", configDir); logger.info("To create your first job, run 'fscrawler job_name' with 'job_name' you want"); return; } } else { jobName = commands.jobName.get(0); } if (commands.restart) { logger.debug("Cleaning existing status for job [{}]...", jobName); new FsJobFileHandler(configDir).clean(jobName); } try { logger.debug("Starting job [{}]...", jobName); fsSettings = fsSettingsFileHandler.read(jobName); if (fsSettings.getFs() == null) { fsSettings.setFs(Fs.DEFAULT); } if (fsSettings.getElasticsearch() == null) { fsSettings.setElasticsearch(Elasticsearch.DEFAULT()); } String username = commands.username; if (fsSettings.getElasticsearch().getUsername() != null) { username = fsSettings.getElasticsearch().getUsername(); } if (username != null && fsSettings.getElasticsearch().getPassword() == null) { logger.info("Password for " + username + ":"); String password = scanner.next(); fsSettings.getElasticsearch().setUsername(username); fsSettings.getElasticsearch().setPassword(password); } } catch (NoSuchFileException e) { logger.warn("job [{}] does not exist", jobName); String yesno = null; while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) { logger.info("Do you want to create it (Y/N)?"); yesno = scanner.next(); } if ("y".equalsIgnoreCase(yesno)) { fsSettings = FsSettings.builder(commands.jobName.get(0)) .setFs(Fs.DEFAULT) .setElasticsearch(Elasticsearch.DEFAULT()) .build(); fsSettingsFileHandler.write(fsSettings); Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML); logger.info("Settings have been created in [{}]. Please review and edit before relaunch", config); } return; } logger.trace("settings used for this crawler: [{}]", FsSettingsParser.toYaml(fsSettings)); if (FsCrawlerValidator.validateSettings(logger, fsSettings, commands.rest)) { return; } FsCrawlerImpl fsCrawler = new FsCrawlerImpl(configDir, fsSettings, commands.loop, commands.rest); Runtime.getRuntime().addShutdownHook(new FSCrawlerShutdownHook(fsCrawler)); try { if (commands.upgrade) { logger.info("Upgrading job [{}]. No rule implemented. Skipping.", jobName); } else { try { fsCrawler.getEsClient().start(); } catch (Exception t) { logger.fatal("We can not start Elasticsearch Client. Exiting.", t); return; } String elasticsearchVersion = fsCrawler.getEsClient().getVersion(); checkForDeprecatedResources(configDir, elasticsearchVersion); fsCrawler.start(); if (commands.rest) { RestServer.start(fsSettings, fsCrawler.getEsClient()); } while (!fsCrawler.getFsParser().isClosed()) { sleep(); } } } catch (Exception e) { logger.fatal("Fatal error received while running the crawler: [{}]", e.getMessage()); logger.debug("error caught", e); } finally { fsCrawler.close(); } }  @SuppressWarnings("deprecation") static void main(String[] args);  }
@Test public void testRestartCommand() throws Exception { String jobName = "fscrawler_restart_command"; FsSettingsFileHandler fsSettingsFileHandler = new FsSettingsFileHandler(metadataDir); FsJobFileHandler fsJobFileHandler = new FsJobFileHandler(metadataDir); Path jobDir = metadataDir.resolve(jobName); Files.createDirectories(jobDir); fsSettingsFileHandler.write(FsSettings.builder(jobName).build()); fsJobFileHandler.write(jobName, FsJob.builder().build()); assertThat(Files.exists(jobDir.resolve(FsJobFileHandler.FILENAME)), is(true)); String[] args = { "--config_dir", metadataDir.toString(), "--loop", "0", "--restart", jobName }; FsCrawlerCli.main(args); assertThat(Files.exists(jobDir.resolve(FsJobFileHandler.FILENAME)), is(false)); }