@Test void canRunWithoutSudo(@TempDir final Path tempDir) throws IOException, GenieException { final JobsProperties jobsProperties = JobsProperties.getJobsPropertiesDefaults(); jobsProperties.getUsers().setRunAsUserEnabled(false); final Path testFile = tempDir.resolve(UUID.randomUUID().toString()); Files.write(testFile, Lists.newArrayList("hi", "bye")); final DiskCleanupProperties properties = new DiskCleanupProperties(); final Instant threshold = TaskUtils.getMidnightUTC().minus(properties.getRetention(), ChronoUnit.DAYS); final String job1Id = UUID.randomUUID().toString(); final String job2Id = UUID.randomUUID().toString(); final String job3Id = UUID.randomUUID().toString(); final String job4Id = UUID.randomUUID().toString(); final String job5Id = UUID.randomUUID().toString(); final Job job1 = Mockito.mock(Job.class); Mockito.when(job1.getStatus()).thenReturn(JobStatus.INIT); final Job job2 = Mockito.mock(Job.class); Mockito.when(job2.getStatus()).thenReturn(JobStatus.RUNNING); final Job job3 = Mockito.mock(Job.class); Mockito.when(job3.getStatus()).thenReturn(JobStatus.SUCCEEDED); Mockito.when(job3.getFinished()).thenReturn(Optional.of(threshold.minus(1, ChronoUnit.MILLIS))); final Job job4 = Mockito.mock(Job.class); Mockito.when(job4.getStatus()).thenReturn(JobStatus.FAILED); Mockito.when(job4.getFinished()).thenReturn(Optional.of(threshold)); this.createJobDir(job1Id, tempDir); this.createJobDir(job2Id, tempDir); this.createJobDir(job3Id, tempDir); this.createJobDir(job4Id, tempDir); this.createJobDir(job5Id, tempDir); final TaskScheduler scheduler = Mockito.mock(TaskScheduler.class); final Resource jobDir = Mockito.mock(Resource.class); Mockito.when(jobDir.exists()).thenReturn(true); Mockito.when(jobDir.getFile()).thenReturn(tempDir.toFile()); final PersistenceService persistenceService = Mockito.mock(PersistenceService.class); Mockito.when(persistenceService.getJob(job1Id)).thenReturn(job1); Mockito.when(persistenceService.getJob(job2Id)).thenReturn(job2); Mockito.when(persistenceService.getJob(job3Id)).thenReturn(job3); Mockito.when(persistenceService.getJob(job4Id)).thenReturn(job4); Mockito.when(persistenceService.getJob(job5Id)).thenThrow(new GenieServerException("blah")); final DataServices dataServices = Mockito.mock(DataServices.class); Mockito.when(dataServices.getPersistenceService()).thenReturn(persistenceService); final DiskCleanupTask task = new DiskCleanupTask( properties, scheduler, jobDir, dataServices, jobsProperties, Mockito.mock(Executor.class), new SimpleMeterRegistry() ); task.run(); Assertions.assertThat(new File(jobDir.getFile(), job1Id).exists()).isTrue(); Assertions.assertThat(new File(jobDir.getFile(), job2Id).exists()).isTrue(); Assertions.assertThat(new File(jobDir.getFile(), job3Id).exists()).isFalse(); Assertions.assertThat(new File(jobDir.getFile(), job4Id).exists()).isTrue(); Assertions.assertThat(new File(jobDir.getFile(), job5Id).exists()).isTrue(); }
@Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.persistenceService.getJob(id); if (job.getStatus().isActive()) { continue; } final Instant midnightUTC = TaskUtils.getMidnightUTC(); final Instant retentionThreshold = midnightUTC.minus(this.properties.getRetention(), ChronoUnit.DAYS); final Optional<Instant> finished = job.getFinished(); if (finished.isPresent() && finished.get().isBefore(retentionThreshold)) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); }
DiskCleanupTask implements Runnable { @Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.persistenceService.getJob(id); if (job.getStatus().isActive()) { continue; } final Instant midnightUTC = TaskUtils.getMidnightUTC(); final Instant retentionThreshold = midnightUTC.minus(this.properties.getRetention(), ChronoUnit.DAYS); final Optional<Instant> finished = job.getFinished(); if (finished.isPresent() && finished.get().isBefore(retentionThreshold)) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); } }
DiskCleanupTask implements Runnable { @Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.persistenceService.getJob(id); if (job.getStatus().isActive()) { continue; } final Instant midnightUTC = TaskUtils.getMidnightUTC(); final Instant retentionThreshold = midnightUTC.minus(this.properties.getRetention(), ChronoUnit.DAYS); final Optional<Instant> finished = job.getFinished(); if (finished.isPresent() && finished.get().isBefore(retentionThreshold)) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); } DiskCleanupTask(
        @NotNull final DiskCleanupProperties properties,
        @NotNull final TaskScheduler scheduler,
        @NotNull final Resource jobsDir,
        @NotNull final DataServices dataServices,
        @NotNull final JobsProperties jobsProperties,
        @NotNull final Executor processExecutor,
        @NotNull final MeterRegistry registry
    ); }
DiskCleanupTask implements Runnable { @Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.persistenceService.getJob(id); if (job.getStatus().isActive()) { continue; } final Instant midnightUTC = TaskUtils.getMidnightUTC(); final Instant retentionThreshold = midnightUTC.minus(this.properties.getRetention(), ChronoUnit.DAYS); final Optional<Instant> finished = job.getFinished(); if (finished.isPresent() && finished.get().isBefore(retentionThreshold)) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); } DiskCleanupTask(
        @NotNull final DiskCleanupProperties properties,
        @NotNull final TaskScheduler scheduler,
        @NotNull final Resource jobsDir,
        @NotNull final DataServices dataServices,
        @NotNull final JobsProperties jobsProperties,
        @NotNull final Executor processExecutor,
        @NotNull final MeterRegistry registry
    ); @Override void run(); }
DiskCleanupTask implements Runnable { @Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.persistenceService.getJob(id); if (job.getStatus().isActive()) { continue; } final Instant midnightUTC = TaskUtils.getMidnightUTC(); final Instant retentionThreshold = midnightUTC.minus(this.properties.getRetention(), ChronoUnit.DAYS); final Optional<Instant> finished = job.getFinished(); if (finished.isPresent() && finished.get().isBefore(retentionThreshold)) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); } DiskCleanupTask(
        @NotNull final DiskCleanupProperties properties,
        @NotNull final TaskScheduler scheduler,
        @NotNull final Resource jobsDir,
        @NotNull final DataServices dataServices,
        @NotNull final JobsProperties jobsProperties,
        @NotNull final Executor processExecutor,
        @NotNull final MeterRegistry registry
    ); @Override void run();  }