Java tutorial
/* * * Copyright 2016 Netflix, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * */ package com.netflix.genie.web.tasks.node; import com.netflix.genie.common.dto.Job; import com.netflix.genie.common.exceptions.GenieException; import com.netflix.genie.core.jobs.JobConstants; import com.netflix.genie.core.properties.JobsProperties; import com.netflix.genie.core.services.JobSearchService; import com.netflix.genie.web.properties.DiskCleanupProperties; import com.netflix.genie.web.tasks.TaskUtils; import com.netflix.spectator.api.Counter; import com.netflix.spectator.api.Registry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.exec.CommandLine; import org.apache.commons.exec.Executor; import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.SystemUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.io.Resource; import org.springframework.scheduling.TaskScheduler; import org.springframework.scheduling.support.CronTrigger; import org.springframework.stereotype.Component; import javax.validation.constraints.NotNull; import java.io.File; import java.io.IOException; import java.util.Calendar; import java.util.Date; import java.util.Optional; import java.util.concurrent.atomic.AtomicLong; /** * This task runs on every Genie node and is responsible for cleaning up the local disk so that space can be * recaptured. */ @ConditionalOnProperty("genie.tasks.databaseCleanup.enabled") @Component @Slf4j public class DiskCleanupTask implements Runnable { private final DiskCleanupProperties properties; private final File jobsDir; private final JobSearchService jobSearchService; private final boolean runAsUser; private final Executor processExecutor; private final AtomicLong numberOfDeletedJobDirs; private final AtomicLong numberOfDirsUnableToDelete; private final Counter unableToGetJobCounter; private final Counter unableToDeleteJobDirCounter; /** * Constructor. Schedules this task to be run by the task scheduler. * * @param properties The disk cleanup properties to use. * @param scheduler The scheduler to use to schedule the cron trigger. * @param jobsDir The resource representing the location of the job directory * @param jobSearchService The service to find jobs with * @param jobsProperties The jobs properties to use * @param processExecutor The process executor to use to delete directories * @param registry The metrics registry * @throws IOException When it is unable to open a file reference to the job directory */ @Autowired public DiskCleanupTask(@NotNull final DiskCleanupProperties properties, @NotNull final TaskScheduler scheduler, @NotNull final Resource jobsDir, @NotNull final JobSearchService jobSearchService, @NotNull final JobsProperties jobsProperties, @NotNull final Executor processExecutor, @NotNull final Registry registry) throws IOException { // Job Directory is guaranteed to exist by the MvcConfig bean creation but just in case someone overrides if (!jobsDir.exists()) { throw new IOException("Jobs dir " + jobsDir + " doesn't exist. Unable to create task to cleanup."); } this.properties = properties; this.jobsDir = jobsDir.getFile(); this.jobSearchService = jobSearchService; this.runAsUser = jobsProperties.getUsers().isRunAsUserEnabled(); this.processExecutor = processExecutor; this.numberOfDeletedJobDirs = registry.gauge("genie.tasks.diskCleanup.numberDeletedJobDirs.gauge", new AtomicLong()); this.numberOfDirsUnableToDelete = registry.gauge("genie.tasks.diskCleanup.numberDirsUnableToDelete.gauge", new AtomicLong()); this.unableToGetJobCounter = registry.counter("genie.tasks.diskCleanup.unableToGetJobs.rate"); this.unableToDeleteJobDirCounter = registry.counter("genie.tasks.diskCleanup.unableToDeleteJobsDir.rate"); // Only schedule the task if we don't need sudo while on a non-unix system if (this.runAsUser && !SystemUtils.IS_OS_UNIX) { log.error("System is not UNIX like. Unable to schedule disk cleanup due to needing Unix commands"); } else { final CronTrigger trigger = new CronTrigger(properties.getExpression(), JobConstants.UTC); scheduler.schedule(this, trigger); } } /** * Checks the disk for jobs on this host. Deletes any job directories that are older than the desired * retention and are complete. */ @Override public void run() { log.info("Running disk cleanup task..."); final File[] jobDirs = this.jobsDir.listFiles(); if (jobDirs == null) { log.warn("No job dirs found. Returning."); this.numberOfDeletedJobDirs.set(0); this.numberOfDirsUnableToDelete.set(0); return; } // For each of the directories figure out if we need to delete the files or not long deletedCount = 0; long unableToDeleteCount = 0; for (final File dir : jobDirs) { if (!dir.isDirectory()) { log.info("File {} isn't a directory. Skipping.", dir.getName()); continue; } final String id = dir.getName(); try { final Job job = this.jobSearchService.getJob(id); if (job.getStatus().isActive()) { // Don't want to delete anything still going continue; } // Delete anything with a finish time before today @12 AM UTC - retention final Calendar retentionThreshold = TaskUtils.getMidnightUTC(); TaskUtils.subtractDaysFromDate(retentionThreshold, this.properties.getRetention()); final Optional<Date> finished = job.getFinished(); if (finished.isPresent() && finished.get().before(retentionThreshold.getTime())) { log.info("Attempting to delete job directory for job {}", id); if (this.runAsUser) { final CommandLine commandLine = new CommandLine("sudo"); commandLine.addArgument("rm"); commandLine.addArgument("-rf"); commandLine.addArgument(dir.getAbsolutePath()); this.processExecutor.execute(commandLine); } else { // Save forking a process ourselves if we don't have to FileUtils.deleteDirectory(dir); } deletedCount++; log.info("Successfully deleted job directory for job {}", id); } } catch (final GenieException ge) { log.error("Unable to get job {}. Continuing.", id, ge); this.unableToGetJobCounter.increment(); unableToDeleteCount++; } catch (final IOException ioe) { log.error("Unable to delete job directory for job with id: {}", id, ioe); this.unableToDeleteJobDirCounter.increment(); unableToDeleteCount++; } } this.numberOfDeletedJobDirs.set(deletedCount); this.numberOfDirsUnableToDelete.set(unableToDeleteCount); } }