Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.raid; import java.io.BufferedReader; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStreamReader; import java.io.PrintStream; import java.lang.reflect.Constructor; import java.net.InetSocketAddress; import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; import java.util.TreeMap; import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadFactory; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.regex.Matcher; import java.util.regex.Pattern; import javax.security.auth.login.LoginException; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.tools.DFSck; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.RPC; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.JobInProgress; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.CounterGroup; import org.apache.hadoop.mapreduce.Counters; import org.apache.hadoop.mapreduce.InputSplit; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.lib.input.FileSplit; import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat; import org.apache.hadoop.mapreduce.lib.output.SequenceFileOutputFormat; import org.apache.hadoop.raid.BlockReconstructor.CorruptBlockReconstructor; import org.apache.hadoop.raid.DistBlockIntegrityMonitor.Worker.LostFileInfo; import org.apache.hadoop.raid.LogUtils.LOGRESULTS; import org.apache.hadoop.raid.LogUtils.LOGTYPES; import org.apache.hadoop.raid.RaidUtils.RaidInfo; import org.apache.hadoop.raid.protocol.RaidProtocol; import org.apache.hadoop.security.UnixUserGroupInformation; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.ToolRunner; /** * distributed block integrity monitor, uses parity to reconstruct lost files * * configuration options * raid.blockfix.filespertask - number of files to reconstruct in a single * map reduce task (i.e., at one mapper node) * * raid.blockfix.fairscheduler.pool - the pool to use for MR jobs * * raid.blockfix.maxpendingjobs - maximum number of MR jobs * running simultaneously */ public class DistBlockIntegrityMonitor extends BlockIntegrityMonitor { public final static String[] BLOCKFIXER_MAPREDUCE_KEYS = { "mapred.job.tracker", "cm.server.address", "cm.server.http.address", "mapred.job.tracker.corona.proxyaddr", "corona.proxy.job.tracker.rpcaddr", "corona.system.dir", "mapred.temp.dir" }; public final static String BLOCKFIXER = "blockfixer"; private static final String IN_FILE_SUFFIX = ".in"; private static final String PART_PREFIX = "part-"; static final Pattern LIST_CORRUPT_FILE_PATTERN = Pattern.compile("blk_-*\\d+\\s+(.*)"); static final Pattern LIST_DECOMMISSION_FILE_PATTERN = Pattern.compile("blk_-*\\d+\\s+(.*)"); // For now this is the same because of how dfsck generates output private static final String FILES_PER_TASK = "raid.blockfix.filespertask"; public static final String MAX_PENDING_JOBS = "raid.blockfix.maxpendingjobs"; private static final String HIGH_PRI_SCHEDULER_OPTION = "raid.blockfix.highpri.scheduleroption"; private static final String LOW_PRI_SCHEDULER_OPTION = "raid.blockfix.lowpri.scheduleroption"; private static final String LOWEST_PRI_SCHEDULER_OPTION = "raid.blockfix.lowestpri.scheduleroption"; private static final String MAX_FIX_TIME_FOR_FILE = "raid.blockfix.max.fix.time.for.file"; private static final String LOST_FILES_LIMIT = "raid.blockfix.corruptfiles.limit"; private static final String RAIDNODE_BLOCK_FIXER_SCAN_NUM_THREADS_KEY = "raid.block.fixer.scan.threads"; private static final int DEFAULT_BLOCK_FIXER_SCAN_NUM_THREADS = 5; private int blockFixerScanThreads = DEFAULT_BLOCK_FIXER_SCAN_NUM_THREADS; // The directories checked by the corrupt file monitor, seperate by comma public static final String RAIDNODE_BLOCK_FIX_SUBMISSION_INTERVAL_KEY = "raid.block.fix.submission.interval"; private static final long DEFAULT_BLOCK_FIX_SUBMISSION_INTERVAL = 5 * 1000; public static final String RAIDNODE_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL_KEY = "raid.block.fix.scan.submission.interval"; private static final long DEFAULT_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL = 5 * 1000; public static final String RAIDNODE_MAX_NUM_DETECTION_TIME_COLLECTED_KEY = "raid.max.num.detection.time.collected"; public static final int DEFAULT_RAIDNODE_MAX_NUM_DETECTION_TIME_COLLECTED = 100; public enum UpdateNumFilesDropped { SET, ADD }; // default number of files to reconstruct in a task private static final long DEFAULT_FILES_PER_TASK = 10L; private static final int TASKS_PER_JOB = 50; // default number of files to reconstruct simultaneously private static final long DEFAULT_MAX_PENDING_JOBS = 100L; private static final long DEFAULT_MAX_FIX_TIME_FOR_FILE = 4 * 60 * 60 * 1000; // 4 hrs. private static final int DEFAULT_LOST_FILES_LIMIT = 200000; public static final String FAILED_FILE = "failed"; public static final String SIMULATION_FAILED_FILE = "simulation_failed"; protected static final Log LOG = LogFactory.getLog(DistBlockIntegrityMonitor.class); private static final String CORRUPT_FILE_DETECT_TIME = "corrupt_detect_time"; // number of files to reconstruct in a task private long filesPerTask; // number of files to reconstruct simultaneously final private long maxPendingJobs; final private long maxFixTimeForFile; final private int lostFilesLimit; private static final SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); private Worker corruptionWorker = new CorruptionWorker(); private Worker decommissioningWorker = new DecommissioningWorker(); private Runnable corruptFileCounterWorker = new CorruptFileCounter(); static enum RaidCounter { FILES_SUCCEEDED, FILES_FAILED, FILES_NOACTION, BLOCK_FIX_SIMULATION_FAILED, BLOCK_FIX_SIMULATION_SUCCEEDED, FILE_FIX_NUM_READBYTES_REMOTERACK } static enum CorruptFileStatus { POTENTIALLY_CORRUPT, RAID_UNRECOVERABLE, NOT_RAIDED_UNRECOVERABLE, NOT_EXIST, RECOVERABLE } static enum Priority { HIGH(HIGH_PRI_SCHEDULER_OPTION, 2), LOW(LOW_PRI_SCHEDULER_OPTION, 1), LOWEST(LOWEST_PRI_SCHEDULER_OPTION, 0); public final String configOption; private final int underlyingValue; private Priority(String s, int value) { configOption = s; underlyingValue = value; } public boolean higherThan(Priority other) { return (underlyingValue > other.underlyingValue); } } static public class TrackingUrlInfo { String trackingUrl; long insertTime; public TrackingUrlInfo(String newUrl, long newTime) { trackingUrl = newUrl; insertTime = newTime; } } /** * Hold information about a failed file with task id */ static public class FailedFileInfo { String taskId; LostFileInfo fileInfo; public FailedFileInfo(String newTaskId, LostFileInfo newFileInfo) { this.taskId = newTaskId; this.fileInfo = newFileInfo; } } public DistBlockIntegrityMonitor(Configuration conf) throws Exception { super(conf); filesPerTask = DistBlockIntegrityMonitor.getFilesPerTask(getConf()); maxPendingJobs = DistBlockIntegrityMonitor.getMaxPendingJobs(getConf()); maxFixTimeForFile = DistBlockIntegrityMonitor.getMaxFixTimeForFile(getConf()); lostFilesLimit = DistBlockIntegrityMonitor.getLostFilesLimit(getConf()); } public static void updateBlockFixerMapreduceConfigs(Configuration conf, String suffix) { for (String configKey : BLOCKFIXER_MAPREDUCE_KEYS) { String newKey = configKey + "." + suffix; String value = conf.get(newKey); if (value != null) { conf.set(configKey, value); } } } /** * determines how many files to reconstruct in a single task */ protected static long getFilesPerTask(Configuration conf) { return conf.getLong(FILES_PER_TASK, DEFAULT_FILES_PER_TASK); } /** * determines how many files to reconstruct simultaneously */ protected static long getMaxPendingJobs(Configuration conf) { return conf.getLong(MAX_PENDING_JOBS, DEFAULT_MAX_PENDING_JOBS); } protected static long getMaxFixTimeForFile(Configuration conf) { return conf.getLong(MAX_FIX_TIME_FOR_FILE, DEFAULT_MAX_FIX_TIME_FOR_FILE); } protected static int getLostFilesLimit(Configuration conf) { return conf.getInt(LOST_FILES_LIMIT, DEFAULT_LOST_FILES_LIMIT); } // Return true if succeed to start one job public static Job startOneJob(Worker newWorker, Priority pri, Set<String> jobFiles, long detectTime, AtomicLong numFilesSubmitted, AtomicLong lastCheckingTime, long maxPendingJobs) throws IOException, InterruptedException, ClassNotFoundException { if (lastCheckingTime != null) { lastCheckingTime.set(System.currentTimeMillis()); } String startTimeStr = dateFormat.format(new Date()); String jobName = newWorker.JOB_NAME_PREFIX + "." + newWorker.jobCounter + "." + pri + "-pri" + "." + startTimeStr; Job job = null; synchronized (jobFiles) { if (jobFiles.size() == 0) { return null; } newWorker.jobCounter++; synchronized (newWorker.jobIndex) { if (newWorker.jobIndex.size() >= maxPendingJobs) { // full return null; } job = newWorker.startJob(jobName, jobFiles, pri, detectTime); } numFilesSubmitted.addAndGet(jobFiles.size()); jobFiles.clear(); } return job; } public abstract class Worker implements Runnable { protected Map<String, LostFileInfo> fileIndex = Collections .synchronizedMap(new HashMap<String, LostFileInfo>()); protected Map<JobID, TrackingUrlInfo> idToTrakcingUrlMap = Collections .synchronizedMap(new HashMap<JobID, TrackingUrlInfo>()); protected Map<Job, List<LostFileInfo>> jobIndex = Collections .synchronizedMap(new HashMap<Job, List<LostFileInfo>>()); protected Map<Job, List<FailedFileInfo>> failJobIndex = new HashMap<Job, List<FailedFileInfo>>(); protected Map<Job, List<FailedFileInfo>> simFailJobIndex = new HashMap<Job, List<FailedFileInfo>>(); private long jobCounter = 0; private AtomicInteger numJobsRunning = new AtomicInteger(0); protected AtomicLong numFilesDropped = new AtomicLong(0); volatile BlockIntegrityMonitor.Status lastStatus = null; AtomicLong recentNumFilesSucceeded = new AtomicLong(); AtomicLong recentNumFilesFailed = new AtomicLong(); AtomicLong recentSlotSeconds = new AtomicLong(); AtomicLong recentNumBlockFixSimulationSucceeded = new AtomicLong(); AtomicLong recentNumBlockFixSimulationFailed = new AtomicLong(); AtomicLong recentNumReadBytesRemoteRack = new AtomicLong(); Map<String, Long> recentLogMetrics = Collections.synchronizedMap(new HashMap<String, Long>()); private static final int POOL_SIZE = 2; private final ExecutorService executor = Executors.newFixedThreadPool(POOL_SIZE); private static final int DEFAULT_CHECK_JOB_TIMEOUT_SEC = 600; //10 mins protected final Log LOG; protected final Class<? extends BlockReconstructor> RECONSTRUCTOR_CLASS; protected final String JOB_NAME_PREFIX; protected Worker(Log log, Class<? extends BlockReconstructor> rClass, String prefix) { this.LOG = log; this.RECONSTRUCTOR_CLASS = rClass; this.JOB_NAME_PREFIX = prefix; Path workingDir = new Path(prefix); try { FileSystem fs = workingDir.getFileSystem(getConf()); // Clean existing working dir fs.delete(workingDir, true); } catch (IOException ioe) { LOG.warn("Get exception when cleaning " + workingDir, ioe); } } public void shutdown() { } /** * runs the worker periodically */ public void run() { try { while (running) { try { updateStatus(); checkAndReconstructBlocks(); } catch (InterruptedException ignore) { LOG.info("interrupted"); } catch (Exception e) { // log exceptions and keep running LOG.error(StringUtils.stringifyException(e)); } catch (Error e) { LOG.error(StringUtils.stringifyException(e)); throw e; } try { Thread.sleep(blockCheckInterval); } catch (InterruptedException ignore) { LOG.info("interrupted"); } } } finally { shutdown(); } } /** * checks for lost blocks and reconstructs them (if any) */ void checkAndReconstructBlocks() throws Exception { checkJobsWithTimeOut(DEFAULT_CHECK_JOB_TIMEOUT_SEC); int size = jobIndex.size(); if (size >= maxPendingJobs) { LOG.info("Waiting for " + size + " pending jobs"); return; } FileSystem fs = new Path("/").getFileSystem(getConf()); Map<String, Integer> lostFiles = getLostFiles(fs); long detectTime = System.currentTimeMillis(); computePrioritiesAndStartJobs(fs, lostFiles, detectTime); } /** * Handle a failed job. */ private void failJob(Job job) { // assume no files have been reconstructed LOG.error("Job " + job.getID() + "(" + job.getJobName() + ") finished (failed)"); // We do not change metrics here since we do not know for sure if file // reconstructing failed. for (LostFileInfo fileInfo : jobIndex.get(job)) { boolean failed = true; addToMap(job, job.getID().toString(), fileInfo, failJobIndex); fileInfo.finishJob(job.getJobName(), failed); } numJobsRunning.decrementAndGet(); } private void addToMap(Job job, String taskId, LostFileInfo fileInfo, Map<Job, List<FailedFileInfo>> index) { List<FailedFileInfo> failFiles = null; if (!index.containsKey(job)) { failFiles = new ArrayList<FailedFileInfo>(); index.put(job, failFiles); } else { failFiles = index.get(job); } failFiles.add(new FailedFileInfo(taskId, fileInfo)); } /** * Handle a successful job. */ private void succeedJob(Job job, long filesSucceeded, long filesFailed) throws IOException { String jobName = job.getJobName(); LOG.info("Job " + job.getID() + "(" + jobName + ") finished (succeeded)"); // we have to look at the output to check which files have failed HashMap<String, String> failedFiles = getFailedFiles(job); for (LostFileInfo fileInfo : jobIndex.get(job)) { String filePath = fileInfo.getFile().toString(); String failedFilePath = DistBlockIntegrityMonitor.FAILED_FILE + "," + filePath; String simulatedFailedFilePath = DistBlockIntegrityMonitor.SIMULATION_FAILED_FILE + "," + filePath; if (failedFiles.containsKey(simulatedFailedFilePath)) { String taskId = failedFiles.get(simulatedFailedFilePath); addToMap(job, taskId, fileInfo, simFailJobIndex); LOG.error("Simulation failed file: " + fileInfo.getFile()); } if (failedFiles.containsKey(failedFilePath)) { String taskId = failedFiles.get(failedFilePath); addToMap(job, taskId, fileInfo, failJobIndex); boolean failed = true; fileInfo.finishJob(jobName, failed); } else { // call succeed for files that have succeeded or for which no action // was taken boolean failed = false; fileInfo.finishJob(jobName, failed); } } // report succeeded files to metrics this.recentNumFilesSucceeded.addAndGet(filesSucceeded); this.recentNumFilesFailed.addAndGet(filesFailed); if (filesSucceeded > 0) { lastSuccessfulFixTime = System.currentTimeMillis(); } numJobsRunning.decrementAndGet(); } /** * Check the jobs with timeout */ void checkJobsWithTimeOut(int timeoutSec) throws ExecutionException { Future<Boolean> future = executor.submit(new Callable<Boolean>() { @Override public Boolean call() throws Exception { checkJobs(); return true; } }); try { future.get(timeoutSec, TimeUnit.SECONDS); } catch (TimeoutException e) { // ignore this. LOG.warn("Timeout when checking jobs' status."); } catch (InterruptedException e) { // ignore this. LOG.warn("checkJobs function is interrupted."); } if (!future.isDone()) { future.cancel(true); } } /** * checks if jobs have completed and updates job and file index * returns a list of failed files for restarting */ void checkJobs() throws IOException { List<Job> nonRunningJobs = new ArrayList<Job>(); synchronized (jobIndex) { Iterator<Job> jobIter = jobIndex.keySet().iterator(); while (jobIter.hasNext()) { Job job = jobIter.next(); try { if (job.isComplete()) { Counters ctrs = job.getCounters(); if (ctrs != null) { // If we got counters, perform extra validation. this.recentSlotSeconds.addAndGet( ctrs.findCounter(JobInProgress.Counter.SLOTS_MILLIS_MAPS).getValue() / 1000); long filesSucceeded = ctrs.findCounter(RaidCounter.FILES_SUCCEEDED) != null ? ctrs.findCounter(RaidCounter.FILES_SUCCEEDED).getValue() : 0; long filesFailed = ctrs.findCounter(RaidCounter.FILES_FAILED) != null ? ctrs.findCounter(RaidCounter.FILES_FAILED).getValue() : 0; long filesNoAction = ctrs.findCounter(RaidCounter.FILES_NOACTION) != null ? ctrs.findCounter(RaidCounter.FILES_NOACTION).getValue() : 0; long blockFixSimulationFailed = ctrs .findCounter(RaidCounter.BLOCK_FIX_SIMULATION_FAILED) != null ? ctrs.findCounter(RaidCounter.BLOCK_FIX_SIMULATION_FAILED) .getValue() : 0; long blockFixSimulationSucceeded = ctrs .findCounter(RaidCounter.BLOCK_FIX_SIMULATION_SUCCEEDED) != null ? ctrs.findCounter(RaidCounter.BLOCK_FIX_SIMULATION_SUCCEEDED) .getValue() : 0; this.recentNumBlockFixSimulationFailed.addAndGet(blockFixSimulationFailed); this.recentNumBlockFixSimulationSucceeded.addAndGet(blockFixSimulationSucceeded); long fileFixNumReadBytesRemoteRack = ctrs .findCounter(RaidCounter.FILE_FIX_NUM_READBYTES_REMOTERACK) != null ? ctrs.findCounter(RaidCounter.FILE_FIX_NUM_READBYTES_REMOTERACK) .getValue() : 0; this.recentNumReadBytesRemoteRack.addAndGet(fileFixNumReadBytesRemoteRack); CounterGroup counterGroup = ctrs.getGroup(LogUtils.LOG_COUNTER_GROUP_NAME); for (Counter ctr : counterGroup) { Long curVal = ctr.getValue(); if (this.recentLogMetrics.containsKey(ctr.getName())) { curVal += this.recentLogMetrics.get(ctr.getName()); } this.recentLogMetrics.put(ctr.getName(), curVal); } int files = jobIndex.get(job).size(); if (job.isSuccessful() && (filesSucceeded + filesFailed + filesNoAction == ((long) files))) { // job has processed all files succeedJob(job, filesSucceeded, filesFailed); } else { failJob(job); } } else { long filesSucceeded = jobIndex.get(job).size(); long filesFailed = 0; if (job.isSuccessful()) { succeedJob(job, filesSucceeded, filesFailed); } else { failJob(job); } } jobIter.remove(); nonRunningJobs.add(job); } else { LOG.info("Job " + job.getID() + "(" + job.getJobName() + " still running"); } } catch (Exception e) { LOG.error(StringUtils.stringifyException(e)); failJob(job); jobIter.remove(); nonRunningJobs.add(job); try { job.killJob(); } catch (Exception ee) { LOG.error(StringUtils.stringifyException(ee)); } } } } purgeFileIndex(); cleanupNonRunningJobs(nonRunningJobs); } /** * Delete (best-effort) the input and output directories of jobs. * @param nonRunningJobs */ private void cleanupNonRunningJobs(List<Job> nonRunningJobs) { for (Job job : nonRunningJobs) { Path outDir = null; try { outDir = SequenceFileOutputFormat.getOutputPath(job); outDir.getFileSystem(getConf()).delete(outDir, true); } catch (IOException e) { LOG.warn("Could not delete output dir " + outDir, e); } Path[] inDir = null; try { // We only create one input directory. inDir = ReconstructionInputFormat.getInputPaths(job); inDir[0].getFileSystem(getConf()).delete(inDir[0], true); } catch (IOException e) { LOG.warn("Could not delete input dir " + inDir[0], e); } } } /** * determines which files have failed for a given job */ private HashMap<String, String> getFailedFiles(Job job) throws IOException { HashMap<String, String> failedFiles = new HashMap<String, String>(); Path outDir = SequenceFileOutputFormat.getOutputPath(job); FileSystem fs = outDir.getFileSystem(getConf()); if (!fs.getFileStatus(outDir).isDir()) { throw new IOException(outDir.toString() + " is not a directory"); } FileStatus[] files = fs.listStatus(outDir); for (FileStatus f : files) { Path fPath = f.getPath(); if ((!f.isDir()) && (fPath.getName().startsWith(PART_PREFIX))) { LOG.info("opening " + fPath.toString()); SequenceFile.Reader reader = new SequenceFile.Reader(fs, fPath, getConf()); Text key = new Text(); Text value = new Text(); while (reader.next(key, value)) { if (LOG.isDebugEnabled()) { LOG.debug("key: " + key.toString() + " , value: " + value.toString()); } failedFiles.put(key.toString(), value.toString()); } reader.close(); } } return failedFiles; } /** * purge expired jobs from the file index */ private void purgeFileIndex() { Iterator<String> fileIter = fileIndex.keySet().iterator(); long now = System.currentTimeMillis(); while (fileIter.hasNext()) { String file = fileIter.next(); if (fileIndex.get(file).isTooOld(now)) { fileIter.remove(); } } Iterator<TrackingUrlInfo> tuiIter = this.idToTrakcingUrlMap.values().iterator(); while (tuiIter.hasNext()) { TrackingUrlInfo tui = tuiIter.next(); if (System.currentTimeMillis() - tui.insertTime > maxWindowTime) { tuiIter.remove(); } } } // Start jobs for all the lost files. public void startJobs(Map<String, Priority> filePriorities, long detectTime) throws IOException, InterruptedException, ClassNotFoundException { AtomicLong numFilesSubmitted = new AtomicLong(0); for (Priority pri : Priority.values()) { Set<String> jobFiles = new HashSet<String>(); for (Map.Entry<String, Priority> entry : filePriorities.entrySet()) { // Check if file priority matches the current round. if (entry.getValue() != pri) { continue; } jobFiles.add(entry.getKey()); // Check if we have hit the threshold for number of files in a job. if (jobFiles.size() == filesPerTask * TASKS_PER_JOB) { boolean succeed = startOneJob(this, pri, jobFiles, detectTime, numFilesSubmitted, null, maxPendingJobs) != null; if (!succeed) { this.numFilesDropped.set(filePriorities.size() - numFilesSubmitted.get()); LOG.debug("Submitted a job with max number of files allowed. " + "Num files dropped is " + this.numFilesDropped.get()); return; } } } if (jobFiles.size() > 0) { boolean succeed = startOneJob(this, pri, jobFiles, detectTime, numFilesSubmitted, null, maxPendingJobs) != null; if (!succeed) { this.numFilesDropped.set(filePriorities.size() - numFilesSubmitted.get()); LOG.debug("Submitted a job with max number of files allowed. " + "Num files dropped is " + this.numFilesDropped.get()); return; } } } this.numFilesDropped.set(filePriorities.size() - numFilesSubmitted.get()); } /** * creates and submits a job, updates file index and job index */ private Job startJob(String jobName, Set<String> lostFiles, Priority priority, long detectTime) throws IOException, InterruptedException, ClassNotFoundException { Path inDir = new Path(JOB_NAME_PREFIX + "/in/" + jobName); Path outDir = new Path(JOB_NAME_PREFIX + "/out/" + jobName); List<String> filesInJob = createInputFile(jobName, inDir, lostFiles); if (filesInJob.isEmpty()) return null; Configuration jobConf = new Configuration(getConf()); DistBlockIntegrityMonitor.updateBlockFixerMapreduceConfigs(jobConf, BLOCKFIXER); RaidUtils.parseAndSetOptions(jobConf, priority.configOption); Job job = new Job(jobConf, jobName); job.getConfiguration().set(CORRUPT_FILE_DETECT_TIME, Long.toString(detectTime)); configureJob(job, this.RECONSTRUCTOR_CLASS); job.setJarByClass(getClass()); job.setMapperClass(ReconstructionMapper.class); job.setNumReduceTasks(0); job.setInputFormatClass(ReconstructionInputFormat.class); job.setOutputFormatClass(SequenceFileOutputFormat.class); job.setOutputKeyClass(Text.class); job.setOutputValueClass(Text.class); ReconstructionInputFormat.setInputPaths(job, inDir); SequenceFileOutputFormat.setOutputPath(job, outDir); submitJob(job, filesInJob, priority); List<LostFileInfo> fileInfos = updateFileIndex(jobName, filesInJob, priority); // The implementation of submitJob() need not update jobIndex. // So check if the job exists in jobIndex before updating jobInfos. if (jobIndex.containsKey(job)) { jobIndex.put(job, fileInfos); } numJobsRunning.incrementAndGet(); return job; } void submitJob(Job job, List<String> filesInJob, Priority priority) throws IOException, InterruptedException, ClassNotFoundException { LOG.info("Submitting job"); DistBlockIntegrityMonitor.this.submitJob(job, filesInJob, priority, this.jobIndex, this.idToTrakcingUrlMap); } /** * inserts new job into file index and job index */ private List<LostFileInfo> updateFileIndex(String jobName, List<String> lostFiles, Priority priority) { List<LostFileInfo> fileInfos = new ArrayList<LostFileInfo>(); for (String file : lostFiles) { LostFileInfo fileInfo = fileIndex.get(file); if (fileInfo != null) { fileInfo.addJob(jobName, priority); } else { fileInfo = new LostFileInfo(file, jobName, priority); fileIndex.put(file, fileInfo); } fileInfos.add(fileInfo); } return fileInfos; } /** * creates the input file (containing the names of the files to be * reconstructed) */ private List<String> createInputFile(String jobName, Path inDir, Set<String> lostFiles) throws IOException { Path file = new Path(inDir, jobName + IN_FILE_SUFFIX); FileSystem fs = file.getFileSystem(getConf()); SequenceFile.Writer fileOut = SequenceFile.createWriter(fs, getConf(), file, LongWritable.class, Text.class); long index = 0L; List<String> filesAdded = new ArrayList<String>(); int count = 0; for (String lostFileName : lostFiles) { fileOut.append(new LongWritable(index++), new Text(lostFileName)); filesAdded.add(lostFileName); count++; if (index % filesPerTask == 0) { fileOut.sync(); // create sync point to make sure we can split here } } fileOut.close(); return filesAdded; } /** * Update {@link lastStatus} so that it can be viewed from outside */ protected void updateStatus() { int highPriorityFiles = 0; int lowPriorityFiles = 0; int lowestPriorityFiles = 0; List<JobStatus> jobs = new ArrayList<JobStatus>(); List<JobStatus> failJobs = new ArrayList<JobStatus>(); List<JobStatus> simFailJobs = new ArrayList<JobStatus>(); List<String> highPriorityFileNames = new ArrayList<String>(); for (Map.Entry<String, LostFileInfo> e : fileIndex.entrySet()) { String fileName = e.getKey(); LostFileInfo fileInfo = e.getValue(); Priority pri = fileInfo.getHighestPriority(); if (pri == Priority.HIGH) { highPriorityFileNames.add(fileName); highPriorityFiles++; } else if (pri == Priority.LOW) { lowPriorityFiles++; } else if (pri == Priority.LOWEST) { lowestPriorityFiles++; } } synchronized (jobIndex) { for (Job job : jobIndex.keySet()) { String url = job.getTrackingURL(); String name = job.getJobName(); JobID jobId = job.getID(); jobs.add(new BlockIntegrityMonitor.JobStatus(jobId, name, url, jobIndex.get(job), null)); } for (Job job : failJobIndex.keySet()) { String url = job.getTrackingURL(); String name = job.getJobName(); JobID jobId = job.getID(); failJobs.add( new BlockIntegrityMonitor.JobStatus(jobId, name, url, null, failJobIndex.get(job))); } for (Job simJob : simFailJobIndex.keySet()) { String url = simJob.getTrackingURL(); String name = simJob.getJobName(); JobID jobId = simJob.getID(); simFailJobs.add(new BlockIntegrityMonitor.JobStatus(jobId, name, url, null, simFailJobIndex.get(simJob))); } } lastStatus = new BlockIntegrityMonitor.Status(highPriorityFiles, lowPriorityFiles, lowestPriorityFiles, jobs, highPriorityFileNames, failJobs, simFailJobs); updateRaidNodeMetrics(); } public Status getStatus() { return lastStatus; } abstract void computePrioritiesAndStartJobs(FileSystem fs, Map<String, Integer> lostFiles, long detectTime) throws IOException, InterruptedException, ClassNotFoundException; protected abstract Map<String, Integer> getLostFiles(FileSystem fs) throws IOException; protected abstract void updateRaidNodeMetrics(); /** * hold information about a lost file that is being reconstructed */ class LostFileInfo { private String file; private List<String> jobNames; // Jobs reconstructing this file. private boolean done; private List<Priority> priorities; private long insertTime; public LostFileInfo(String file, String jobName, Priority priority) { this.file = file; this.jobNames = new ArrayList<String>(); this.priorities = new ArrayList<Priority>(); this.done = false; this.insertTime = System.currentTimeMillis(); addJob(jobName, priority); } public boolean isTooOld(long now) { return now - insertTime > maxFixTimeForFile; } public boolean isDone() { return done; } public void addJob(String jobName, Priority priority) { this.jobNames.add(jobName); this.priorities.add(priority); } public Priority getHighestPriority() { Priority max = Priority.LOWEST; for (Priority p : priorities) { if (p.higherThan(max)) max = p; } return max; } public String getFile() { return file; } /** * Updates state with the completion of a job. If all jobs for this file * are done, the file index is updated. */ public void finishJob(String jobName, boolean failed) { int idx = jobNames.indexOf(jobName); if (idx == -1) return; jobNames.remove(idx); priorities.remove(idx); LOG.info("reconstructing " + file + (failed ? " failed in " : " succeeded in ") + jobName); if (jobNames.isEmpty()) { // All jobs dealing with this file are done, // remove this file from the index LostFileInfo removed = fileIndex.remove(file); if (removed == null) { LOG.error("trying to remove file not in file index: " + file); } done = true; } } } public String getTrackingUrl(JobID jobId) { TrackingUrlInfo tui = this.idToTrakcingUrlMap.get(jobId); if (tui == null) { return ""; } else { return tui.trackingUrl; } } } /** * CorruptFileCounter is a periodical running daemon that keeps running raidfsck * to get the number of the corrupt files under the give directories defined by * RAIDNODE_CORRUPT_FILE_COUNTER_DIRECTORIES_KEY * @author weiyan * */ public class CorruptFileCounter implements Runnable { private long filesWithMissingBlksCnt = 0; private Map<String, long[]> numStrpWithMissingBlksMap = new HashMap<String, long[]>(); private Object counterMapLock = new Object(); private long numNonRaidedMissingBlocks = 0; public CorruptFileCounter() { for (Codec codec : Codec.getCodecs()) { this.numStrpWithMissingBlksMap.put(codec.id, new long[codec.stripeLength + codec.parityLength]); } } public void run() { RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).initCorruptFilesMetrics(getConf()); while (running) { TreeMap<String, Long> newUnRecoverableCounterMap = new TreeMap<String, Long>(); Map<String, Long> newRecoverableCounterMap = new HashMap<String, Long>(); long newfilesWithMissingBlksCnt = 0; String srcDir = "/"; try { ByteArrayOutputStream bout = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(bout, true); RaidShell shell = new RaidShell(getConf(), ps); int res = ToolRunner.run(shell, new String[] { "-fsck", srcDir, "-count", "-retNumStrpsMissingBlks" }); shell.close(); ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray()); BufferedReader reader = new BufferedReader(new InputStreamReader(bin)); String line = reader.readLine(); if (line == null) { throw new IOException("Raidfsck fails without output"); } Long corruptCount = Long.parseLong(line); LOG.info("The number of corrupt files under " + srcDir + " is " + corruptCount); newUnRecoverableCounterMap.put(srcDir, corruptCount); line = reader.readLine(); if (line == null) { throw new IOException("Raidfsck did not print number " + "of files with missing blocks"); } // get files with Missing Blks // fsck with '-count' prints this number in line2 long incfilesWithMissingBlks = Long.parseLong(line); LOG.info("The number of files with missing blocks under " + srcDir + " is " + incfilesWithMissingBlks); long numRecoverableFiles = incfilesWithMissingBlks - corruptCount; newRecoverableCounterMap.put(srcDir, numRecoverableFiles); approximateNumRecoverableFiles = numRecoverableFiles; // Add filesWithMissingBlks and numStrpWithMissingBlks only for "/" // dir to avoid duplicates Map<String, long[]> newNumStrpWithMissingBlksMap = new HashMap<String, long[]>(); newfilesWithMissingBlksCnt += incfilesWithMissingBlks; // read the array for num stripes with missing blocks line = reader.readLine(); if (line == null) { throw new IOException( "Raidfsck did not print the number of " + "missing blocks in non raided files"); } long numNonRaided = Long.parseLong(line); for (int i = 0; i < Codec.getCodecs().size(); i++) { line = reader.readLine(); if (line == null) { throw new IOException( "Raidfsck did not print the missing " + "block info for codec at index " + i); } Codec codec = Codec.getCodec(line); long[] incNumStrpWithMissingBlks = new long[codec.stripeLength + codec.parityLength]; for (int j = 0; j < incNumStrpWithMissingBlks.length; j++) { line = reader.readLine(); if (line == null) { throw new IOException("Raidfsck did not print the array " + "for number stripes with missing blocks for index " + j); } incNumStrpWithMissingBlks[j] = Long.parseLong(line); LOG.info("The number of stripes with missing blocks at index" + j + "under" + srcDir + " is " + incNumStrpWithMissingBlks[j]); } newNumStrpWithMissingBlksMap.put(codec.id, incNumStrpWithMissingBlks); } synchronized (counterMapLock) { this.numNonRaidedMissingBlocks = numNonRaided; for (String codeId : newNumStrpWithMissingBlksMap.keySet()) { numStrpWithMissingBlksMap.put(codeId, newNumStrpWithMissingBlksMap.get(codeId)); } } reader.close(); bin.close(); } catch (Exception e) { LOG.error("Fail to count the corrupt files under " + srcDir, e); } synchronized (counterMapLock) { this.filesWithMissingBlksCnt = newfilesWithMissingBlksCnt; } updateRaidNodeMetrics(); if (!running) { break; } try { Thread.sleep(corruptFileCountInterval); } catch (InterruptedException ignore) { LOG.info("interrupted"); } } } public long getNumNonRaidedMissingBlks() { synchronized (counterMapLock) { return this.numNonRaidedMissingBlocks; } } public long getFilesWithMissingBlksCnt() { synchronized (counterMapLock) { return filesWithMissingBlksCnt; } } public long[] getNumStrpWithMissingBlksRS() { synchronized (counterMapLock) { return numStrpWithMissingBlksMap.get("rs"); } } protected void updateRaidNodeMetrics() { RaidNodeMetrics rnm = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID); synchronized (counterMapLock) { rnm.numFilesWithMissingBlks.set(this.filesWithMissingBlksCnt); long[] numStrpWithMissingBlksRS = this.numStrpWithMissingBlksMap.get("rs"); if (numStrpWithMissingBlksRS != null) { rnm.numStrpsOneMissingBlk.set(numStrpWithMissingBlksRS[0]); rnm.numStrpsTwoMissingBlk.set(numStrpWithMissingBlksRS[1]); rnm.numStrpsThreeMissingBlk.set(numStrpWithMissingBlksRS[2]); rnm.numStrpsFourMissingBlk.set(numStrpWithMissingBlksRS[3]); long tmp_sum = 0; for (int idx = 4; idx < numStrpWithMissingBlksRS.length; idx++) { tmp_sum += numStrpWithMissingBlksRS[idx]; } rnm.numStrpsFiveMoreMissingBlk.set(tmp_sum); } } } public String getMissingBlksHtmlTable() { synchronized (counterMapLock) { return RaidUtils.getMissingBlksHtmlTable(this.numNonRaidedMissingBlocks, this.numStrpWithMissingBlksMap); } } } /** * Get the lost blocks numbers per stripe in the source file. */ private Map<Integer, Integer> getLostStripes(Configuration conf, FileStatus stat, FileSystem fs) throws IOException { Map<Integer, Integer> lostStripes = new HashMap<Integer, Integer>(); RaidInfo raidInfo = RaidUtils.getFileRaidInfo(stat, conf); if (raidInfo.codec == null) { // Can not find the parity file, the file is not raided. return lostStripes; } Codec codec = raidInfo.codec; if (codec.isDirRaid) { RaidUtils.collectDirectoryCorruptBlocksInStripe(conf, (DistributedFileSystem) fs, raidInfo, stat, lostStripes); } else { RaidUtils.collectFileCorruptBlocksInStripe((DistributedFileSystem) fs, raidInfo, stat, lostStripes); } return lostStripes; } public class CorruptFile { public String path; public long detectTime; public volatile int numCorrupt; public volatile CorruptFileStatus fileStatus; public volatile long lastSubmitTime; public CorruptFile(String newPath, int newNumCorrupt, long newDetectTime) { this.path = newPath; this.numCorrupt = newNumCorrupt; this.fileStatus = CorruptFileStatus.POTENTIALLY_CORRUPT; this.lastSubmitTime = System.currentTimeMillis(); this.detectTime = newDetectTime; } public String toString() { return fileStatus.name(); } } public class MonitorSet { public ConcurrentHashMap<String, CorruptFile> toScanFiles; public ExecutorService executor; public BlockingQueue<Runnable> scanningQueue; public MonitorSet(final String monitorDir) { this.scanningQueue = new LinkedBlockingQueue<Runnable>(); ThreadFactory factory = new ThreadFactory() { final AtomicInteger numThreads = new AtomicInteger(); public Thread newThread(Runnable r) { Thread t = new Thread(r); t.setName("BlockFix-Scanner-" + monitorDir + "-" + numThreads.getAndIncrement()); return t; } }; this.executor = new ThreadPoolExecutor(blockFixerScanThreads, blockFixerScanThreads, 0L, TimeUnit.MILLISECONDS, scanningQueue, factory); this.toScanFiles = new ConcurrentHashMap<String, CorruptFile>(); } } public class CorruptionWorker extends Worker { public static final String RAIDNODE_JOB_SUBMIT_NUM_THREADS_KEY = "raid.job.submit.num.threads"; public static final int RAIDNODE_JOB_SUBMIT_NUM_THREADS_DEFAULT = 5; public String[] corruptMonitorDirs = null; public HashMap<String, MonitorSet> monitorSets; public final String OTHERS = "others"; public HashMap<Priority, HashSet<String>> jobFilesMap; public HashMap<Priority, AtomicLong> lastCheckingTimes; public AtomicLong numFilesSubmitted = new AtomicLong(0); public AtomicLong totalFilesToSubmit = new AtomicLong(0); private long blockFixSubmissionInterval = DEFAULT_BLOCK_FIX_SUBMISSION_INTERVAL; private long blockFixScanSubmissionInterval = DEFAULT_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL; private int maxNumDetectionTime; // Collection of recent X samples; private long[] detectionTimeCollection; private int currPos; private long totalDetectionTime; private long totalCollecitonSize; private ExecutorService jobSubmitExecutor; private BlockingQueue<Runnable> jobSubmitQueue; private int jobSubmitThreads = RAIDNODE_JOB_SUBMIT_NUM_THREADS_DEFAULT; public CorruptionWorker() { super(LogFactory.getLog(CorruptionWorker.class), CorruptBlockReconstructor.class, "blockfixer"); blockFixerScanThreads = getConf().getInt(RAIDNODE_BLOCK_FIXER_SCAN_NUM_THREADS_KEY, DEFAULT_BLOCK_FIXER_SCAN_NUM_THREADS); this.blockFixSubmissionInterval = getConf().getLong(RAIDNODE_BLOCK_FIX_SUBMISSION_INTERVAL_KEY, DEFAULT_BLOCK_FIX_SUBMISSION_INTERVAL); this.blockFixScanSubmissionInterval = getConf().getLong(RAIDNODE_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL_KEY, DEFAULT_BLOCK_FIX_SCAN_SUBMISSION_INTERVAL); this.corruptMonitorDirs = DistBlockIntegrityMonitor.getCorruptMonitorDirs(getConf()); this.monitorSets = new HashMap<String, MonitorSet>(); for (String monitorDir : this.corruptMonitorDirs) { this.monitorSets.put(monitorDir, new MonitorSet(monitorDir)); } this.monitorSets.put(OTHERS, new MonitorSet(OTHERS)); this.jobFilesMap = new HashMap<Priority, HashSet<String>>(); lastCheckingTimes = new HashMap<Priority, AtomicLong>(); for (Priority priority : Priority.values()) { this.jobFilesMap.put(priority, new HashSet<String>()); this.lastCheckingTimes.put(priority, new AtomicLong(System.currentTimeMillis())); } this.maxNumDetectionTime = getConf().getInt(RAIDNODE_MAX_NUM_DETECTION_TIME_COLLECTED_KEY, DEFAULT_RAIDNODE_MAX_NUM_DETECTION_TIME_COLLECTED); detectionTimeCollection = new long[maxNumDetectionTime]; this.totalCollecitonSize = 0; this.totalDetectionTime = 0; this.currPos = 0; this.jobSubmitThreads = getConf().getInt(RAIDNODE_JOB_SUBMIT_NUM_THREADS_KEY, RAIDNODE_JOB_SUBMIT_NUM_THREADS_DEFAULT); this.jobSubmitQueue = new LinkedBlockingQueue<Runnable>(); ThreadFactory factory = new ThreadFactory() { final AtomicInteger numThreads = new AtomicInteger(); public Thread newThread(Runnable r) { Thread t = new Thread(r); t.setName("BlockFix-Job-Submit-" + numThreads.getAndIncrement()); return t; } }; this.jobSubmitExecutor = new ThreadPoolExecutor(this.jobSubmitThreads, this.jobSubmitThreads, 0L, TimeUnit.MILLISECONDS, this.jobSubmitQueue, factory); } public void putDetectionTime(long detectionTime) { synchronized (detectionTimeCollection) { long oldVal = detectionTimeCollection[currPos]; detectionTimeCollection[currPos] = detectionTime; totalDetectionTime += detectionTime - oldVal; currPos++; if (currPos == maxNumDetectionTime) { currPos = 0; } if (totalCollecitonSize < maxNumDetectionTime) { totalCollecitonSize++; } } } public double getNumDetectionsPerSec() { synchronized (detectionTimeCollection) { if (totalCollecitonSize == 0) { return 0; } else { return ((double) totalCollecitonSize) * 1000 / totalDetectionTime * blockFixerScanThreads; } } } @Override protected Map<String, Integer> getLostFiles(FileSystem fs) throws IOException { Map<String, Integer> lostFiles = new HashMap<String, Integer>(); RemoteIterator<Path> cfb = fs.listCorruptFileBlocks(new Path("/")); while (cfb.hasNext()) { String lostFile = cfb.next().toString(); Integer count = lostFiles.get(lostFile); if (count == null) { lostFiles.put(lostFile, 1); } else { lostFiles.put(lostFile, count + 1); } } LOG.info("ListCorruptFileBlocks returned " + lostFiles.size() + " files"); RaidUtils.filterTrash(getConf(), lostFiles.keySet().iterator()); LOG.info("getLostFiles returning " + lostFiles.size() + " files"); return lostFiles; } public void addToScanSet(String p, int numCorrupt, String monitorDir, ConcurrentHashMap<String, CorruptFile> newScanSet, FileSystem fs, long detectTime) throws IOException { CorruptFile cf = new CorruptFile(p, numCorrupt, detectTime); MonitorSet monitorSet = monitorSets.get(monitorDir); CorruptFile oldCf = monitorSet.toScanFiles.get(p); FileCheckRunnable fcr = new FileCheckRunnable(cf, monitorSet, fs, detectTime, this); if (oldCf == null) { newScanSet.put(p, cf); monitorSet.toScanFiles.put(p, cf); // Check the file cf.lastSubmitTime = System.currentTimeMillis(); monitorSet.executor.submit(fcr); } else { if (oldCf.numCorrupt == numCorrupt) { newScanSet.put(p, oldCf); if (System.currentTimeMillis() - oldCf.lastSubmitTime > this.blockFixScanSubmissionInterval) { // if a block hasn't been checked for a while, check it again. oldCf.lastSubmitTime = System.currentTimeMillis(); monitorSet.executor.submit(fcr); } } else { cf.detectTime = oldCf.detectTime; newScanSet.put(p, cf); cf.lastSubmitTime = System.currentTimeMillis(); monitorSet.executor.submit(fcr); } } } /** * In JobSubmitRunnable, a mapreduce job to fix files under tmpJobFiles will * be created and submitted to mapreduce cluster. * If it fails to do that, numFilesDropped will be updated and files under * tmpJobFiles will be move back to the original jobFiles so that they could * be fixed in the next job. */ public class JobSubmitRunnable implements Runnable { private final Priority priority; private final HashSet<String> tmpJobFiles; private final HashSet<String> jobFiles; private final long detectTime; private final AtomicLong lastCheckingTime; private final UpdateNumFilesDropped type; public JobSubmitRunnable(Priority newPriority, HashSet<String> tmpJobFiles, HashSet<String> originalJobFiles, long newDetectTime, AtomicLong newLastCheckingTime, UpdateNumFilesDropped newType) { this.priority = newPriority; this.tmpJobFiles = tmpJobFiles; this.jobFiles = originalJobFiles; this.detectTime = newDetectTime; this.lastCheckingTime = newLastCheckingTime; this.type = newType; } public void run() { boolean succeed = false; try { succeed = startOneJob(CorruptionWorker.this, priority, tmpJobFiles, detectTime, numFilesSubmitted, lastCheckingTime, maxPendingJobs) != null; } catch (Throwable ex) { LOG.error("Get Error in blockSubmitRunnable", ex); } finally { if (!succeed) { if (type == UpdateNumFilesDropped.SET) { numFilesDropped.set(tmpJobFiles.size()); } else if (type == UpdateNumFilesDropped.ADD) { numFilesDropped.addAndGet(tmpJobFiles.size()); } else { LOG.error("Hit an unexpected type:" + type.name()); } // add back to original job files synchronized (jobFiles) { this.jobFiles.addAll(tmpJobFiles); } } } } } // Return used time public long addToJobFilesMap(HashMap<Priority, HashSet<String>> jobFilesMap, Priority priority, String path, long detectTime) throws IOException, InterruptedException, ClassNotFoundException { long startTime = System.currentTimeMillis(); HashSet<String> jobFiles = jobFilesMap.get(priority); synchronized (jobFiles) { if (!jobFiles.add(path)) { return System.currentTimeMillis() - startTime; } totalFilesToSubmit.incrementAndGet(); // Check if we have hit the threshold for number of files in a job. AtomicLong lastCheckingTime = lastCheckingTimes.get(priority); if ((jobFiles.size() >= filesPerTask * TASKS_PER_JOB)) { // Collect enough files this.asyncSubmitJob(jobFiles, priority, detectTime, UpdateNumFilesDropped.ADD); } else if (System.currentTimeMillis() - lastCheckingTime.get() > this.blockFixSubmissionInterval && jobFiles.size() > 0) { // Wait enough time this.asyncSubmitJob(jobFiles, priority, detectTime, UpdateNumFilesDropped.SET); } } return System.currentTimeMillis() - startTime; } @Override public void shutdown() { for (MonitorSet ms : monitorSets.values()) { ms.executor.shutdownNow(); } this.jobSubmitExecutor.shutdownNow(); } public Map<String, Map<CorruptFileStatus, Long>> getCounterMap() { TreeMap<String, Map<CorruptFileStatus, Long>> results = new TreeMap<String, Map<CorruptFileStatus, Long>>(); for (String monitorDir : monitorSets.keySet()) { MonitorSet ms = monitorSets.get(monitorDir); HashMap<CorruptFileStatus, Long> counters = new HashMap<CorruptFileStatus, Long>(); for (CorruptFileStatus cfs : CorruptFileStatus.values()) { counters.put(cfs, 0L); } for (CorruptFile cf : ms.toScanFiles.values()) { Long counter = counters.get(cf.fileStatus); if (counter == null) { counter = 0L; } counters.put(cf.fileStatus, counter + 1); } results.put(monitorDir, counters); } return results; } public ArrayList<CorruptFile> getCorruptFileList(String monitorDir, CorruptFileStatus cfs) { ArrayList<CorruptFile> corruptFiles = new ArrayList<CorruptFile>(); MonitorSet ms = monitorSets.get(monitorDir); if (ms == null) { return corruptFiles; } for (CorruptFile cf : ms.toScanFiles.values()) { if (cf.fileStatus == cfs) { corruptFiles.add(cf); } } return corruptFiles; } public Map<String, Map<CorruptFileStatus, Long>> getCorruptFilesCounterMap() { return this.getCounterMap(); } public class FileCheckRunnable implements Runnable { CorruptFile corruptFile; MonitorSet monitorSet; FileSystem fs; CorruptionWorker worker; long detectTime; public FileCheckRunnable(CorruptFile newCorruptFile, MonitorSet newMonitorSet, FileSystem newFs, long newDetectTime, CorruptionWorker newWorker) { corruptFile = newCorruptFile; monitorSet = newMonitorSet; fs = newFs; detectTime = newDetectTime; worker = newWorker; } public void run() { long startTime = System.currentTimeMillis(); try { if (corruptFile.numCorrupt <= 0) { // Not corrupt return; } ConcurrentHashMap<String, CorruptFile> toScanFiles = monitorSet.toScanFiles; // toScanFiles could be switched before the task get executed CorruptFile cf = toScanFiles.get(corruptFile.path); if (cf == null || cf.numCorrupt != corruptFile.numCorrupt) { // Not exist or doesn't match return; } FileStatus stat = null; try { stat = fs.getFileStatus(new Path(corruptFile.path)); } catch (FileNotFoundException fnfe) { cf.fileStatus = CorruptFileStatus.NOT_EXIST; return; } Codec codec = BlockIntegrityMonitor.isParityFile(corruptFile.path); long addJobTime = 0; if (codec == null) { if (stat.getReplication() >= notRaidedReplication) { cf.fileStatus = CorruptFileStatus.NOT_RAIDED_UNRECOVERABLE; return; } if (BlockIntegrityMonitor.doesParityDirExist(fs, corruptFile.path)) { Priority priority = Priority.LOW; if (stat.getReplication() > 1) { // If we have a missing block when replication > 1, it is high pri. priority = Priority.HIGH; } else { // Replication == 1. Assume Reed Solomon parity exists. // If we have more than one missing block when replication == 1, then // high pri. priority = (corruptFile.numCorrupt > 1) ? Priority.HIGH : Priority.LOW; } LostFileInfo fileInfo = fileIndex.get(corruptFile.path); if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) { addJobTime = addToJobFilesMap(jobFilesMap, priority, corruptFile.path, detectTime); } } } else { // Dikang: for parity files, we use the total numbers for now. Priority priority = (corruptFile.numCorrupt > 1) ? Priority.HIGH : (codec.parityLength == 1) ? Priority.HIGH : Priority.LOW; LostFileInfo fileInfo = fileIndex.get(corruptFile.path); if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) { addJobTime = addToJobFilesMap(jobFilesMap, priority, corruptFile.path, detectTime); } } boolean isFileCorrupt = RaidShell.isFileCorrupt((DistributedFileSystem) fs, stat, false, getConf(), null, null); if (isFileCorrupt) { cf.fileStatus = CorruptFileStatus.RAID_UNRECOVERABLE; } else { cf.fileStatus = CorruptFileStatus.RECOVERABLE; } long elapseTime = System.currentTimeMillis() - startTime - addJobTime; worker.putDetectionTime(elapseTime); } catch (Exception e) { LOG.error("Get Exception ", e); } } } /** * Acquire a lock and dump files of jobFiles into a tmpJobFiles * Then it clears the jobFiles and submits a jobSubmitRunnable to the thread pool * to submit a mapreduce job in the background. * No need to wait for job submission to finish. */ void asyncSubmitJob(HashSet<String> jobFiles, Priority pri, long detectTime, UpdateNumFilesDropped type) throws IOException { synchronized (jobFiles) { if (jobFiles.size() == 0) return; HashSet<String> tmpJobFiles = new HashSet<String>(); tmpJobFiles.addAll(jobFiles); jobFiles.clear(); JobSubmitRunnable jsr = new JobSubmitRunnable(pri, tmpJobFiles, jobFiles, detectTime, lastCheckingTimes.get(pri), type); this.jobSubmitExecutor.submit(jsr); } } @Override // Compute integer priority and start jobs. Urgency is indicated by higher numbers. void computePrioritiesAndStartJobs(FileSystem fs, Map<String, Integer> corruptFiles, long detectTime) throws IOException, InterruptedException, ClassNotFoundException { HashMap<String, ConcurrentHashMap<String, CorruptFile>> newToScanSet = new HashMap<String, ConcurrentHashMap<String, CorruptFile>>(); // Include "others" for (String monitorDir : this.monitorSets.keySet()) { newToScanSet.put(monitorDir, new ConcurrentHashMap<String, CorruptFile>()); } numFilesSubmitted.set(0); totalFilesToSubmit.set(0); for (Iterator<String> it = corruptFiles.keySet().iterator(); it.hasNext();) { String p = it.next(); int numCorrupt = corruptFiles.get(p); // Filter through monitor dirs boolean match = false; for (String monitorDir : this.corruptMonitorDirs) { if (p.startsWith(monitorDir)) { match = true; addToScanSet(p, numCorrupt, monitorDir, newToScanSet.get(monitorDir), fs, detectTime); } } if (match == false) { addToScanSet(p, numCorrupt, OTHERS, newToScanSet.get(OTHERS), fs, detectTime); } } // switch to new toScanSet for (String monitorDir : this.monitorSets.keySet()) { MonitorSet ms = this.monitorSets.get(monitorDir); ms.toScanFiles = newToScanSet.get(monitorDir); } for (Priority pri : Priority.values()) { HashSet<String> jobFiles = jobFilesMap.get(pri); if (System.currentTimeMillis() - lastCheckingTimes.get(pri).get() > this.blockFixSubmissionInterval && jobFiles.size() > 0) { this.asyncSubmitJob(jobFiles, pri, detectTime, UpdateNumFilesDropped.SET); } } } @Override protected void updateRaidNodeMetrics() { RaidNodeMetrics rnm = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID); rnm.corruptFilesHighPri.set(lastStatus.highPriorityFiles); rnm.corruptFilesLowPri.set(lastStatus.lowPriorityFiles); rnm.numFilesToFix.set(this.fileIndex.size()); rnm.numFilesToFixDropped.set(this.numFilesDropped.get()); // Flush statistics out to the RaidNode incrFilesFixed(this.recentNumFilesSucceeded.get()); incrFileFixFailures(this.recentNumFilesFailed.get()); incrNumBlockFixSimulationFailures(this.recentNumBlockFixSimulationFailed.get()); incrNumBlockFixSimulationSuccess(this.recentNumBlockFixSimulationSucceeded.get()); incrFileFixReadBytesRemoteRack(this.recentNumReadBytesRemoteRack.get()); LogUtils.incrLogMetrics(this.recentLogMetrics); rnm.blockFixSlotSeconds.inc(this.recentSlotSeconds.get()); this.recentNumFilesSucceeded.set(0); this.recentNumFilesFailed.set(0); this.recentSlotSeconds.set(0); this.recentNumBlockFixSimulationFailed.set(0); this.recentNumBlockFixSimulationSucceeded.set(0); this.recentNumReadBytesRemoteRack.set(0); this.recentLogMetrics.clear(); Map<String, Map<CorruptFileStatus, Long>> corruptFilesCounterMap = this.getCounterMap(); if (rnm.corruptFiles == null) { return; } for (String dir : this.corruptMonitorDirs) { if (corruptFilesCounterMap.containsKey(dir) && rnm.corruptFiles.containsKey(dir)) { Map<CorruptFileStatus, Long> maps = corruptFilesCounterMap.get(dir); Long raidUnrecoverable = maps.get(CorruptFileStatus.RAID_UNRECOVERABLE); Long notRaidUnrecoverable = maps.get(CorruptFileStatus.NOT_RAIDED_UNRECOVERABLE); if (raidUnrecoverable == null) { raidUnrecoverable = 0L; } if (notRaidUnrecoverable == null) { notRaidUnrecoverable = 0L; } rnm.corruptFiles.get(dir).set(raidUnrecoverable + notRaidUnrecoverable); } else { rnm.corruptFiles.get(dir).set(-1L); } } } } public class DecommissioningWorker extends Worker { DecommissioningWorker() { super(LogFactory.getLog(DecommissioningWorker.class), BlockReconstructor.DecommissioningBlockReconstructor.class, "blockcopier"); } /** * gets a list of decommissioning files from the namenode * and filters out files that are currently being regenerated or * that were recently regenerated */ @Override protected Map<String, Integer> getLostFiles(FileSystem fs) throws IOException { return DistBlockIntegrityMonitor.this.getLostFiles(LIST_DECOMMISSION_FILE_PATTERN, new String[] { "-list-corruptfileblocks", "-list-decommissioningblocks", "-limit", new Integer(lostFilesLimit).toString() }); } @Override void computePrioritiesAndStartJobs(FileSystem fs, Map<String, Integer> decommissioningFiles, long detectTime) throws IOException, InterruptedException, ClassNotFoundException { Map<String, Priority> fileToPriority = new HashMap<String, Priority>(decommissioningFiles.size()); for (String file : decommissioningFiles.keySet()) { // Replication == 1. Assume Reed Solomon parity exists. // Files with more than 4 blocks being decommissioned get a bump. // Otherwise, copying jobs have the lowest priority. Priority priority = ((decommissioningFiles.get(file) > Codec.getCodec("rs").parityLength) ? Priority.LOW : Priority.LOWEST); LostFileInfo fileInfo = fileIndex.get(file); if (fileInfo == null || priority.higherThan(fileInfo.getHighestPriority())) { fileToPriority.put(file, priority); } } LOG.info("Found " + fileToPriority.size() + " new lost files"); startJobs(fileToPriority, detectTime); } @Override protected void updateRaidNodeMetrics() { RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).decomFilesLowPri .set(lastStatus.highPriorityFiles); RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).decomFilesLowestPri .set(lastStatus.lowPriorityFiles); RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).numFilesToCopy.set(fileIndex.size()); incrFilesCopied(recentNumFilesSucceeded.get()); incrFileCopyFailures(recentNumFilesFailed.get()); incrNumBlockFixSimulationFailures(this.recentNumBlockFixSimulationFailed.get()); incrNumBlockFixSimulationSuccess(this.recentNumBlockFixSimulationSucceeded.get()); LogUtils.incrLogMetrics(this.recentLogMetrics); RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID).blockCopySlotSeconds .inc(recentSlotSeconds.get()); // Reset temporary values now that they've been flushed recentNumFilesSucceeded.set(0); recentNumFilesFailed.set(0); recentSlotSeconds.set(0); recentNumBlockFixSimulationFailed.set(0); recentNumBlockFixSimulationSucceeded.set(0); recentLogMetrics.clear(); } } // ---- Methods which can be overridden by tests ---- /** * Gets a list of lost files from the name node via DFSck * * @param pattern A pattern matching a single file in DFSck's output * @param dfsckArgs Arguments to pass to DFSck * @return A map of lost files' filenames to num lost blocks for that file */ protected Map<String, Integer> getLostFiles(Pattern pattern, String[] dfsckArgs) throws IOException { Map<String, Integer> lostFiles = new HashMap<String, Integer>(); BufferedReader reader = getLostFileReader(dfsckArgs); String line = reader.readLine(); // remove the header line while ((line = reader.readLine()) != null) { Matcher m = pattern.matcher(line); if (!m.find()) { continue; } String fileName = m.group(1).trim(); Integer numLost = lostFiles.get(fileName); numLost = numLost == null ? 0 : numLost; numLost += 1; lostFiles.put(fileName, numLost); } LOG.info("FSCK returned " + lostFiles.size() + " files with args " + Arrays.toString(dfsckArgs)); RaidUtils.filterTrash(getConf(), lostFiles.keySet().iterator()); LOG.info("getLostFiles returning " + lostFiles.size() + " files with args " + Arrays.toString(dfsckArgs)); return lostFiles; } private BufferedReader getLostFileReader(String[] dfsckArgs) throws IOException { ByteArrayOutputStream bout = new ByteArrayOutputStream(); PrintStream ps = new PrintStream(bout, true); DFSck dfsck = new DFSck(getConf(), ps); try { dfsck.run(dfsckArgs); } catch (Exception e) { throw new IOException(e); } ByteArrayInputStream bin = new ByteArrayInputStream(bout.toByteArray()); return new BufferedReader(new InputStreamReader(bin)); } public void configureJob(Job job, Class<? extends BlockReconstructor> reconstructorClass) { ((JobConf) job.getConfiguration()).setUser(RaidNode.JOBUSER); ((JobConf) job.getConfiguration()).setClass(ReconstructionMapper.RECONSTRUCTOR_CLASS_TAG, reconstructorClass, BlockReconstructor.class); } void submitJob(Job job, List<String> filesInJob, Priority priority, Map<Job, List<LostFileInfo>> jobIndex, Map<JobID, TrackingUrlInfo> idToTrackingUrlMap) throws IOException, InterruptedException, ClassNotFoundException { job.submit(); LOG.info("Job " + job.getID() + "(" + job.getJobName() + ") started"); jobIndex.put(job, null); idToTrackingUrlMap.put(job.getID(), new TrackingUrlInfo(job.getTrackingURL(), System.currentTimeMillis())); } /** * returns the number of map reduce jobs running */ public int jobsRunning() { return (corruptionWorker.numJobsRunning.get() + decommissioningWorker.numJobsRunning.get()); } static class ReconstructionInputFormat extends SequenceFileInputFormat<LongWritable, Text> { protected static final Log LOG = LogFactory.getLog(ReconstructionMapper.class); /** * splits the input files into tasks handled by a single node * we have to read the input files to do this based on a number of * items in a sequence */ @Override public List<InputSplit> getSplits(JobContext job) throws IOException { long filesPerTask = DistBlockIntegrityMonitor.getFilesPerTask(job.getConfiguration()); Path[] inPaths = getInputPaths(job); List<InputSplit> splits = new ArrayList<InputSplit>(); long fileCounter = 0; for (Path inPath : inPaths) { FileSystem fs = inPath.getFileSystem(job.getConfiguration()); if (!fs.getFileStatus(inPath).isDir()) { throw new IOException(inPath.toString() + " is not a directory"); } FileStatus[] inFiles = fs.listStatus(inPath); for (FileStatus inFileStatus : inFiles) { Path inFile = inFileStatus.getPath(); if (!inFileStatus.isDir() && (inFile.getName().equals(job.getJobName() + IN_FILE_SUFFIX))) { fileCounter++; SequenceFile.Reader inFileReader = new SequenceFile.Reader(fs, inFile, job.getConfiguration()); long startPos = inFileReader.getPosition(); long counter = 0; // create an input split every filesPerTask items in the sequence LongWritable key = new LongWritable(); Text value = new Text(); try { while (inFileReader.next(key, value)) { if (counter % filesPerTask == filesPerTask - 1L) { splits.add(new FileSplit(inFile, startPos, inFileReader.getPosition() - startPos, null)); startPos = inFileReader.getPosition(); } counter++; } // create input split for remaining items if necessary // this includes the case where no splits were created by the loop if (startPos != inFileReader.getPosition()) { splits.add(new FileSplit(inFile, startPos, inFileReader.getPosition() - startPos, null)); } } finally { inFileReader.close(); } } } } LOG.info("created " + splits.size() + " input splits from " + fileCounter + " files"); return splits; } /** * indicates that input file can be split */ @Override public boolean isSplitable(JobContext job, Path file) { return true; } } /** * Mapper for reconstructing stripes with lost blocks */ static class ReconstructionMapper extends Mapper<LongWritable, Text, Text, Text> { protected static final Log LOG = LogFactory.getLog(ReconstructionMapper.class); public static final String RECONSTRUCTOR_CLASS_TAG = "hdfs.blockintegrity.reconstructor"; private BlockReconstructor reconstructor; public RaidProtocol raidnode; private UnixUserGroupInformation ugi; RaidProtocol rpcRaidnode; private long detectTimeInput; private String taskId; void initializeRpc(Configuration conf, InetSocketAddress address) throws IOException { try { this.ugi = UnixUserGroupInformation.login(conf, true); } catch (LoginException e) { throw (IOException) (new IOException().initCause(e)); } this.rpcRaidnode = RaidShell.createRPCRaidnode(address, conf, ugi); this.raidnode = RaidShell.createRaidnode(rpcRaidnode); } @Override protected void setup(Context context) throws IOException, InterruptedException { super.setup(context); Configuration conf = context.getConfiguration(); taskId = conf.get("mapred.task.id"); Codec.initializeCodecs(conf); initializeRpc(conf, RaidNode.getAddress(conf)); Class<? extends BlockReconstructor> reconstructorClass = context.getConfiguration() .getClass(RECONSTRUCTOR_CLASS_TAG, null, BlockReconstructor.class); if (reconstructorClass == null) { LOG.error("No class supplied for reconstructor " + "(prop " + RECONSTRUCTOR_CLASS_TAG + ")"); context.progress(); return; } // We dynamically instantiate the helper based on the helperClass member try { Constructor<? extends BlockReconstructor> ctor = reconstructorClass .getConstructor(new Class[] { Configuration.class }); reconstructor = ctor.newInstance(conf); } catch (Exception ex) { throw new IOException( "Could not instantiate a block reconstructor " + "based on class " + reconstructorClass, ex); } detectTimeInput = Long.parseLong(conf.get("corrupt_detect_time")); } @Override protected void cleanup(Context context) throws IOException, InterruptedException { RPC.stopProxy(rpcRaidnode); } /** * Reconstruct a stripe */ @Override public void map(LongWritable key, Text fileText, Context context) throws IOException, InterruptedException { long sTime = System.currentTimeMillis(); String fileStr = fileText.toString(); Path file = new Path(fileStr); String prefix = "[" + fileStr + "] "; LOG.info(""); LOG.info(prefix + "============================= BEGIN ============================="); LOG.info(prefix + "Reconstruct File: " + fileStr); LOG.info(prefix + "Block Missing Detection Time: " + dateFormat.format(detectTimeInput)); long waitTime = sTime - detectTimeInput; LOG.info(prefix + "Scheduling Time: " + (waitTime / 1000) + " seconds"); FileSystem fs = file.getFileSystem(context.getConfiguration()); LogUtils.logWaitTimeMetrics(waitTime, getMaxPendingJobs(context.getConfiguration()), getFilesPerTask(context.getConfiguration()), LOGTYPES.FILE_FIX_WAITTIME, fs, context); long recoveryTime = -1; try { boolean reconstructed = reconstructor.reconstructFile(file, context); if (reconstructed) { recoveryTime = System.currentTimeMillis() - detectTimeInput; context.getCounter(RaidCounter.FILES_SUCCEEDED).increment(1L); LogUtils.logRaidReconstructionMetrics(LOGRESULTS.SUCCESS, 0, null, file, -1, LOGTYPES.OFFLINE_RECONSTRUCTION_FILE, fs, null, context, recoveryTime); LOG.info(prefix + "File Reconstruction Time: " + ((System.currentTimeMillis() - sTime) / 1000) + " seconds"); LOG.info(prefix + "Total Recovery Time: " + (recoveryTime / 1000) + " seconds"); } else { LOG.info(prefix + "File has already been fixed, No action"); context.getCounter(RaidCounter.FILES_NOACTION).increment(1L); } } catch (Throwable e) { LOG.error(prefix + "Reconstructing file " + file + " failed", e); LogUtils.logRaidReconstructionMetrics(LOGRESULTS.FAILURE, 0, null, file, -1, LOGTYPES.OFFLINE_RECONSTRUCTION_FILE, fs, e, context, -1); recoveryTime = Integer.MAX_VALUE; // report file as failed context.getCounter(RaidCounter.FILES_FAILED).increment(1L); String outkey = DistBlockIntegrityMonitor.FAILED_FILE + "," + fileStr; context.write(new Text(outkey), new Text(taskId)); } finally { if (recoveryTime > 0) { // Send recoveryTime to raidnode try { raidnode.sendRecoveryTime(fileStr, recoveryTime, taskId); } catch (Exception e) { LOG.error(prefix + "Failed to send recovery time ", e); } } LOG.info(prefix + "============================= END ============================="); LOG.info(""); } context.progress(); } } /** * Get the status of the entire block integrity monitor. * The status returned represents the aggregation of the statuses of all the * integrity monitor's components. * * @return The status of the block integrity monitor */ @Override public BlockIntegrityMonitor.Status getAggregateStatus() { Status fixer = corruptionWorker.getStatus(); Status copier = decommissioningWorker.getStatus(); List<JobStatus> jobs = new ArrayList<JobStatus>(); List<JobStatus> simFailedJobs = new ArrayList<JobStatus>(); List<JobStatus> failedJobs = new ArrayList<JobStatus>(); List<String> highPriFileNames = new ArrayList<String>(); int numHighPriFiles = 0; int numLowPriFiles = 0; int numLowestPriFiles = 0; if (fixer != null) { jobs.addAll(fixer.jobs); simFailedJobs.addAll(fixer.simFailJobs); failedJobs.addAll(fixer.failJobs); if (fixer.highPriorityFileNames != null) { highPriFileNames.addAll(fixer.highPriorityFileNames); } numHighPriFiles += fixer.highPriorityFiles; numLowPriFiles += fixer.lowPriorityFiles; numLowestPriFiles += fixer.lowestPriorityFiles; } if (copier != null) { jobs.addAll(copier.jobs); simFailedJobs.addAll(copier.simFailJobs); failedJobs.addAll(copier.failJobs); if (copier.highPriorityFileNames != null) { highPriFileNames.addAll(copier.highPriorityFileNames); } numHighPriFiles += copier.highPriorityFiles; numLowPriFiles += copier.lowPriorityFiles; numLowestPriFiles += copier.lowestPriorityFiles; } return new Status(numHighPriFiles, numLowPriFiles, numLowestPriFiles, jobs, highPriFileNames, failedJobs, simFailedJobs); } public Worker getCorruptionMonitor() { return this.corruptionWorker; } @Override public Worker getDecommissioningMonitor() { return this.decommissioningWorker; } @Override public Runnable getCorruptFileCounter() { return this.corruptFileCounterWorker; } }