Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.raid; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingDeque; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedAvatarFileSystem; import org.apache.hadoop.raid.protocol.PolicyInfo; /** * Traverses the directory tree and gets the desired FileStatus specified by * a given {@link DirectoryTraversal.Filter}. This class is not thread safe. */ public class DirectoryTraversal { static final public Log LOG = LogFactory.getLog(DirectoryTraversal.class); static final public FileStatus FINISH_TOKEN = new FileStatus(); static final int OUTPUT_QUEUE_SIZE = 10000; final private FileSystem fs; final private DistributedAvatarFileSystem avatarFs; final private BlockingQueue<FileStatus> output; final private BlockingDeque<Path> directories; final private Filter filter; final private Processor[] processors; final private AtomicInteger totalDirectories; final private AtomicInteger activeThreads; final private boolean doShuffle; final private boolean allowStandby; private volatile boolean finished = false; // For enabling/disabling avatar. private static final boolean ENABLE_AVATAR_STANDBY = false; /** * Filters the elements to output */ public interface Filter { boolean check(FileStatus f) throws IOException; } public DirectoryTraversal(Collection<Path> roots, FileSystem fs, Filter filter, int numThreads, boolean doShuffle) throws IOException { this(DirectoryTraversal.class.getSimpleName(), roots, fs, filter, numThreads, doShuffle); } public DirectoryTraversal(String friendlyName, Collection<Path> roots, FileSystem fs, Filter filter, int numThreads, boolean doShuffle) throws IOException { this(friendlyName, roots, fs, filter, numThreads, doShuffle, false, false); } public DirectoryTraversal(String friendlyName, Collection<Path> roots, FileSystem fs, Filter filter, int numThreads, boolean doShuffle, boolean allowUseStandby) throws IOException { this(friendlyName, roots, fs, filter, numThreads, doShuffle, allowUseStandby, false); } public DirectoryTraversal(String friendlyName, Collection<Path> roots, FileSystem fs, Filter filter, int numThreads, boolean doShuffle, boolean allowUseStandby, boolean checkLeafDir) throws IOException { this.output = new ArrayBlockingQueue<FileStatus>(OUTPUT_QUEUE_SIZE); this.directories = new LinkedBlockingDeque<Path>(); this.fs = fs; if (ENABLE_AVATAR_STANDBY && allowUseStandby && fs instanceof DistributedAvatarFileSystem) { avatarFs = (DistributedAvatarFileSystem) fs; } else { avatarFs = null; } this.filter = filter; this.totalDirectories = new AtomicInteger(roots.size()); this.processors = new Processor[numThreads]; this.activeThreads = new AtomicInteger(numThreads); this.doShuffle = doShuffle; this.allowStandby = allowUseStandby; if (doShuffle) { List<Path> toShuffleAndAdd = new ArrayList<Path>(); toShuffleAndAdd.addAll(roots); Collections.shuffle(toShuffleAndAdd); this.directories.addAll(toShuffleAndAdd); } else { this.directories.addAll(roots); } LOG.info("Starting with directories:" + roots.toString() + " numThreads:" + numThreads); if (roots.isEmpty()) { try { output.put(FINISH_TOKEN); } catch (InterruptedException e) { throw new IOException(e); } return; } for (int i = 0; i < processors.length; ++i) { if (checkLeafDir) { processors[i] = new LeafDirectoryProcessor(); } else { processors[i] = new Processor(); } processors[i].setName(friendlyName + i); } for (int i = 0; i < processors.length; ++i) { processors[i].start(); } } /** * Retrieves the next filtered element. * Returns {@link FINISH_TOKEN} when traversal is done. Calling this after * {@link FINISH_TOKEN} is returned is not allowed. */ public FileStatus next() throws IOException { if (finished) { LOG.warn("Should not call next() after FINISH_TOKEN is obtained."); return FINISH_TOKEN; } FileStatus f = null; try { f = output.take(); } catch (InterruptedException e) { finished = true; interruptProcessors(); throw new IOException(e); } if (f == FINISH_TOKEN) { LOG.info("traversal is done. Returning FINISH_TOKEN"); finished = true; } return f; } private void interruptProcessors() { for (Thread processor : processors) { if (processor != null) { processor.interrupt(); } } } private class LeafDirectoryProcessor extends Processor { @Override protected void filterDirectory(Path dir, List<Path> subDirs, List<FileStatus> filtered) throws IOException { subDirs.clear(); filtered.clear(); if (dir == null) { return; } FileStatus[] elements; if (avatarFs != null) { elements = avatarFs.listStatus(dir, true); } else { elements = fs.listStatus(dir); } cache.clear(); if (elements != null) { boolean isLeafDir = true; for (FileStatus element : elements) { if (element.isDir()) { subDirs.add(element.getPath()); isLeafDir = false; } } if (isLeafDir && elements.length > 0) { FileStatus dirStat = avatarFs != null ? avatarFs.getFileStatus(dir) : fs.getFileStatus(dir); if (filter.check(dirStat)) { filtered.add(dirStat); } } } } } private class Processor extends Thread { /* This cache is used to reduce the number of RPC calls, instead of running listLocatedStatus for each file, * We run listLocatedStatus for files' parent path and cache the results. Because one processor processes the * files under the same directory, only few RPC call is needed to get LocatedFileStatus of these files. * Please check PlacementMonitor.getLocatedFileStatus for more details. */ protected HashMap<String, LocatedFileStatus> cache; @Override public void run() { this.cache = PlacementMonitor.locatedFileStatusCache.get(); List<Path> subDirs = new ArrayList<Path>(); List<FileStatus> filtered = new ArrayList<FileStatus>(); try { while (!finished && totalDirectories.get() > 0) { Path dir = null; try { dir = directories.poll(1000L, TimeUnit.MILLISECONDS); } catch (InterruptedException e) { continue; } if (dir == null) { continue; } try { filterDirectory(dir, subDirs, filtered); } catch (Throwable ex) { LOG.error(getName() + " throws Throwable. Skip " + dir, ex); totalDirectories.decrementAndGet(); continue; } int numOfDirectoriesChanged = -1 + subDirs.size(); if (totalDirectories.addAndGet(numOfDirectoriesChanged) == 0) { interruptProcessors(); } submitOutputs(filtered, subDirs); } } finally { // clear the cache to avoid memory leak cache.clear(); PlacementMonitor.locatedFileStatusCache.remove(); int active = activeThreads.decrementAndGet(); if (active == 0) { while (true) { try { output.put(FINISH_TOKEN); break; } catch (InterruptedException e) { } } } } } protected void filterDirectory(Path dir, List<Path> subDirs, List<FileStatus> filtered) throws IOException { subDirs.clear(); filtered.clear(); if (dir == null) { return; } FileStatus[] elements; if (avatarFs != null) { elements = avatarFs.listStatus(dir, true); } else { elements = fs.listStatus(dir); } cache.clear(); if (elements != null) { for (FileStatus element : elements) { if (filter.check(element)) { filtered.add(element); } if (element.isDir()) { subDirs.add(element.getPath()); } } } } /** * Submit filtered result to output and directories. Will swallow interrupt * unless {@link finished} is set to true. */ private void submitOutputs(List<FileStatus> filtered, List<Path> subDirs) { if (doShuffle) { Collections.shuffle(subDirs); } for (Path subDir : subDirs) { while (!finished) { try { directories.putFirst(subDir); break; } catch (InterruptedException e) { } } } for (FileStatus out : filtered) { while (!finished) { try { output.put(out); break; } catch (InterruptedException e) { } } } } } public static DirectoryTraversal fileRetriever(List<Path> roots, FileSystem fs, int numThreads, boolean doShuffle, boolean allowUseStandby) throws IOException { Filter filter = new Filter() { @Override public boolean check(FileStatus f) throws IOException { return !f.isDir(); } }; return new DirectoryTraversal("File Retriever ", roots, fs, filter, numThreads, doShuffle, allowUseStandby); } public static DirectoryTraversal directoryRetriever(List<Path> roots, FileSystem fs, int numThreads, boolean doShuffle, boolean allowUseStandby, boolean checkLeafDir) throws IOException { Filter filter = new Filter() { @Override public boolean check(FileStatus f) throws IOException { return f.isDir(); } }; return new DirectoryTraversal("Directory Retriever ", roots, fs, filter, numThreads, doShuffle, allowUseStandby, checkLeafDir); } public static DirectoryTraversal directoryRetriever(List<Path> roots, FileSystem fs, int numThreads, boolean doShuffle, boolean allowUseStandby) throws IOException { return directoryRetriever(roots, fs, numThreads, doShuffle, allowUseStandby, false); } public static DirectoryTraversal directoryRetriever(List<Path> roots, FileSystem fs, int numThreads, boolean doShuffle) throws IOException { return directoryRetriever(roots, fs, numThreads, doShuffle, false); } public static DirectoryTraversal raidFileRetriever(final PolicyInfo info, List<Path> roots, Collection<PolicyInfo> allInfos, Configuration conf, int numThreads, boolean doShuffle, boolean allowStandby) throws IOException { final RaidState.Checker checker = new RaidState.Checker(allInfos, conf); Filter filter = new Filter() { @Override public boolean check(FileStatus f) throws IOException { long now = RaidNode.now(); if (f.isDir()) { return false; } RaidState state = checker.check(info, f, now, false); LOG.debug(f.getPath().toUri().getPath() + " : " + state); return state == RaidState.NOT_RAIDED_BUT_SHOULD; } }; FileSystem fs = new Path(Path.SEPARATOR).getFileSystem(conf); return new DirectoryTraversal("Raid File Retriever ", roots, fs, filter, numThreads, doShuffle, allowStandby, false); } public static DirectoryTraversal raidLeafDirectoryRetriever(final PolicyInfo info, List<Path> roots, Collection<PolicyInfo> allInfos, final Configuration conf, int numThreads, boolean doShuffle, boolean allowStandby) throws IOException { final RaidState.Checker checker = new RaidState.Checker(allInfos, conf); final FileSystem fs = FileSystem.get(conf); Filter filter = new Filter() { @Override public boolean check(FileStatus f) throws IOException { long now = RaidNode.now(); if (!f.isDir()) { return false; } List<FileStatus> lfs = RaidNode.listDirectoryRaidFileStatus(conf, fs, f.getPath()); RaidState state = checker.check(info, f, now, false, lfs); if (LOG.isDebugEnabled()) { LOG.debug(f.getPath() + " : " + state); } return state == RaidState.NOT_RAIDED_BUT_SHOULD; } }; return new DirectoryTraversal("Raid File Retriever ", roots, fs, filter, numThreads, doShuffle, allowStandby, true); } }