Java tutorial
/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.hadoop.raid; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.EnumMap; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Map.Entry; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlock; import org.apache.hadoop.hdfs.protocol.LocatedBlocks; import org.apache.hadoop.hdfs.protocol.LocatedBlocksWithMetaInfo; import org.apache.hadoop.hdfs.protocol.LocatedBlockWithMetaInfo; import org.apache.hadoop.hdfs.protocol.VersionedLocatedBlocks; import org.apache.hadoop.metrics.util.MetricsLongValue; import org.apache.hadoop.raid.protocol.PolicyInfo; import org.apache.hadoop.util.StringUtils; /** * Monitors and potentially fixes placement of blocks in RAIDed files. */ public class PlacementMonitor { public static final Log LOG = LogFactory.getLog(PlacementMonitor.class); /** * Maps number of neighbor blocks to number of blocks */ Map<String, Map<Integer, Long>> blockHistograms; Map<String, Map<Integer, Long>> blockHistogramsPerRack; Configuration conf; private volatile Map<String, Map<Integer, Long>> lastBlockHistograms; private volatile Map<String, Map<Integer, Long>> lastBlockHistogramsPerRack; private volatile long lastUpdateStartTime = 0L; private volatile long lastUpdateFinishTime = 0L; private volatile long lastUpdateUsedTime = 0L; public static ThreadLocal<HashMap<String, LocatedFileStatus>> locatedFileStatusCache = new ThreadLocal<HashMap<String, LocatedFileStatus>>() { @Override protected HashMap<String, LocatedFileStatus> initialValue() { return new HashMap<String, LocatedFileStatus>(); } }; RaidNodeMetrics metrics; BlockMover blockMover; int blockMoveMinRepl = DEFAULT_BLOCK_MOVE_MIN_REPLICATION; final static String NUM_MOVING_THREADS_KEY = "hdfs.raid.block.move.threads"; final static String SIMULATE_KEY = "hdfs.raid.block.move.simulate"; final static String BLOCK_MOVE_QUEUE_LENGTH_KEY = "hdfs.raid.block.move.queue.length"; final static String BLOCK_MOVE_MIN_REPLICATION_KEY = "hdfs.raid.block.move.min.replication"; final static int DEFAULT_NUM_MOVING_THREADS = 10; final static int DEFAULT_BLOCK_MOVE_QUEUE_LENGTH = 30000; final static int ALWAYS_SUBMIT_PRIORITY = 3; final static int DEFAULT_BLOCK_MOVE_MIN_REPLICATION = 2; final static int DEFAULT_NON_RAID_FILE_REPLICATION = 3; PlacementMonitor(Configuration conf) throws IOException { this.conf = conf; createEmptyHistograms(); int numMovingThreads = conf.getInt(NUM_MOVING_THREADS_KEY, DEFAULT_NUM_MOVING_THREADS); int maxMovingQueueSize = conf.getInt(BLOCK_MOVE_QUEUE_LENGTH_KEY, DEFAULT_BLOCK_MOVE_QUEUE_LENGTH); this.blockMoveMinRepl = conf.getInt(BLOCK_MOVE_MIN_REPLICATION_KEY, DEFAULT_BLOCK_MOVE_MIN_REPLICATION); boolean simulate = conf.getBoolean(SIMULATE_KEY, true); blockMover = new BlockMover(numMovingThreads, maxMovingQueueSize, simulate, ALWAYS_SUBMIT_PRIORITY, conf); this.metrics = RaidNodeMetrics.getInstance(RaidNodeMetrics.DEFAULT_NAMESPACE_ID); } private void createEmptyHistograms() { blockHistograms = new HashMap<String, Map<Integer, Long>>(); blockHistogramsPerRack = new HashMap<String, Map<Integer, Long>>(); for (Codec codec : Codec.getCodecs()) { blockHistograms.put(codec.id, new HashMap<Integer, Long>()); blockHistogramsPerRack.put(codec.id, new HashMap<Integer, Long>()); } } public void start() { blockMover.start(); } public void stop() { blockMover.stop(); } public void startCheckingFiles() { lastUpdateStartTime = RaidNode.now(); } public int getMovingQueueSize() { return blockMover.getQueueSize(); } public void checkFile(FileSystem srcFs, FileStatus srcFile, FileSystem parityFs, Path partFile, HarIndex.IndexEntry entry, Codec codec, PolicyInfo policy) throws IOException { if (srcFile.getReplication() > blockMoveMinRepl) { // We only check placement for the file with 0..blockMoveMinRepl replicas. return; } if (srcFs.getUri().equals(parityFs.getUri())) { BlockAndDatanodeResolver resolver = new BlockAndDatanodeResolver(srcFile.getPath(), srcFs, partFile, parityFs); checkBlockLocations(getBlockInfos(srcFs, srcFile), getBlockInfos(parityFs, partFile, entry.startOffset, entry.length), codec, policy, srcFile, resolver); } else { // TODO: Move blocks in two clusters separately LOG.warn("Source and parity are in different file system. " + " source:" + srcFs.getUri() + " parity:" + parityFs.getUri() + ". Skip."); } } /** * Check the block placement info of the src file only. * (This is used for non-raided file) * @throws IOException */ public void checkSrcFile(FileSystem srcFs, FileStatus srcFile) throws IOException { List<BlockInfo> srcBlocks = getBlockInfos(srcFs, srcFile); if (srcBlocks.size() == 0) { return; } BlockAndDatanodeResolver resolver = new BlockAndDatanodeResolver(srcFile.getPath(), srcFs); checkSrcBlockLocations(srcBlocks, srcFile, resolver); } public void checkSrcBlockLocations(List<BlockInfo> srcBlocks, FileStatus srcFile, BlockAndDatanodeResolver resolver) throws IOException { // check the block placement policy for (BlockInfo srcBlock : srcBlocks) { LocatedBlockWithMetaInfo locatedBlock = resolver.getLocatedBlock(srcBlock); DatanodeInfo[] datanodes = locatedBlock.getLocations(); if (datanodes.length != DEFAULT_NON_RAID_FILE_REPLICATION) { continue; } // move the block if all the 3 replicas are in the same rack. if (blockMover.isOnSameRack(datanodes[0], datanodes[1]) && blockMover.isOnSameRack(datanodes[1], datanodes[2])) { Set<DatanodeInfo> excludedNodes = new HashSet<DatanodeInfo>(Arrays.asList(datanodes)); DatanodeInfo target = blockMover.chooseTargetNodes(excludedNodes); blockMover.move(locatedBlock, datanodes[2], target, excludedNodes, 3, locatedBlock.getDataProtocolVersion(), locatedBlock.getNamespaceID()); // log the move submit info if (LOG.isDebugEnabled()) { LOG.debug("Move src block : " + locatedBlock.getBlock().getBlockId() + " from " + datanodes[2] + " to target " + target + ", priority: 3. Replica info: " + datanodes[0] + ", " + datanodes[1] + ", " + datanodes[2] + ". Src file: " + srcFile.getPath()); } } } } public void checkFile(FileSystem srcFs, FileStatus srcFile, FileSystem parityFs, FileStatus parityFile, Codec codec, PolicyInfo policy) throws IOException { if (!codec.isDirRaid) { if (srcFile.getReplication() > blockMoveMinRepl) { // We only check placement for the file with 0..blockMoveMinRepl replicas. return; } } List<BlockInfo> srcLstBI = getBlockInfos(srcFs, srcFile); if (srcLstBI.size() == 0) return; if (codec.isDirRaid) { if (srcLstBI.get(0).file.getReplication() > blockMoveMinRepl) { return; } } if (srcFs.equals(parityFs)) { BlockAndDatanodeResolver resolver = new BlockAndDatanodeResolver(srcFile.getPath(), srcFs, parityFile.getPath(), parityFs); checkBlockLocations(srcLstBI, getBlockInfos(parityFs, parityFile), codec, policy, srcFile, resolver); } else { // TODO: Move blocks in two clusters separately LOG.warn("Source and parity are in different file systems. Skip"); } } LocatedFileStatus getLocatedFileStatus(FileSystem fs, Path p) throws IOException { HashMap<String, LocatedFileStatus> cache = locatedFileStatusCache.get(); LocatedFileStatus result = cache.get(p.toUri().getPath()); if (result != null) { return result; } Path parent = p.getParent(); String parentPath = parent.toUri().getPath(); //If we already did listlocatedStatus on parent path, //it means path p doesn't exist, we don't need to list again if (cache.containsKey(parentPath) && cache.get(parentPath) == null) { return null; } RemoteIterator<LocatedFileStatus> iter = fs.listLocatedStatus(parent); while (iter.hasNext()) { LocatedFileStatus stat = iter.next(); cache.put(stat.getPath().toUri().getPath(), stat); } // trick: add parent path to the cache with value = null cache.put(parentPath, null); result = cache.get(p.toUri().getPath()); // This may still return null return result; } static class BlockInfo { final BlockLocation blockLocation; final FileStatus file; BlockInfo(BlockLocation blockLocation, FileStatus file) { this.blockLocation = blockLocation; this.file = file; } String[] getNames() { try { return blockLocation.getNames(); } catch (IOException e) { return new String[] {}; } } } List<BlockInfo> getBlockInfos(FileSystem fs, FileStatus stat) throws IOException { if (stat.isDir()) { return getDirBlockInfos(fs, stat.getPath()); } else { return getBlockInfos(fs, stat.getPath(), 0, stat.getLen()); } } List<BlockInfo> getDirBlockInfos(FileSystem fs, Path dirPath) throws IOException { List<LocatedFileStatus> lfs = RaidNode.listDirectoryRaidLocatedFileStatus(conf, fs, dirPath); List<BlockInfo> result = new ArrayList<BlockInfo>(); for (LocatedFileStatus stat : lfs) { for (BlockLocation loc : stat.getBlockLocations()) { result.add(new BlockInfo(loc, stat)); } } return result; } List<BlockInfo> getBlockInfos(FileSystem fs, Path path, long start, long length) throws IOException { LocatedFileStatus stat = getLocatedFileStatus(fs, path); List<BlockInfo> result = new ArrayList<BlockInfo>(); long end = start + length; if (stat != null) { for (BlockLocation loc : stat.getBlockLocations()) { if (loc.getOffset() >= start && loc.getOffset() < end) { result.add(new BlockInfo(loc, stat)); } } } return result; } void checkBlockLocations(List<BlockInfo> srcBlocks, List<BlockInfo> parityBlocks, Codec codec, PolicyInfo policy, FileStatus srcFile, BlockAndDatanodeResolver resolver) throws IOException { if (srcBlocks == null || parityBlocks == null) { return; } int stripeLength = codec.stripeLength; int parityLength = codec.parityLength; int numBlocks = 0; int numStripes = 0; numBlocks = srcBlocks.size(); numStripes = (int) RaidNode.numStripes(numBlocks, stripeLength); Map<String, Integer> nodeToNumBlocks = new HashMap<String, Integer>(); Map<DatanodeInfo, Integer> rackToNumBlocks = new HashMap<DatanodeInfo, Integer>(); Set<String> nodesInThisStripe = new HashSet<String>(); for (int stripeIndex = 0; stripeIndex < numStripes; ++stripeIndex) { List<BlockInfo> stripeBlocks = getStripeBlocks(stripeIndex, srcBlocks, stripeLength, parityBlocks, parityLength); countBlocksOnEachNode(stripeBlocks, nodeToNumBlocks, nodesInThisStripe); countBlocksOnEachRack(nodeToNumBlocks, rackToNumBlocks, resolver); logBadFile(nodeToNumBlocks, stripeIndex, parityLength, srcFile); updateBlockPlacementHistogram(nodeToNumBlocks, rackToNumBlocks, blockHistograms.get(codec.id), blockHistogramsPerRack.get(codec.id)); submitBlockMoves(srcFile, stripeIndex, policy, nodeToNumBlocks, stripeBlocks, nodesInThisStripe, resolver); } } private static void logBadFile(Map<String, Integer> nodeToNumBlocks, int stripeIndex, int parityLength, FileStatus srcFile) { int max = 0; for (Integer n : nodeToNumBlocks.values()) { if (max < n) { max = n; } } int maxNeighborBlocks = max - 1; if (maxNeighborBlocks >= parityLength) { LOG.warn("Bad placement found. file:" + srcFile.getPath() + " stripeIndex " + stripeIndex + " neighborBlocks:" + maxNeighborBlocks + " parityLength:" + parityLength); } } private static List<BlockInfo> getStripeBlocks(int stripeIndex, List<BlockInfo> srcBlocks, int stripeLength, List<BlockInfo> parityBlocks, int parityLength) { List<BlockInfo> stripeBlocks = new ArrayList<BlockInfo>(); // Adding source blocks int stripeStart = stripeLength * stripeIndex; int stripeEnd = Math.min(stripeStart + stripeLength, srcBlocks.size()); if (stripeStart < stripeEnd) { stripeBlocks.addAll(srcBlocks.subList(stripeStart, stripeEnd)); } // Adding parity blocks stripeStart = parityLength * stripeIndex; stripeEnd = Math.min(stripeStart + parityLength, parityBlocks.size()); if (stripeStart < stripeEnd) { stripeBlocks.addAll(parityBlocks.subList(stripeStart, stripeEnd)); } return stripeBlocks; } static void countBlocksOnEachNode(List<BlockInfo> stripeBlocks, Map<String, Integer> nodeToNumBlocks, Set<String> nodesInThisStripe) throws IOException { nodeToNumBlocks.clear(); nodesInThisStripe.clear(); for (BlockInfo block : stripeBlocks) { for (String node : block.getNames()) { Integer n = nodeToNumBlocks.get(node); if (n == null) { n = 0; } nodeToNumBlocks.put(node, n + 1); nodesInThisStripe.add(node); } } } private void countBlocksOnEachRack(Map<String, Integer> nodeToNumBlocks, Map<DatanodeInfo, Integer> rackToNumBlocks, BlockAndDatanodeResolver resolver) throws IOException { rackToNumBlocks.clear(); // calculate the number of blocks on each rack. for (String node : nodeToNumBlocks.keySet()) { DatanodeInfo nodeInfo = resolver.getDatanodeInfo(node); if (nodeInfo == null) { continue; } int n = nodeToNumBlocks.get(node); boolean foundOnSameRack = false; for (DatanodeInfo nodeOnRack : rackToNumBlocks.keySet()) { if (blockMover.isOnSameRack(nodeInfo, nodeOnRack)) { rackToNumBlocks.put(nodeOnRack, rackToNumBlocks.get(nodeOnRack) + n); foundOnSameRack = true; break; } } if (!foundOnSameRack) { Integer v = rackToNumBlocks.get(nodeInfo); if (v == null) { v = 0; } rackToNumBlocks.put(nodeInfo, n + v); } } } private void updateBlockPlacementHistogram(Map<String, Integer> nodeToNumBlocks, Map<DatanodeInfo, Integer> rackToNumBlocks, Map<Integer, Long> blockHistogram, Map<Integer, Long> blockHistogramPerRack) { for (Integer numBlocks : nodeToNumBlocks.values()) { Long n = blockHistogram.get(numBlocks - 1); if (n == null) { n = 0L; } // Number of neighbor blocks to number of blocks blockHistogram.put(numBlocks - 1, n + 1); } for (Integer numBlocks : rackToNumBlocks.values()) { Long n = blockHistogramPerRack.get(numBlocks - 1); if (n == null) { n = 0L; } // Number of neighbor blocks to number of blocks blockHistogramPerRack.put(numBlocks - 1, n + 1); } } private void submitBlockMoves(FileStatus srcFile, int stripeIndex, PolicyInfo policy, Map<String, Integer> nodeToNumBlocks, List<BlockInfo> stripeBlocks, Set<String> excludedNodes, BlockAndDatanodeResolver resolver) throws IOException { if (!shouldSubmitMove(policy, nodeToNumBlocks, stripeBlocks)) { LOG.warn("We skip the block movement for " + srcFile + ", stripe index " + stripeIndex); return; } // Initialize resolver for (BlockInfo block : stripeBlocks) { resolver.initialize(block.file.getPath(), resolver.srcFs); } Set<DatanodeInfo> excludedDatanodes = new HashSet<DatanodeInfo>(); for (String name : excludedNodes) { excludedDatanodes.add(resolver.getDatanodeInfo(name)); } Map<String, Integer> numBlocksOnSameRack = getNodeToNumBlocksOnSameRack(nodeToNumBlocks, resolver); Set<String> processedNode = new HashSet<String>(); // For all the nodes/racks that has more than 2 blocks, find and move the blocks // so that there are only one block left on this node. for (String node : nodeToNumBlocks.keySet()) { int numBlocks = numBlocksOnSameRack.get(node) - 1; if (processedNode.contains(node) || numBlocks == 0) { continue; } DatanodeInfo datanode = resolver.getDatanodeInfo(node); if (datanode == null) { LOG.warn("Couldn't find information for " + node + " in resolver"); continue; } boolean skip = true; for (BlockInfo block : stripeBlocks) { for (String otherNode : block.getNames()) { DatanodeInfo replicaNode = resolver.getDatanodeInfo(otherNode); if (node.equals(otherNode) || blockMover.isOnSameRack(datanode, replicaNode)) { if (skip) { // leave the first block where it is skip = false; continue; } int priority = numBlocks; LocatedBlockWithMetaInfo lb = resolver.getLocatedBlock(block); processedNode.add(otherNode); DatanodeInfo target = blockMover.chooseTargetNodes(excludedDatanodes); excludedDatanodes.add(target); if (lb != null) { blockMover.move(lb, replicaNode, target, excludedDatanodes, priority, lb.getDataProtocolVersion(), lb.getNamespaceID()); // log the move submit info if (LOG.isDebugEnabled()) { String stripeStr = getStripeStr(srcFile, stripeBlocks, resolver); LOG.debug("Move block : " + lb.getBlock().getBlockId() + " from " + replicaNode + " to " + target + ", priority:" + priority + ". Stripe info: " + stripeStr); } } } } } } } private String getStripeStr(FileStatus srcFile, List<BlockInfo> stripeBlocks, BlockAndDatanodeResolver resolver) throws IOException { StringBuilder sb = new StringBuilder(); sb.append("File: " + srcFile.getPath() + ", "); for (BlockInfo block : stripeBlocks) { LocatedBlockWithMetaInfo lb = resolver.getLocatedBlock(block); sb.append("Block: " + lb.getBlock().getBlockId() + ", nodes: "); for (DatanodeInfo node : lb.getLocations()) { sb.append(node).append(","); } } return sb.toString(); } /** * We will not submit more block move if the namenode hasn't deleted the * over replicated blocks yet. */ private boolean shouldSubmitMove(PolicyInfo policy, Map<String, Integer> nodeToNumBlocks, List<BlockInfo> stripeBlocks) { if (policy == null) { return true; } int targetRepl = Integer.parseInt(policy.getProperty("targetReplication")); int parityRepl = Integer.parseInt(policy.getProperty("metaReplication")); Codec codec = Codec.getCodec(policy.getCodecId()); int numParityBlks = codec.parityLength; int numSrcBlks = stripeBlocks.size() - numParityBlks; int expectNumReplicas = numSrcBlks * targetRepl + numParityBlks * parityRepl; int actualNumReplicas = 0; for (int num : nodeToNumBlocks.values()) { actualNumReplicas += num; } if (actualNumReplicas != expectNumReplicas) { String msg = "Expected number of replicas in the stripe: " + expectNumReplicas + ", but actual number is: " + actualNumReplicas + ". "; if (stripeBlocks.size() > 0) { msg += "filePath: " + stripeBlocks.get(0).file.getPath(); } LOG.warn(msg); } return actualNumReplicas == expectNumReplicas; } private Map<String, Integer> getNodeToNumBlocksOnSameRack(Map<String, Integer> nodeToNumBlocks, BlockAndDatanodeResolver resolver) throws IOException { Map<String, Integer> blocksOnSameRack = new HashMap<String, Integer>(); for (Entry<String, Integer> e : nodeToNumBlocks.entrySet()) { int n = e.getValue(); for (Entry<String, Integer> e1 : nodeToNumBlocks.entrySet()) { if (e.getKey().equals(e1.getKey())) { continue; } if (blockMover.isOnSameRack(resolver.getDatanodeInfo(e.getKey()), resolver.getDatanodeInfo(e1.getKey()))) { n += e1.getValue(); } } blocksOnSameRack.put(e.getKey(), n); } return blocksOnSameRack; } /** * Report the placement histogram to {@link RaidNodeMetrics}. This should only * be called right after a complete parity file traversal is done. */ public void clearAndReport() { synchronized (metrics) { for (Codec codec : Codec.getCodecs()) { String id = codec.id; int extra = 0; Map<Integer, MetricsLongValue> codecStatsMap = metrics.codecToMisplacedBlocks.get(id); // Reset the values. for (Entry<Integer, MetricsLongValue> e : codecStatsMap.entrySet()) { e.getValue().set(0); } for (Entry<Integer, Long> e : blockHistograms.get(id).entrySet()) { if (e.getKey() < RaidNodeMetrics.MAX_MONITORED_MISPLACED_BLOCKS - 1) { MetricsLongValue v = codecStatsMap.get(e.getKey()); v.set(e.getValue()); } else { extra += e.getValue(); } } MetricsLongValue v = codecStatsMap.get(RaidNodeMetrics.MAX_MONITORED_MISPLACED_BLOCKS - 1); v.set(extra); } } lastBlockHistograms = blockHistograms; lastBlockHistogramsPerRack = blockHistogramsPerRack; lastUpdateFinishTime = RaidNode.now(); lastUpdateUsedTime = lastUpdateFinishTime - lastUpdateStartTime; LOG.info("Reporting metrices:\n" + toString()); createEmptyHistograms(); } @Override public String toString() { if (lastBlockHistograms == null || lastBlockHistogramsPerRack == null) { return "Not available"; } String result = ""; for (Codec codec : Codec.getCodecs()) { String code = codec.id; Map<Integer, Long> histo = lastBlockHistograms.get(code); result += code + " Blocks\n"; List<Integer> neighbors = new ArrayList<Integer>(); neighbors.addAll(histo.keySet()); Collections.sort(neighbors); for (Integer i : neighbors) { Long numBlocks = histo.get(i); result += i + " co-localted blocks:" + numBlocks + "\n"; } } result += "\n"; for (Codec codec : Codec.getCodecs()) { String code = codec.id; Map<Integer, Long> histo = lastBlockHistogramsPerRack.get(code); result += code + " Blocks\n"; List<Integer> neighbors = new ArrayList<Integer>(); neighbors.addAll(histo.keySet()); Collections.sort(neighbors); for (Integer i : neighbors) { Long numBlocks = histo.get(i); result += i + " rack co-localted blocks:" + numBlocks + "\n"; } } return result; } public String htmlTable() { return htmlTable(lastBlockHistograms); } public String htmlTablePerRack() { return htmlTable(lastBlockHistogramsPerRack); } public String htmlTable(Map<String, Map<Integer, Long>> lastBlockHistograms) { if (lastBlockHistograms == null) { return "Not available"; } int max = computeMaxColocatedBlocks(lastBlockHistograms); String head = ""; for (int i = 0; i <= max; ++i) { head += JspUtils.td(i + ""); } head = JspUtils.tr(JspUtils.td("CODE") + head); String result = head; for (Codec codec : Codec.getCodecs()) { String code = codec.id; String row = JspUtils.td(code); Map<Integer, Long> histo = lastBlockHistograms.get(code); for (int i = 0; i <= max; ++i) { Long numBlocks = histo.get(i); numBlocks = numBlocks == null ? 0 : numBlocks; row += JspUtils.td(StringUtils.humanReadableInt(numBlocks)); } row = JspUtils.tr(row); result += row; } return JspUtils.table(result); } public long lastUpdateTime() { return lastUpdateFinishTime; } public long lastUpdateUsedTime() { return lastUpdateUsedTime; } private int computeMaxColocatedBlocks(Map<String, Map<Integer, Long>> lastBlockHistograms) { int max = 0; for (Codec codec : Codec.getCodecs()) { String code = codec.id; Map<Integer, Long> histo = lastBlockHistograms.get(code); for (Integer i : histo.keySet()) { max = Math.max(i, max); } } return max; } /** * Translates {@link BlockLocation} to {@link LocatedBlockLocation} and * Datanode host:port to {@link DatanodeInfo} */ static class BlockAndDatanodeResolver { final Path src; final FileSystem srcFs; final Path parity; final FileSystem parityFs; private boolean inited = false; private Map<String, DatanodeInfo> nameToDatanodeInfo = new HashMap<String, DatanodeInfo>(); private Map<Path, Map<Long, LocatedBlockWithMetaInfo>> pathAndOffsetToLocatedBlock = new HashMap<Path, Map<Long, LocatedBlockWithMetaInfo>>(); // For test BlockAndDatanodeResolver() { this(null, null, null, null); } // For src file only checking BlockAndDatanodeResolver(Path src, FileSystem srcFs) { this(src, srcFs, null, null); } BlockAndDatanodeResolver(Path src, FileSystem srcFs, Path parity, FileSystem parityFs) { this.src = src; this.srcFs = srcFs; this.parity = parity; this.parityFs = parityFs; } public LocatedBlockWithMetaInfo getLocatedBlock(BlockInfo blk) throws IOException { checkParityInitialized(); initialize(blk.file.getPath(), srcFs); Map<Long, LocatedBlockWithMetaInfo> offsetToLocatedBlock = pathAndOffsetToLocatedBlock .get(blk.file.getPath()); if (offsetToLocatedBlock != null) { LocatedBlockWithMetaInfo lb = offsetToLocatedBlock.get(blk.blockLocation.getOffset()); if (lb != null) { return lb; } } // This should not happen throw new IOException("Cannot find the " + LocatedBlock.class + " for the block in file:" + blk.file.getPath() + " offset:" + blk.blockLocation.getOffset()); } public DatanodeInfo getDatanodeInfo(String name) throws IOException { checkParityInitialized(); return nameToDatanodeInfo.get(name); } private void checkParityInitialized() throws IOException { if (parity == null || parityFs == null) { return; } if (inited) { return; } initialize(parity, parityFs); inited = true; } public void initialize(Path path, FileSystem fs) throws IOException { if (pathAndOffsetToLocatedBlock.containsKey(path)) { return; } VersionedLocatedBlocks pathLbs = getLocatedBlocks(path, fs); pathAndOffsetToLocatedBlock.put(path, createOffsetToLocatedBlockMap(pathLbs)); for (LocatedBlocks lbs : Arrays.asList(pathLbs)) { for (LocatedBlock lb : lbs.getLocatedBlocks()) { for (DatanodeInfo dn : lb.getLocations()) { nameToDatanodeInfo.put(dn.getName(), dn); } } } } private Map<Long, LocatedBlockWithMetaInfo> createOffsetToLocatedBlockMap(VersionedLocatedBlocks lbs) { Map<Long, LocatedBlockWithMetaInfo> result = new HashMap<Long, LocatedBlockWithMetaInfo>(); if (lbs instanceof LocatedBlocksWithMetaInfo) { LocatedBlocksWithMetaInfo lbsm = (LocatedBlocksWithMetaInfo) lbs; for (LocatedBlock lb : lbs.getLocatedBlocks()) { result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(lb.getBlock(), lb.getLocations(), lb.getStartOffset(), lbsm.getDataProtocolVersion(), lbsm.getNamespaceID(), lbsm.getMethodFingerPrint())); } } else { for (LocatedBlock lb : lbs.getLocatedBlocks()) { result.put(lb.getStartOffset(), new LocatedBlockWithMetaInfo(lb.getBlock(), lb.getLocations(), lb.getStartOffset(), lbs.getDataProtocolVersion(), 0, 0)); } } return result; } private VersionedLocatedBlocks getLocatedBlocks(Path file, FileSystem fs) throws IOException { if (!(fs instanceof DistributedFileSystem)) { throw new IOException( "Cannot obtain " + LocatedBlocks.class + " from " + fs.getClass().getSimpleName()); } DistributedFileSystem dfs = (DistributedFileSystem) fs; if (DFSClient.isMetaInfoSuppoted(dfs.getClient().namenodeProtocolProxy)) { LocatedBlocksWithMetaInfo lbwmi = dfs.getClient().namenode .openAndFetchMetaInfo(file.toUri().getPath(), 0, Long.MAX_VALUE); dfs.getClient().getNewNameNodeIfNeeded(lbwmi.getMethodFingerPrint()); return lbwmi; } return dfs.getClient().namenode.open(file.toUri().getPath(), 0, Long.MAX_VALUE); } } }